2 // ========================================================================
3 // Copyright (c) 1995-2014 Mort Bay Consulting Pty. Ltd.
4 // ------------------------------------------------------------------------
5 // All rights reserved. This program and the accompanying materials
6 // are made available under the terms of the Eclipse Public License v1.0
7 // and Apache License v2.0 which accompanies this distribution.
9 // The Eclipse Public License is available at
10 // http://www.eclipse.org/legal/epl-v10.html
12 // The Apache License v2.0 is available at
13 // http://www.opensource.org/licenses/apache2.0.php
15 // You may elect to redistribute this code under either of these licenses.
16 // ========================================================================
19 package org.eclipse.jetty.server;
21 import java.util.Arrays;
22 import java.util.Collection;
23 import java.util.Collections;
24 import java.util.concurrent.Executor;
25 import java.util.concurrent.TimeUnit;
26 import java.util.concurrent.atomic.AtomicBoolean;
28 import org.eclipse.jetty.io.EndPoint;
29 import org.eclipse.jetty.util.annotation.ManagedAttribute;
30 import org.eclipse.jetty.util.annotation.ManagedObject;
31 import org.eclipse.jetty.util.annotation.Name;
32 import org.eclipse.jetty.util.component.AbstractLifeCycle;
33 import org.eclipse.jetty.util.log.Log;
34 import org.eclipse.jetty.util.log.Logger;
35 import org.eclipse.jetty.util.thread.ScheduledExecutorScheduler;
36 import org.eclipse.jetty.util.thread.Scheduler;
37 import org.eclipse.jetty.util.thread.ThreadPool;
40 /* ------------------------------------------------------------ */
41 /** A monitor for low resources
42 * <p>An instance of this class will monitor all the connectors of a server (or a set of connectors
43 * configured with {@link #setMonitoredConnectors(Collection)}) for a low resources state.
44 * Low resources can be detected by:<ul>
45 * <li>{@link ThreadPool#isLowOnThreads()} if {@link Connector#getExecutor()} is
46 * an instance of {@link ThreadPool} and {@link #setMonitorThreads(boolean)} is true.<li>
47 * <li>If {@link #setMaxMemory(long)} is non zero then low resources is detected if the JVMs
48 * {@link Runtime} instance has {@link Runtime#totalMemory()} minus {@link Runtime#freeMemory()}
49 * greater than {@link #getMaxMemory()}</li>
50 * <li>If {@link #setMaxConnections(int)} is non zero then low resources is dected if the total number
51 * of connections exceeds {@link #getMaxConnections()}</li>
54 * <p>Once low resources state is detected, the cause is logged and all existing connections returned
55 * by {@link Connector#getConnectedEndPoints()} have {@link EndPoint#setIdleTimeout(long)} set
56 * to {@link #getLowResourcesIdleTimeout()}. New connections are not affected, however if the low
57 * resources state persists for more than {@link #getMaxLowResourcesTime()}, then the
58 * {@link #getLowResourcesIdleTimeout()} to all connections again. Once the low resources state is
59 * cleared, the idle timeout is reset to the connector default given by {@link Connector#getIdleTimeout()}.
62 @ManagedObject ("Monitor for low resource conditions and activate a low resource mode if detected")
63 public class LowResourceMonitor extends AbstractLifeCycle
65 private static final Logger LOG = Log.getLogger(LowResourceMonitor.class);
66 private final Server _server;
67 private Scheduler _scheduler;
68 private Connector[] _monitoredConnectors;
69 private int _period=1000;
70 private int _maxConnections;
71 private long _maxMemory;
72 private int _lowResourcesIdleTimeout=1000;
73 private int _maxLowResourcesTime=0;
74 private boolean _monitorThreads=true;
75 private final AtomicBoolean _low = new AtomicBoolean();
76 private String _cause;
77 private String _reasons;
78 private long _lowStarted;
81 private final Runnable _monitor = new Runnable()
89 _scheduler.schedule(_monitor,_period,TimeUnit.MILLISECONDS);
94 public LowResourceMonitor(@Name("server") Server server)
99 @ManagedAttribute("Are the monitored connectors low on resources?")
100 public boolean isLowOnResources()
105 @ManagedAttribute("The reason(s) the monitored connectors are low on resources")
106 public String getLowResourcesReasons()
111 @ManagedAttribute("Get the timestamp in ms since epoch that low resources state started")
112 public long getLowResourcesStarted()
117 @ManagedAttribute("The monitored connectors. If null then all server connectors are monitored")
118 public Collection<Connector> getMonitoredConnectors()
120 if (_monitoredConnectors==null)
121 return Collections.emptyList();
122 return Arrays.asList(_monitoredConnectors);
126 * @param monitoredConnectors The collections of Connectors that should be monitored for low resources.
128 public void setMonitoredConnectors(Collection<Connector> monitoredConnectors)
130 if (monitoredConnectors==null || monitoredConnectors.size()==0)
131 _monitoredConnectors=null;
133 _monitoredConnectors = monitoredConnectors.toArray(new Connector[monitoredConnectors.size()]);
136 @ManagedAttribute("The monitor period in ms")
137 public int getPeriod()
143 * @param periodMS The period in ms to monitor for low resources
145 public void setPeriod(int periodMS)
150 @ManagedAttribute("True if low available threads status is monitored")
151 public boolean getMonitorThreads()
153 return _monitorThreads;
157 * @param monitorThreads If true, check connectors executors to see if they are
158 * {@link ThreadPool} instances that are low on threads.
160 public void setMonitorThreads(boolean monitorThreads)
162 _monitorThreads = monitorThreads;
165 @ManagedAttribute("The maximum connections allowed for the monitored connectors before low resource handling is activated")
166 public int getMaxConnections()
168 return _maxConnections;
172 * @param maxConnections The maximum connections before low resources state is triggered
174 public void setMaxConnections(int maxConnections)
176 _maxConnections = maxConnections;
179 @ManagedAttribute("The maximum memory (in bytes) that can be used before low resources is triggered. Memory used is calculated as (totalMemory-freeMemory).")
180 public long getMaxMemory()
186 * @param maxMemoryBytes The maximum memory in bytes in use before low resources is triggered.
188 public void setMaxMemory(long maxMemoryBytes)
190 _maxMemory = maxMemoryBytes;
193 @ManagedAttribute("The idletimeout in ms to apply to all existing connections when low resources is detected")
194 public int getLowResourcesIdleTimeout()
196 return _lowResourcesIdleTimeout;
200 * @param lowResourcesIdleTimeoutMS The timeout in ms to apply to EndPoints when in the low resources state.
202 public void setLowResourcesIdleTimeout(int lowResourcesIdleTimeoutMS)
204 _lowResourcesIdleTimeout = lowResourcesIdleTimeoutMS;
207 @ManagedAttribute("The maximum time in ms that low resources condition can persist before lowResourcesIdleTimeout is applied to new connections as well as existing connections")
208 public int getMaxLowResourcesTime()
210 return _maxLowResourcesTime;
214 * @param maxLowResourcesTimeMS The time in milliseconds that a low resource state can persist before the low resource idle timeout is reapplied to all connections
216 public void setMaxLowResourcesTime(int maxLowResourcesTimeMS)
218 _maxLowResourcesTime = maxLowResourcesTimeMS;
222 protected void doStart() throws Exception
224 _scheduler = _server.getBean(Scheduler.class);
226 if (_scheduler==null)
228 _scheduler=new LRMScheduler();
233 _scheduler.schedule(_monitor,_period,TimeUnit.MILLISECONDS);
237 protected void doStop() throws Exception
239 if (_scheduler instanceof LRMScheduler)
244 protected Connector[] getMonitoredOrServerConnectors()
246 if (_monitoredConnectors!=null && _monitoredConnectors.length>0)
247 return _monitoredConnectors;
248 return _server.getConnectors();
251 protected void monitor()
257 for(Connector connector : getMonitoredOrServerConnectors())
259 connections+=connector.getConnectedEndPoints().size();
261 Executor executor = connector.getExecutor();
262 if (executor instanceof ThreadPool)
264 ThreadPool threadpool=(ThreadPool) executor;
265 if (_monitorThreads && threadpool.isLowOnThreads())
267 reasons=low(reasons,"Low on threads: "+threadpool);
273 if (_maxConnections>0 && connections>_maxConnections)
275 reasons=low(reasons,"Max Connections exceeded: "+connections+">"+_maxConnections);
279 long memory=Runtime.getRuntime().totalMemory()-Runtime.getRuntime().freeMemory();
280 if (_maxMemory>0 && memory>_maxMemory)
282 reasons=low(reasons,"Max memory exceeded: "+memory+">"+_maxMemory);
289 // Log the reasons if there is any change in the cause
290 if (!cause.equals(_cause))
292 LOG.warn("Low Resources: {}",reasons);
296 // Enter low resources state?
297 if (_low.compareAndSet(false,true))
300 _lowStarted=System.currentTimeMillis();
304 // Too long in low resources state?
305 if (_maxLowResourcesTime>0 && (System.currentTimeMillis()-_lowStarted)>_maxLowResourcesTime)
310 if (_low.compareAndSet(true,false))
312 LOG.info("Low Resources cleared");
321 protected void setLowResources()
323 for(Connector connector : getMonitoredOrServerConnectors())
325 for (EndPoint endPoint : connector.getConnectedEndPoints())
326 endPoint.setIdleTimeout(_lowResourcesIdleTimeout);
330 protected void clearLowResources()
332 for(Connector connector : getMonitoredOrServerConnectors())
334 for (EndPoint endPoint : connector.getConnectedEndPoints())
335 endPoint.setIdleTimeout(connector.getIdleTimeout());
339 private String low(String reasons, String newReason)
343 return reasons+", "+newReason;
347 private static class LRMScheduler extends ScheduledExecutorScheduler