opensim-development – Blame information for rev 1
?pathlinks?
Rev | Author | Line No. | Line |
---|---|---|---|
1 | eva | 1 | /* |
2 | * Copyright (c) Contributors, http://opensimulator.org/ |
||
3 | * See CONTRIBUTORS.TXT for a full list of copyright holders. |
||
4 | * |
||
5 | * Redistribution and use in source and binary forms, with or without |
||
6 | * modification, are permitted provided that the following conditions are met: |
||
7 | * * Redistributions of source code must retain the above copyright |
||
8 | * notice, this list of conditions and the following disclaimer. |
||
9 | * * Redistributions in binary form must reproduce the above copyright |
||
10 | * notice, this list of conditions and the following disclaimer in the |
||
11 | * documentation and/or other materials provided with the distribution. |
||
12 | * * Neither the name of the OpenSimulator Project nor the |
||
13 | * names of its contributors may be used to endorse or promote products |
||
14 | * derived from this software without specific prior written permission. |
||
15 | * |
||
16 | * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY |
||
17 | * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
||
18 | * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE |
||
19 | * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY |
||
20 | * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES |
||
21 | * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
||
22 | * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND |
||
23 | * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
||
24 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS |
||
25 | * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
||
26 | */ |
||
27 | |||
28 | using System; |
||
29 | using System.Collections.Generic; |
||
30 | using System.Linq; |
||
31 | using System.Threading; |
||
32 | using log4net; |
||
33 | |||
34 | namespace OpenSim.Framework.Monitoring |
||
35 | { |
||
36 | /// <summary> |
||
37 | /// Manages launching threads and keeping watch over them for timeouts |
||
38 | /// </summary> |
||
39 | public static class Watchdog |
||
40 | { |
||
41 | /// <summary>Timer interval in milliseconds for the watchdog timer</summary> |
||
42 | public const double WATCHDOG_INTERVAL_MS = 2500.0d; |
||
43 | |||
44 | /// <summary>Default timeout in milliseconds before a thread is considered dead</summary> |
||
45 | public const int DEFAULT_WATCHDOG_TIMEOUT_MS = 5000; |
||
46 | |||
47 | [System.Diagnostics.DebuggerDisplay("{Thread.Name}")] |
||
48 | public class ThreadWatchdogInfo |
||
49 | { |
||
50 | public Thread Thread { get; private set; } |
||
51 | |||
52 | /// <summary> |
||
53 | /// Approximate tick when this thread was started. |
||
54 | /// </summary> |
||
55 | /// <remarks> |
||
56 | /// Not terribly good since this quickly wraps around. |
||
57 | /// </remarks> |
||
58 | public int FirstTick { get; private set; } |
||
59 | |||
60 | /// <summary> |
||
61 | /// Last time this heartbeat update was invoked |
||
62 | /// </summary> |
||
63 | public int LastTick { get; set; } |
||
64 | |||
65 | /// <summary> |
||
66 | /// Number of milliseconds before we notify that the thread is having a problem. |
||
67 | /// </summary> |
||
68 | public int Timeout { get; set; } |
||
69 | |||
70 | /// <summary> |
||
71 | /// Is this thread considered timed out? |
||
72 | /// </summary> |
||
73 | public bool IsTimedOut { get; set; } |
||
74 | |||
75 | /// <summary> |
||
76 | /// Will this thread trigger the alarm function if it has timed out? |
||
77 | /// </summary> |
||
78 | public bool AlarmIfTimeout { get; set; } |
||
79 | |||
80 | /// <summary> |
||
81 | /// Method execute if alarm goes off. If null then no alarm method is fired. |
||
82 | /// </summary> |
||
83 | public Func<string> AlarmMethod { get; set; } |
||
84 | |||
85 | public ThreadWatchdogInfo(Thread thread, int timeout) |
||
86 | { |
||
87 | Thread = thread; |
||
88 | Timeout = timeout; |
||
89 | FirstTick = Environment.TickCount & Int32.MaxValue; |
||
90 | LastTick = FirstTick; |
||
91 | } |
||
92 | |||
93 | public ThreadWatchdogInfo(ThreadWatchdogInfo previousTwi) |
||
94 | { |
||
95 | Thread = previousTwi.Thread; |
||
96 | FirstTick = previousTwi.FirstTick; |
||
97 | LastTick = previousTwi.LastTick; |
||
98 | Timeout = previousTwi.Timeout; |
||
99 | IsTimedOut = previousTwi.IsTimedOut; |
||
100 | AlarmIfTimeout = previousTwi.AlarmIfTimeout; |
||
101 | AlarmMethod = previousTwi.AlarmMethod; |
||
102 | } |
||
103 | } |
||
104 | |||
105 | /// <summary> |
||
106 | /// This event is called whenever a tracked thread is |
||
107 | /// stopped or has not called UpdateThread() in time< |
||
108 | /// /summary> |
||
109 | public static event Action<ThreadWatchdogInfo> OnWatchdogTimeout; |
||
110 | |||
111 | /// <summary> |
||
112 | /// Is this watchdog active? |
||
113 | /// </summary> |
||
114 | public static bool Enabled |
||
115 | { |
||
116 | get { return m_enabled; } |
||
117 | set |
||
118 | { |
||
119 | // m_log.DebugFormat("[MEMORY WATCHDOG]: Setting MemoryWatchdog.Enabled to {0}", value); |
||
120 | |||
121 | if (value == m_enabled) |
||
122 | return; |
||
123 | |||
124 | m_enabled = value; |
||
125 | |||
126 | if (m_enabled) |
||
127 | { |
||
128 | // Set now so we don't get alerted on the first run |
||
129 | LastWatchdogThreadTick = Environment.TickCount & Int32.MaxValue; |
||
130 | } |
||
131 | |||
132 | m_watchdogTimer.Enabled = m_enabled; |
||
133 | } |
||
134 | } |
||
135 | private static bool m_enabled; |
||
136 | |||
137 | private static readonly ILog m_log = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType); |
||
138 | private static Dictionary<int, ThreadWatchdogInfo> m_threads; |
||
139 | private static System.Timers.Timer m_watchdogTimer; |
||
140 | |||
141 | /// <summary> |
||
142 | /// Last time the watchdog thread ran. |
||
143 | /// </summary> |
||
144 | /// <remarks> |
||
145 | /// Should run every WATCHDOG_INTERVAL_MS |
||
146 | /// </remarks> |
||
147 | public static int LastWatchdogThreadTick { get; private set; } |
||
148 | |||
149 | static Watchdog() |
||
150 | { |
||
151 | m_threads = new Dictionary<int, ThreadWatchdogInfo>(); |
||
152 | m_watchdogTimer = new System.Timers.Timer(WATCHDOG_INTERVAL_MS); |
||
153 | m_watchdogTimer.AutoReset = false; |
||
154 | m_watchdogTimer.Elapsed += WatchdogTimerElapsed; |
||
155 | } |
||
156 | |||
157 | /// <summary> |
||
158 | /// Start a new thread that is tracked by the watchdog timer. |
||
159 | /// </summary> |
||
160 | /// <param name="start">The method that will be executed in a new thread</param> |
||
161 | /// <param name="name">A name to give to the new thread</param> |
||
162 | /// <param name="priority">Priority to run the thread at</param> |
||
163 | /// <param name="isBackground">True to run this thread as a background thread, otherwise false</param> |
||
164 | /// <param name="alarmIfTimeout">Trigger an alarm function is we have timed out</param> |
||
165 | /// <returns>The newly created Thread object</returns> |
||
166 | public static Thread StartThread( |
||
167 | ThreadStart start, string name, ThreadPriority priority, bool isBackground, bool alarmIfTimeout) |
||
168 | { |
||
169 | return StartThread(start, name, priority, isBackground, alarmIfTimeout, null, DEFAULT_WATCHDOG_TIMEOUT_MS); |
||
170 | } |
||
171 | |||
172 | /// <summary> |
||
173 | /// Start a new thread that is tracked by the watchdog timer |
||
174 | /// </summary> |
||
175 | /// <param name="start">The method that will be executed in a new thread</param> |
||
176 | /// <param name="name">A name to give to the new thread</param> |
||
177 | /// <param name="priority">Priority to run the thread at</param> |
||
178 | /// <param name="isBackground">True to run this thread as a background |
||
179 | /// thread, otherwise false</param> |
||
180 | /// <param name="alarmIfTimeout">Trigger an alarm function is we have timed out</param> |
||
181 | /// <param name="alarmMethod"> |
||
182 | /// Alarm method to call if alarmIfTimeout is true and there is a timeout. |
||
183 | /// Normally, this will just return some useful debugging information. |
||
184 | /// </param> |
||
185 | /// <param name="timeout">Number of milliseconds to wait until we issue a warning about timeout.</param> |
||
186 | /// <returns>The newly created Thread object</returns> |
||
187 | public static Thread StartThread( |
||
188 | ThreadStart start, string name, ThreadPriority priority, bool isBackground, |
||
189 | bool alarmIfTimeout, Func<string> alarmMethod, int timeout) |
||
190 | { |
||
191 | Thread thread = new Thread(start); |
||
192 | thread.Name = name; |
||
193 | thread.Priority = priority; |
||
194 | thread.IsBackground = isBackground; |
||
195 | |||
196 | ThreadWatchdogInfo twi |
||
197 | = new ThreadWatchdogInfo(thread, timeout) |
||
198 | { AlarmIfTimeout = alarmIfTimeout, AlarmMethod = alarmMethod }; |
||
199 | |||
200 | m_log.DebugFormat( |
||
201 | "[WATCHDOG]: Started tracking thread {0}, ID {1}", twi.Thread.Name, twi.Thread.ManagedThreadId); |
||
202 | |||
203 | lock (m_threads) |
||
204 | m_threads.Add(twi.Thread.ManagedThreadId, twi); |
||
205 | |||
206 | thread.Start(); |
||
207 | |||
208 | return thread; |
||
209 | } |
||
210 | |||
211 | /// <summary> |
||
212 | /// Marks the current thread as alive |
||
213 | /// </summary> |
||
214 | public static void UpdateThread() |
||
215 | { |
||
216 | UpdateThread(Thread.CurrentThread.ManagedThreadId); |
||
217 | } |
||
218 | |||
219 | /// <summary> |
||
220 | /// Stops watchdog tracking on the current thread |
||
221 | /// </summary> |
||
222 | /// <returns> |
||
223 | /// True if the thread was removed from the list of tracked |
||
224 | /// threads, otherwise false |
||
225 | /// </returns> |
||
226 | public static bool RemoveThread() |
||
227 | { |
||
228 | return RemoveThread(Thread.CurrentThread.ManagedThreadId); |
||
229 | } |
||
230 | |||
231 | private static bool RemoveThread(int threadID) |
||
232 | { |
||
233 | lock (m_threads) |
||
234 | { |
||
235 | ThreadWatchdogInfo twi; |
||
236 | if (m_threads.TryGetValue(threadID, out twi)) |
||
237 | { |
||
238 | m_log.DebugFormat( |
||
239 | "[WATCHDOG]: Removing thread {0}, ID {1}", twi.Thread.Name, twi.Thread.ManagedThreadId); |
||
240 | |||
241 | m_threads.Remove(threadID); |
||
242 | |||
243 | return true; |
||
244 | } |
||
245 | else |
||
246 | { |
||
247 | m_log.WarnFormat( |
||
248 | "[WATCHDOG]: Requested to remove thread with ID {0} but this is not being monitored", threadID); |
||
249 | |||
250 | return false; |
||
251 | } |
||
252 | } |
||
253 | } |
||
254 | |||
255 | public static bool AbortThread(int threadID) |
||
256 | { |
||
257 | lock (m_threads) |
||
258 | { |
||
259 | if (m_threads.ContainsKey(threadID)) |
||
260 | { |
||
261 | ThreadWatchdogInfo twi = m_threads[threadID]; |
||
262 | twi.Thread.Abort(); |
||
263 | RemoveThread(threadID); |
||
264 | |||
265 | return true; |
||
266 | } |
||
267 | else |
||
268 | { |
||
269 | return false; |
||
270 | } |
||
271 | } |
||
272 | } |
||
273 | |||
274 | private static void UpdateThread(int threadID) |
||
275 | { |
||
276 | ThreadWatchdogInfo threadInfo; |
||
277 | |||
278 | // Although TryGetValue is not a thread safe operation, we use a try/catch here instead |
||
279 | // of a lock for speed. Adding/removing threads is a very rare operation compared to |
||
280 | // UpdateThread(), and a single UpdateThread() failure here and there won't break |
||
281 | // anything |
||
282 | try |
||
283 | { |
||
284 | if (m_threads.TryGetValue(threadID, out threadInfo)) |
||
285 | { |
||
286 | threadInfo.LastTick = Environment.TickCount & Int32.MaxValue; |
||
287 | threadInfo.IsTimedOut = false; |
||
288 | } |
||
289 | else |
||
290 | { |
||
291 | m_log.WarnFormat("[WATCHDOG]: Asked to update thread {0} which is not being monitored", threadID); |
||
292 | } |
||
293 | } |
||
294 | catch { } |
||
295 | } |
||
296 | |||
297 | /// <summary> |
||
298 | /// Get currently watched threads for diagnostic purposes |
||
299 | /// </summary> |
||
300 | /// <returns></returns> |
||
301 | public static ThreadWatchdogInfo[] GetThreadsInfo() |
||
302 | { |
||
303 | lock (m_threads) |
||
304 | return m_threads.Values.ToArray(); |
||
305 | } |
||
306 | |||
307 | /// <summary> |
||
308 | /// Return the current thread's watchdog info. |
||
309 | /// </summary> |
||
310 | /// <returns>The watchdog info. null if the thread isn't being monitored.</returns> |
||
311 | public static ThreadWatchdogInfo GetCurrentThreadInfo() |
||
312 | { |
||
313 | lock (m_threads) |
||
314 | { |
||
315 | if (m_threads.ContainsKey(Thread.CurrentThread.ManagedThreadId)) |
||
316 | return m_threads[Thread.CurrentThread.ManagedThreadId]; |
||
317 | } |
||
318 | |||
319 | return null; |
||
320 | } |
||
321 | |||
322 | /// <summary> |
||
323 | /// Check watched threads. Fire alarm if appropriate. |
||
324 | /// </summary> |
||
325 | /// <param name="sender"></param> |
||
326 | /// <param name="e"></param> |
||
327 | private static void WatchdogTimerElapsed(object sender, System.Timers.ElapsedEventArgs e) |
||
328 | { |
||
329 | int now = Environment.TickCount & Int32.MaxValue; |
||
330 | int msElapsed = now - LastWatchdogThreadTick; |
||
331 | |||
332 | if (msElapsed > WATCHDOG_INTERVAL_MS * 2) |
||
333 | m_log.WarnFormat( |
||
334 | "[WATCHDOG]: {0} ms since Watchdog last ran. Interval should be approximately {1} ms", |
||
335 | msElapsed, WATCHDOG_INTERVAL_MS); |
||
336 | |||
337 | LastWatchdogThreadTick = Environment.TickCount & Int32.MaxValue; |
||
338 | |||
339 | Action<ThreadWatchdogInfo> callback = OnWatchdogTimeout; |
||
340 | |||
341 | if (callback != null) |
||
342 | { |
||
343 | List<ThreadWatchdogInfo> callbackInfos = null; |
||
344 | |||
345 | lock (m_threads) |
||
346 | { |
||
347 | foreach (ThreadWatchdogInfo threadInfo in m_threads.Values) |
||
348 | { |
||
349 | if (threadInfo.Thread.ThreadState == ThreadState.Stopped) |
||
350 | { |
||
351 | RemoveThread(threadInfo.Thread.ManagedThreadId); |
||
352 | |||
353 | if (callbackInfos == null) |
||
354 | callbackInfos = new List<ThreadWatchdogInfo>(); |
||
355 | |||
356 | callbackInfos.Add(threadInfo); |
||
357 | } |
||
358 | else if (!threadInfo.IsTimedOut && now - threadInfo.LastTick >= threadInfo.Timeout) |
||
359 | { |
||
360 | threadInfo.IsTimedOut = true; |
||
361 | |||
362 | if (threadInfo.AlarmIfTimeout) |
||
363 | { |
||
364 | if (callbackInfos == null) |
||
365 | callbackInfos = new List<ThreadWatchdogInfo>(); |
||
366 | |||
367 | // Send a copy of the watchdog info to prevent race conditions where the watchdog |
||
368 | // thread updates the monitoring info after an alarm has been sent out. |
||
369 | callbackInfos.Add(new ThreadWatchdogInfo(threadInfo)); |
||
370 | } |
||
371 | } |
||
372 | } |
||
373 | } |
||
374 | |||
375 | if (callbackInfos != null) |
||
376 | foreach (ThreadWatchdogInfo callbackInfo in callbackInfos) |
||
377 | callback(callbackInfo); |
||
378 | } |
||
379 | |||
380 | if (MemoryWatchdog.Enabled) |
||
381 | MemoryWatchdog.Update(); |
||
382 | |||
383 | ChecksManager.CheckChecks(); |
||
384 | StatsManager.RecordStats(); |
||
385 | |||
386 | m_watchdogTimer.Start(); |
||
387 | } |
||
388 | } |
||
389 | } |