opensim-development – Blame information for rev 1

Subversion Repositories:
Rev:
Rev Author Line No. Line
1 eva 1 /*
2 * Copyright (c) Contributors, http://opensimulator.org/
3 * See CONTRIBUTORS.TXT for a full list of copyright holders.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 * * Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * * Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 * * Neither the name of the OpenSimulator Project nor the
13 * names of its contributors may be used to endorse or promote products
14 * derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY
17 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY
20 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
23 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27  
28 using System;
29 using System.Collections.Generic;
30 using System.Linq;
31 using System.Threading;
32 using log4net;
33  
34 namespace OpenSim.Framework.Monitoring
35 {
36 /// <summary>
37 /// Manages launching threads and keeping watch over them for timeouts
38 /// </summary>
39 public static class Watchdog
40 {
41 /// <summary>Timer interval in milliseconds for the watchdog timer</summary>
42 public const double WATCHDOG_INTERVAL_MS = 2500.0d;
43  
44 /// <summary>Default timeout in milliseconds before a thread is considered dead</summary>
45 public const int DEFAULT_WATCHDOG_TIMEOUT_MS = 5000;
46  
47 [System.Diagnostics.DebuggerDisplay("{Thread.Name}")]
48 public class ThreadWatchdogInfo
49 {
50 public Thread Thread { get; private set; }
51  
52 /// <summary>
53 /// Approximate tick when this thread was started.
54 /// </summary>
55 /// <remarks>
56 /// Not terribly good since this quickly wraps around.
57 /// </remarks>
58 public int FirstTick { get; private set; }
59  
60 /// <summary>
61 /// Last time this heartbeat update was invoked
62 /// </summary>
63 public int LastTick { get; set; }
64  
65 /// <summary>
66 /// Number of milliseconds before we notify that the thread is having a problem.
67 /// </summary>
68 public int Timeout { get; set; }
69  
70 /// <summary>
71 /// Is this thread considered timed out?
72 /// </summary>
73 public bool IsTimedOut { get; set; }
74  
75 /// <summary>
76 /// Will this thread trigger the alarm function if it has timed out?
77 /// </summary>
78 public bool AlarmIfTimeout { get; set; }
79  
80 /// <summary>
81 /// Method execute if alarm goes off. If null then no alarm method is fired.
82 /// </summary>
83 public Func<string> AlarmMethod { get; set; }
84  
85 public ThreadWatchdogInfo(Thread thread, int timeout)
86 {
87 Thread = thread;
88 Timeout = timeout;
89 FirstTick = Environment.TickCount & Int32.MaxValue;
90 LastTick = FirstTick;
91 }
92  
93 public ThreadWatchdogInfo(ThreadWatchdogInfo previousTwi)
94 {
95 Thread = previousTwi.Thread;
96 FirstTick = previousTwi.FirstTick;
97 LastTick = previousTwi.LastTick;
98 Timeout = previousTwi.Timeout;
99 IsTimedOut = previousTwi.IsTimedOut;
100 AlarmIfTimeout = previousTwi.AlarmIfTimeout;
101 AlarmMethod = previousTwi.AlarmMethod;
102 }
103 }
104  
105 /// <summary>
106 /// This event is called whenever a tracked thread is
107 /// stopped or has not called UpdateThread() in time<
108 /// /summary>
109 public static event Action<ThreadWatchdogInfo> OnWatchdogTimeout;
110  
111 /// <summary>
112 /// Is this watchdog active?
113 /// </summary>
114 public static bool Enabled
115 {
116 get { return m_enabled; }
117 set
118 {
119 // m_log.DebugFormat("[MEMORY WATCHDOG]: Setting MemoryWatchdog.Enabled to {0}", value);
120  
121 if (value == m_enabled)
122 return;
123  
124 m_enabled = value;
125  
126 if (m_enabled)
127 {
128 // Set now so we don't get alerted on the first run
129 LastWatchdogThreadTick = Environment.TickCount & Int32.MaxValue;
130 }
131  
132 m_watchdogTimer.Enabled = m_enabled;
133 }
134 }
135 private static bool m_enabled;
136  
137 private static readonly ILog m_log = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType);
138 private static Dictionary<int, ThreadWatchdogInfo> m_threads;
139 private static System.Timers.Timer m_watchdogTimer;
140  
141 /// <summary>
142 /// Last time the watchdog thread ran.
143 /// </summary>
144 /// <remarks>
145 /// Should run every WATCHDOG_INTERVAL_MS
146 /// </remarks>
147 public static int LastWatchdogThreadTick { get; private set; }
148  
149 static Watchdog()
150 {
151 m_threads = new Dictionary<int, ThreadWatchdogInfo>();
152 m_watchdogTimer = new System.Timers.Timer(WATCHDOG_INTERVAL_MS);
153 m_watchdogTimer.AutoReset = false;
154 m_watchdogTimer.Elapsed += WatchdogTimerElapsed;
155 }
156  
157 /// <summary>
158 /// Start a new thread that is tracked by the watchdog timer.
159 /// </summary>
160 /// <param name="start">The method that will be executed in a new thread</param>
161 /// <param name="name">A name to give to the new thread</param>
162 /// <param name="priority">Priority to run the thread at</param>
163 /// <param name="isBackground">True to run this thread as a background thread, otherwise false</param>
164 /// <param name="alarmIfTimeout">Trigger an alarm function is we have timed out</param>
165 /// <returns>The newly created Thread object</returns>
166 public static Thread StartThread(
167 ThreadStart start, string name, ThreadPriority priority, bool isBackground, bool alarmIfTimeout)
168 {
169 return StartThread(start, name, priority, isBackground, alarmIfTimeout, null, DEFAULT_WATCHDOG_TIMEOUT_MS);
170 }
171  
172 /// <summary>
173 /// Start a new thread that is tracked by the watchdog timer
174 /// </summary>
175 /// <param name="start">The method that will be executed in a new thread</param>
176 /// <param name="name">A name to give to the new thread</param>
177 /// <param name="priority">Priority to run the thread at</param>
178 /// <param name="isBackground">True to run this thread as a background
179 /// thread, otherwise false</param>
180 /// <param name="alarmIfTimeout">Trigger an alarm function is we have timed out</param>
181 /// <param name="alarmMethod">
182 /// Alarm method to call if alarmIfTimeout is true and there is a timeout.
183 /// Normally, this will just return some useful debugging information.
184 /// </param>
185 /// <param name="timeout">Number of milliseconds to wait until we issue a warning about timeout.</param>
186 /// <returns>The newly created Thread object</returns>
187 public static Thread StartThread(
188 ThreadStart start, string name, ThreadPriority priority, bool isBackground,
189 bool alarmIfTimeout, Func<string> alarmMethod, int timeout)
190 {
191 Thread thread = new Thread(start);
192 thread.Name = name;
193 thread.Priority = priority;
194 thread.IsBackground = isBackground;
195  
196 ThreadWatchdogInfo twi
197 = new ThreadWatchdogInfo(thread, timeout)
198 { AlarmIfTimeout = alarmIfTimeout, AlarmMethod = alarmMethod };
199  
200 m_log.DebugFormat(
201 "[WATCHDOG]: Started tracking thread {0}, ID {1}", twi.Thread.Name, twi.Thread.ManagedThreadId);
202  
203 lock (m_threads)
204 m_threads.Add(twi.Thread.ManagedThreadId, twi);
205  
206 thread.Start();
207  
208 return thread;
209 }
210  
211 /// <summary>
212 /// Marks the current thread as alive
213 /// </summary>
214 public static void UpdateThread()
215 {
216 UpdateThread(Thread.CurrentThread.ManagedThreadId);
217 }
218  
219 /// <summary>
220 /// Stops watchdog tracking on the current thread
221 /// </summary>
222 /// <returns>
223 /// True if the thread was removed from the list of tracked
224 /// threads, otherwise false
225 /// </returns>
226 public static bool RemoveThread()
227 {
228 return RemoveThread(Thread.CurrentThread.ManagedThreadId);
229 }
230  
231 private static bool RemoveThread(int threadID)
232 {
233 lock (m_threads)
234 {
235 ThreadWatchdogInfo twi;
236 if (m_threads.TryGetValue(threadID, out twi))
237 {
238 m_log.DebugFormat(
239 "[WATCHDOG]: Removing thread {0}, ID {1}", twi.Thread.Name, twi.Thread.ManagedThreadId);
240  
241 m_threads.Remove(threadID);
242  
243 return true;
244 }
245 else
246 {
247 m_log.WarnFormat(
248 "[WATCHDOG]: Requested to remove thread with ID {0} but this is not being monitored", threadID);
249  
250 return false;
251 }
252 }
253 }
254  
255 public static bool AbortThread(int threadID)
256 {
257 lock (m_threads)
258 {
259 if (m_threads.ContainsKey(threadID))
260 {
261 ThreadWatchdogInfo twi = m_threads[threadID];
262 twi.Thread.Abort();
263 RemoveThread(threadID);
264  
265 return true;
266 }
267 else
268 {
269 return false;
270 }
271 }
272 }
273  
274 private static void UpdateThread(int threadID)
275 {
276 ThreadWatchdogInfo threadInfo;
277  
278 // Although TryGetValue is not a thread safe operation, we use a try/catch here instead
279 // of a lock for speed. Adding/removing threads is a very rare operation compared to
280 // UpdateThread(), and a single UpdateThread() failure here and there won't break
281 // anything
282 try
283 {
284 if (m_threads.TryGetValue(threadID, out threadInfo))
285 {
286 threadInfo.LastTick = Environment.TickCount & Int32.MaxValue;
287 threadInfo.IsTimedOut = false;
288 }
289 else
290 {
291 m_log.WarnFormat("[WATCHDOG]: Asked to update thread {0} which is not being monitored", threadID);
292 }
293 }
294 catch { }
295 }
296  
297 /// <summary>
298 /// Get currently watched threads for diagnostic purposes
299 /// </summary>
300 /// <returns></returns>
301 public static ThreadWatchdogInfo[] GetThreadsInfo()
302 {
303 lock (m_threads)
304 return m_threads.Values.ToArray();
305 }
306  
307 /// <summary>
308 /// Return the current thread's watchdog info.
309 /// </summary>
310 /// <returns>The watchdog info. null if the thread isn't being monitored.</returns>
311 public static ThreadWatchdogInfo GetCurrentThreadInfo()
312 {
313 lock (m_threads)
314 {
315 if (m_threads.ContainsKey(Thread.CurrentThread.ManagedThreadId))
316 return m_threads[Thread.CurrentThread.ManagedThreadId];
317 }
318  
319 return null;
320 }
321  
322 /// <summary>
323 /// Check watched threads. Fire alarm if appropriate.
324 /// </summary>
325 /// <param name="sender"></param>
326 /// <param name="e"></param>
327 private static void WatchdogTimerElapsed(object sender, System.Timers.ElapsedEventArgs e)
328 {
329 int now = Environment.TickCount & Int32.MaxValue;
330 int msElapsed = now - LastWatchdogThreadTick;
331  
332 if (msElapsed > WATCHDOG_INTERVAL_MS * 2)
333 m_log.WarnFormat(
334 "[WATCHDOG]: {0} ms since Watchdog last ran. Interval should be approximately {1} ms",
335 msElapsed, WATCHDOG_INTERVAL_MS);
336  
337 LastWatchdogThreadTick = Environment.TickCount & Int32.MaxValue;
338  
339 Action<ThreadWatchdogInfo> callback = OnWatchdogTimeout;
340  
341 if (callback != null)
342 {
343 List<ThreadWatchdogInfo> callbackInfos = null;
344  
345 lock (m_threads)
346 {
347 foreach (ThreadWatchdogInfo threadInfo in m_threads.Values)
348 {
349 if (threadInfo.Thread.ThreadState == ThreadState.Stopped)
350 {
351 RemoveThread(threadInfo.Thread.ManagedThreadId);
352  
353 if (callbackInfos == null)
354 callbackInfos = new List<ThreadWatchdogInfo>();
355  
356 callbackInfos.Add(threadInfo);
357 }
358 else if (!threadInfo.IsTimedOut && now - threadInfo.LastTick >= threadInfo.Timeout)
359 {
360 threadInfo.IsTimedOut = true;
361  
362 if (threadInfo.AlarmIfTimeout)
363 {
364 if (callbackInfos == null)
365 callbackInfos = new List<ThreadWatchdogInfo>();
366  
367 // Send a copy of the watchdog info to prevent race conditions where the watchdog
368 // thread updates the monitoring info after an alarm has been sent out.
369 callbackInfos.Add(new ThreadWatchdogInfo(threadInfo));
370 }
371 }
372 }
373 }
374  
375 if (callbackInfos != null)
376 foreach (ThreadWatchdogInfo callbackInfo in callbackInfos)
377 callback(callbackInfo);
378 }
379  
380 if (MemoryWatchdog.Enabled)
381 MemoryWatchdog.Update();
382  
383 ChecksManager.CheckChecks();
384 StatsManager.RecordStats();
385  
386 m_watchdogTimer.Start();
387 }
388 }
389 }