clockwerk-opensim – Blame information for rev 1
?pathlinks?
Rev | Author | Line No. | Line |
---|---|---|---|
1 | vero | 1 | /* |
2 | * Copyright (c) Contributors, http://opensimulator.org/ |
||
3 | * See CONTRIBUTORS.TXT for a full list of copyright holders. |
||
4 | * |
||
5 | * Redistribution and use in source and binary forms, with or without |
||
6 | * modification, are permitted provided that the following conditions are met: |
||
7 | * * Redistributions of source code must retain the above copyright |
||
8 | * notice, this list of conditions and the following disclaimer. |
||
9 | * * Redistributions in binary form must reproduce the above copyright |
||
10 | * notice, this list of conditions and the following disclaimer in the |
||
11 | * documentation and/or other materials provided with the distribution. |
||
12 | * * Neither the name of the OpenSimulator Project nor the |
||
13 | * names of its contributors may be used to endorse or promote products |
||
14 | * derived from this software without specific prior written permission. |
||
15 | * |
||
16 | * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY |
||
17 | * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
||
18 | * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE |
||
19 | * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY |
||
20 | * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES |
||
21 | * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
||
22 | * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND |
||
23 | * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
||
24 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS |
||
25 | * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
||
26 | */ |
||
27 | |||
28 | // Uncomment to make asset Get requests for existing |
||
29 | // #define WAIT_ON_INPROGRESS_REQUESTS |
||
30 | |||
31 | using System; |
||
32 | using System.IO; |
||
33 | using System.Collections.Generic; |
||
34 | using System.Linq; |
||
35 | using System.Reflection; |
||
36 | using System.Runtime.Serialization; |
||
37 | using System.Runtime.Serialization.Formatters.Binary; |
||
38 | using System.Threading; |
||
39 | using System.Timers; |
||
40 | using log4net; |
||
41 | using Nini.Config; |
||
42 | using Mono.Addins; |
||
43 | using OpenMetaverse; |
||
44 | using OpenSim.Framework; |
||
45 | using OpenSim.Framework.Console; |
||
46 | using OpenSim.Framework.Monitoring; |
||
47 | using OpenSim.Region.Framework.Interfaces; |
||
48 | using OpenSim.Region.Framework.Scenes; |
||
49 | using OpenSim.Services.Interfaces; |
||
50 | |||
51 | |||
52 | //[assembly: Addin("FlotsamAssetCache", "1.1")] |
||
53 | //[assembly: AddinDependency("OpenSim", "0.5")] |
||
54 | |||
55 | namespace OpenSim.Region.CoreModules.Asset |
||
56 | { |
||
57 | [Extension(Path = "/OpenSim/RegionModules", NodeName = "RegionModule", Id = "FlotsamAssetCache")] |
||
58 | public class FlotsamAssetCache : ISharedRegionModule, IImprovedAssetCache, IAssetService |
||
59 | { |
||
60 | private static readonly ILog m_log = |
||
61 | LogManager.GetLogger( |
||
62 | MethodBase.GetCurrentMethod().DeclaringType); |
||
63 | |||
64 | private bool m_Enabled; |
||
65 | |||
66 | private const string m_ModuleName = "FlotsamAssetCache"; |
||
67 | private const string m_DefaultCacheDirectory = "./assetcache"; |
||
68 | private string m_CacheDirectory = m_DefaultCacheDirectory; |
||
69 | |||
70 | private readonly List<char> m_InvalidChars = new List<char>(); |
||
71 | |||
72 | private int m_LogLevel = 0; |
||
73 | private ulong m_HitRateDisplay = 100; // How often to display hit statistics, given in requests |
||
74 | |||
75 | private static ulong m_Requests; |
||
76 | private static ulong m_RequestsForInprogress; |
||
77 | private static ulong m_DiskHits; |
||
78 | private static ulong m_MemoryHits; |
||
79 | |||
80 | #if WAIT_ON_INPROGRESS_REQUESTS |
||
81 | private Dictionary<string, ManualResetEvent> m_CurrentlyWriting = new Dictionary<string, ManualResetEvent>(); |
||
82 | private int m_WaitOnInprogressTimeout = 3000; |
||
83 | #else |
||
84 | private HashSet<string> m_CurrentlyWriting = new HashSet<string>(); |
||
85 | #endif |
||
86 | |||
87 | private bool m_FileCacheEnabled = true; |
||
88 | |||
89 | private ExpiringCache<string, AssetBase> m_MemoryCache; |
||
90 | private bool m_MemoryCacheEnabled = false; |
||
91 | |||
92 | // Expiration is expressed in hours. |
||
93 | private const double m_DefaultMemoryExpiration = 2; |
||
94 | private const double m_DefaultFileExpiration = 48; |
||
95 | private TimeSpan m_MemoryExpiration = TimeSpan.FromHours(m_DefaultMemoryExpiration); |
||
96 | private TimeSpan m_FileExpiration = TimeSpan.FromHours(m_DefaultFileExpiration); |
||
97 | private TimeSpan m_FileExpirationCleanupTimer = TimeSpan.FromHours(0.166); |
||
98 | |||
99 | private static int m_CacheDirectoryTiers = 1; |
||
100 | private static int m_CacheDirectoryTierLen = 3; |
||
101 | private static int m_CacheWarnAt = 30000; |
||
102 | |||
103 | private System.Timers.Timer m_CacheCleanTimer; |
||
104 | |||
105 | private IAssetService m_AssetService; |
||
106 | private List<Scene> m_Scenes = new List<Scene>(); |
||
107 | |||
108 | public FlotsamAssetCache() |
||
109 | { |
||
110 | m_InvalidChars.AddRange(Path.GetInvalidPathChars()); |
||
111 | m_InvalidChars.AddRange(Path.GetInvalidFileNameChars()); |
||
112 | } |
||
113 | |||
114 | public Type ReplaceableInterface |
||
115 | { |
||
116 | get { return null; } |
||
117 | } |
||
118 | |||
119 | public string Name |
||
120 | { |
||
121 | get { return m_ModuleName; } |
||
122 | } |
||
123 | |||
124 | public void Initialise(IConfigSource source) |
||
125 | { |
||
126 | IConfig moduleConfig = source.Configs["Modules"]; |
||
127 | |||
128 | if (moduleConfig != null) |
||
129 | { |
||
130 | string name = moduleConfig.GetString("AssetCaching", String.Empty); |
||
131 | |||
132 | if (name == Name) |
||
133 | { |
||
134 | m_MemoryCache = new ExpiringCache<string, AssetBase>(); |
||
135 | m_Enabled = true; |
||
136 | |||
137 | m_log.InfoFormat("[FLOTSAM ASSET CACHE]: {0} enabled", this.Name); |
||
138 | |||
139 | IConfig assetConfig = source.Configs["AssetCache"]; |
||
140 | if (assetConfig == null) |
||
141 | { |
||
142 | m_log.Debug( |
||
143 | "[FLOTSAM ASSET CACHE]: AssetCache section missing from config (not copied config-include/FlotsamCache.ini.example? Using defaults."); |
||
144 | } |
||
145 | else |
||
146 | { |
||
147 | m_FileCacheEnabled = assetConfig.GetBoolean("FileCacheEnabled", m_FileCacheEnabled); |
||
148 | m_CacheDirectory = assetConfig.GetString("CacheDirectory", m_DefaultCacheDirectory); |
||
149 | |||
150 | m_MemoryCacheEnabled = assetConfig.GetBoolean("MemoryCacheEnabled", m_MemoryCacheEnabled); |
||
151 | m_MemoryExpiration = TimeSpan.FromHours(assetConfig.GetDouble("MemoryCacheTimeout", m_DefaultMemoryExpiration)); |
||
152 | |||
153 | #if WAIT_ON_INPROGRESS_REQUESTS |
||
154 | m_WaitOnInprogressTimeout = assetConfig.GetInt("WaitOnInprogressTimeout", 3000); |
||
155 | #endif |
||
156 | |||
157 | m_LogLevel = assetConfig.GetInt("LogLevel", m_LogLevel); |
||
158 | m_HitRateDisplay = (ulong)assetConfig.GetLong("HitRateDisplay", (long)m_HitRateDisplay); |
||
159 | |||
160 | m_FileExpiration = TimeSpan.FromHours(assetConfig.GetDouble("FileCacheTimeout", m_DefaultFileExpiration)); |
||
161 | m_FileExpirationCleanupTimer |
||
162 | = TimeSpan.FromHours( |
||
163 | assetConfig.GetDouble("FileCleanupTimer", m_FileExpirationCleanupTimer.TotalHours)); |
||
164 | |||
165 | m_CacheDirectoryTiers = assetConfig.GetInt("CacheDirectoryTiers", m_CacheDirectoryTiers); |
||
166 | m_CacheDirectoryTierLen = assetConfig.GetInt("CacheDirectoryTierLength", m_CacheDirectoryTierLen); |
||
167 | |||
168 | m_CacheWarnAt = assetConfig.GetInt("CacheWarnAt", m_CacheWarnAt); |
||
169 | } |
||
170 | |||
171 | m_log.InfoFormat("[FLOTSAM ASSET CACHE]: Cache Directory {0}", m_CacheDirectory); |
||
172 | |||
173 | if (m_FileCacheEnabled && (m_FileExpiration > TimeSpan.Zero) && (m_FileExpirationCleanupTimer > TimeSpan.Zero)) |
||
174 | { |
||
175 | m_CacheCleanTimer = new System.Timers.Timer(m_FileExpirationCleanupTimer.TotalMilliseconds); |
||
176 | m_CacheCleanTimer.AutoReset = true; |
||
177 | m_CacheCleanTimer.Elapsed += CleanupExpiredFiles; |
||
178 | lock (m_CacheCleanTimer) |
||
179 | m_CacheCleanTimer.Start(); |
||
180 | } |
||
181 | |||
182 | if (m_CacheDirectoryTiers < 1) |
||
183 | { |
||
184 | m_CacheDirectoryTiers = 1; |
||
185 | } |
||
186 | else if (m_CacheDirectoryTiers > 3) |
||
187 | { |
||
188 | m_CacheDirectoryTiers = 3; |
||
189 | } |
||
190 | |||
191 | if (m_CacheDirectoryTierLen < 1) |
||
192 | { |
||
193 | m_CacheDirectoryTierLen = 1; |
||
194 | } |
||
195 | else if (m_CacheDirectoryTierLen > 4) |
||
196 | { |
||
197 | m_CacheDirectoryTierLen = 4; |
||
198 | } |
||
199 | |||
200 | MainConsole.Instance.Commands.AddCommand("Assets", true, "fcache status", "fcache status", "Display cache status", HandleConsoleCommand); |
||
201 | MainConsole.Instance.Commands.AddCommand("Assets", true, "fcache clear", "fcache clear [file] [memory]", "Remove all assets in the cache. If file or memory is specified then only this cache is cleared.", HandleConsoleCommand); |
||
202 | MainConsole.Instance.Commands.AddCommand("Assets", true, "fcache assets", "fcache assets", "Attempt a deep scan and cache of all assets in all scenes", HandleConsoleCommand); |
||
203 | MainConsole.Instance.Commands.AddCommand("Assets", true, "fcache expire", "fcache expire <datetime>", "Purge cached assets older then the specified date/time", HandleConsoleCommand); |
||
204 | } |
||
205 | } |
||
206 | } |
||
207 | |||
208 | public void PostInitialise() |
||
209 | { |
||
210 | } |
||
211 | |||
212 | public void Close() |
||
213 | { |
||
214 | } |
||
215 | |||
216 | public void AddRegion(Scene scene) |
||
217 | { |
||
218 | if (m_Enabled) |
||
219 | { |
||
220 | scene.RegisterModuleInterface<IImprovedAssetCache>(this); |
||
221 | m_Scenes.Add(scene); |
||
222 | |||
223 | } |
||
224 | } |
||
225 | |||
226 | public void RemoveRegion(Scene scene) |
||
227 | { |
||
228 | if (m_Enabled) |
||
229 | { |
||
230 | scene.UnregisterModuleInterface<IImprovedAssetCache>(this); |
||
231 | m_Scenes.Remove(scene); |
||
232 | } |
||
233 | } |
||
234 | |||
235 | public void RegionLoaded(Scene scene) |
||
236 | { |
||
237 | if (m_Enabled && m_AssetService == null) |
||
238 | m_AssetService = scene.RequestModuleInterface<IAssetService>(); |
||
239 | } |
||
240 | |||
241 | //////////////////////////////////////////////////////////// |
||
242 | // IImprovedAssetCache |
||
243 | // |
||
244 | |||
245 | private void UpdateMemoryCache(string key, AssetBase asset) |
||
246 | { |
||
247 | m_MemoryCache.AddOrUpdate(key, asset, m_MemoryExpiration); |
||
248 | } |
||
249 | |||
250 | private void UpdateFileCache(string key, AssetBase asset) |
||
251 | { |
||
252 | string filename = GetFileName(key); |
||
253 | |||
254 | try |
||
255 | { |
||
256 | // If the file is already cached, don't cache it, just touch it so access time is updated |
||
257 | if (File.Exists(filename)) |
||
258 | { |
||
259 | // We don't really want to know about sharing |
||
260 | // violations here. If the file is locked, then |
||
261 | // the other thread has updated the time for us. |
||
262 | try |
||
263 | { |
||
264 | lock (m_CurrentlyWriting) |
||
265 | { |
||
266 | if (!m_CurrentlyWriting.Contains(filename)) |
||
267 | File.SetLastAccessTime(filename, DateTime.Now); |
||
268 | } |
||
269 | } |
||
270 | catch |
||
271 | { |
||
272 | } |
||
273 | } |
||
274 | else |
||
275 | { |
||
276 | // Once we start writing, make sure we flag that we're writing |
||
277 | // that object to the cache so that we don't try to write the |
||
278 | // same file multiple times. |
||
279 | lock (m_CurrentlyWriting) |
||
280 | { |
||
281 | #if WAIT_ON_INPROGRESS_REQUESTS |
||
282 | if (m_CurrentlyWriting.ContainsKey(filename)) |
||
283 | { |
||
284 | return; |
||
285 | } |
||
286 | else |
||
287 | { |
||
288 | m_CurrentlyWriting.Add(filename, new ManualResetEvent(false)); |
||
289 | } |
||
290 | |||
291 | #else |
||
292 | if (m_CurrentlyWriting.Contains(filename)) |
||
293 | { |
||
294 | return; |
||
295 | } |
||
296 | else |
||
297 | { |
||
298 | m_CurrentlyWriting.Add(filename); |
||
299 | } |
||
300 | #endif |
||
301 | |||
302 | } |
||
303 | |||
304 | Util.FireAndForget( |
||
305 | delegate { WriteFileCache(filename, asset); }); |
||
306 | } |
||
307 | } |
||
308 | catch (Exception e) |
||
309 | { |
||
310 | m_log.ErrorFormat( |
||
311 | "[FLOTSAM ASSET CACHE]: Failed to update cache for asset {0}. Exception {1} {2}", |
||
312 | asset.ID, e.Message, e.StackTrace); |
||
313 | } |
||
314 | } |
||
315 | |||
316 | public void Cache(AssetBase asset) |
||
317 | { |
||
318 | // TODO: Spawn this off to some seperate thread to do the actual writing |
||
319 | if (asset != null) |
||
320 | { |
||
321 | //m_log.DebugFormat("[FLOTSAM ASSET CACHE]: Caching asset with id {0}", asset.ID); |
||
322 | |||
323 | if (m_MemoryCacheEnabled) |
||
324 | UpdateMemoryCache(asset.ID, asset); |
||
325 | |||
326 | if (m_FileCacheEnabled) |
||
327 | UpdateFileCache(asset.ID, asset); |
||
328 | } |
||
329 | } |
||
330 | |||
331 | /// <summary> |
||
332 | /// Try to get an asset from the in-memory cache. |
||
333 | /// </summary> |
||
334 | /// <param name="id"></param> |
||
335 | /// <returns></returns> |
||
336 | private AssetBase GetFromMemoryCache(string id) |
||
337 | { |
||
338 | AssetBase asset = null; |
||
339 | |||
340 | if (m_MemoryCache.TryGetValue(id, out asset)) |
||
341 | m_MemoryHits++; |
||
342 | |||
343 | return asset; |
||
344 | } |
||
345 | |||
346 | private bool CheckFromMemoryCache(string id) |
||
347 | { |
||
348 | return m_MemoryCache.Contains(id); |
||
349 | } |
||
350 | |||
351 | /// <summary> |
||
352 | /// Try to get an asset from the file cache. |
||
353 | /// </summary> |
||
354 | /// <param name="id"></param> |
||
355 | /// <returns>An asset retrieved from the file cache. null if there was a problem retrieving an asset.</returns> |
||
356 | private AssetBase GetFromFileCache(string id) |
||
357 | { |
||
358 | string filename = GetFileName(id); |
||
359 | |||
360 | #if WAIT_ON_INPROGRESS_REQUESTS |
||
361 | // Check if we're already downloading this asset. If so, try to wait for it to |
||
362 | // download. |
||
363 | if (m_WaitOnInprogressTimeout > 0) |
||
364 | { |
||
365 | m_RequestsForInprogress++; |
||
366 | |||
367 | ManualResetEvent waitEvent; |
||
368 | if (m_CurrentlyWriting.TryGetValue(filename, out waitEvent)) |
||
369 | { |
||
370 | waitEvent.WaitOne(m_WaitOnInprogressTimeout); |
||
371 | return Get(id); |
||
372 | } |
||
373 | } |
||
374 | #else |
||
375 | // Track how often we have the problem that an asset is requested while |
||
376 | // it is still being downloaded by a previous request. |
||
377 | if (m_CurrentlyWriting.Contains(filename)) |
||
378 | { |
||
379 | m_RequestsForInprogress++; |
||
380 | return null; |
||
381 | } |
||
382 | #endif |
||
383 | |||
384 | AssetBase asset = null; |
||
385 | |||
386 | if (File.Exists(filename)) |
||
387 | { |
||
388 | try |
||
389 | { |
||
390 | using (FileStream stream = File.Open(filename, FileMode.Open, FileAccess.Read, FileShare.Read)) |
||
391 | { |
||
392 | BinaryFormatter bformatter = new BinaryFormatter(); |
||
393 | |||
394 | asset = (AssetBase)bformatter.Deserialize(stream); |
||
395 | |||
396 | m_DiskHits++; |
||
397 | } |
||
398 | } |
||
399 | catch (System.Runtime.Serialization.SerializationException e) |
||
400 | { |
||
401 | m_log.WarnFormat( |
||
402 | "[FLOTSAM ASSET CACHE]: Failed to get file {0} for asset {1}. Exception {2} {3}", |
||
403 | filename, id, e.Message, e.StackTrace); |
||
404 | |||
405 | // If there was a problem deserializing the asset, the asset may |
||
406 | // either be corrupted OR was serialized under an old format |
||
407 | // {different version of AssetBase} -- we should attempt to |
||
408 | // delete it and re-cache |
||
409 | File.Delete(filename); |
||
410 | } |
||
411 | catch (Exception e) |
||
412 | { |
||
413 | m_log.WarnFormat( |
||
414 | "[FLOTSAM ASSET CACHE]: Failed to get file {0} for asset {1}. Exception {2} {3}", |
||
415 | filename, id, e.Message, e.StackTrace); |
||
416 | } |
||
417 | } |
||
418 | |||
419 | return asset; |
||
420 | } |
||
421 | |||
422 | private bool CheckFromFileCache(string id) |
||
423 | { |
||
424 | bool found = false; |
||
425 | |||
426 | string filename = GetFileName(id); |
||
427 | |||
428 | if (File.Exists(filename)) |
||
429 | { |
||
430 | try |
||
431 | { |
||
432 | using (FileStream stream = File.Open(filename, FileMode.Open, FileAccess.Read, FileShare.Read)) |
||
433 | { |
||
434 | if (stream != null) |
||
435 | found = true; |
||
436 | } |
||
437 | } |
||
438 | catch (Exception e) |
||
439 | { |
||
440 | m_log.ErrorFormat( |
||
441 | "[FLOTSAM ASSET CACHE]: Failed to check file {0} for asset {1}. Exception {2} {3}", |
||
442 | filename, id, e.Message, e.StackTrace); |
||
443 | } |
||
444 | } |
||
445 | |||
446 | return found; |
||
447 | } |
||
448 | |||
449 | public AssetBase Get(string id) |
||
450 | { |
||
451 | m_Requests++; |
||
452 | |||
453 | AssetBase asset = null; |
||
454 | |||
455 | if (m_MemoryCacheEnabled) |
||
456 | asset = GetFromMemoryCache(id); |
||
457 | |||
458 | if (asset == null && m_FileCacheEnabled) |
||
459 | { |
||
460 | asset = GetFromFileCache(id); |
||
461 | |||
462 | if (m_MemoryCacheEnabled && asset != null) |
||
463 | UpdateMemoryCache(id, asset); |
||
464 | } |
||
465 | |||
466 | if (((m_LogLevel >= 1)) && (m_HitRateDisplay != 0) && (m_Requests % m_HitRateDisplay == 0)) |
||
467 | { |
||
468 | m_log.InfoFormat("[FLOTSAM ASSET CACHE]: Cache Get :: {0} :: {1}", id, asset == null ? "Miss" : "Hit"); |
||
469 | |||
470 | GenerateCacheHitReport().ForEach(l => m_log.InfoFormat("[FLOTSAM ASSET CACHE]: {0}", l)); |
||
471 | } |
||
472 | |||
473 | return asset; |
||
474 | } |
||
475 | |||
476 | public bool Check(string id) |
||
477 | { |
||
478 | if (m_MemoryCacheEnabled && CheckFromMemoryCache(id)) |
||
479 | return true; |
||
480 | |||
481 | if (m_FileCacheEnabled && CheckFromFileCache(id)) |
||
482 | return true; |
||
483 | return false; |
||
484 | } |
||
485 | |||
486 | public AssetBase GetCached(string id) |
||
487 | { |
||
488 | return Get(id); |
||
489 | } |
||
490 | |||
491 | public void Expire(string id) |
||
492 | { |
||
493 | if (m_LogLevel >= 2) |
||
494 | m_log.DebugFormat("[FLOTSAM ASSET CACHE]: Expiring Asset {0}", id); |
||
495 | |||
496 | try |
||
497 | { |
||
498 | if (m_FileCacheEnabled) |
||
499 | { |
||
500 | string filename = GetFileName(id); |
||
501 | if (File.Exists(filename)) |
||
502 | { |
||
503 | File.Delete(filename); |
||
504 | } |
||
505 | } |
||
506 | |||
507 | if (m_MemoryCacheEnabled) |
||
508 | m_MemoryCache.Remove(id); |
||
509 | } |
||
510 | catch (Exception e) |
||
511 | { |
||
512 | m_log.WarnFormat( |
||
513 | "[FLOTSAM ASSET CACHE]: Failed to expire cached file {0}. Exception {1} {2}", |
||
514 | id, e.Message, e.StackTrace); |
||
515 | } |
||
516 | } |
||
517 | |||
518 | public void Clear() |
||
519 | { |
||
520 | if (m_LogLevel >= 2) |
||
521 | m_log.Debug("[FLOTSAM ASSET CACHE]: Clearing caches."); |
||
522 | |||
523 | if (m_FileCacheEnabled) |
||
524 | { |
||
525 | foreach (string dir in Directory.GetDirectories(m_CacheDirectory)) |
||
526 | { |
||
527 | Directory.Delete(dir); |
||
528 | } |
||
529 | } |
||
530 | |||
531 | if (m_MemoryCacheEnabled) |
||
532 | m_MemoryCache.Clear(); |
||
533 | } |
||
534 | |||
535 | private void CleanupExpiredFiles(object source, ElapsedEventArgs e) |
||
536 | { |
||
537 | if (m_LogLevel >= 2) |
||
538 | m_log.DebugFormat("[FLOTSAM ASSET CACHE]: Checking for expired files older then {0}.", m_FileExpiration); |
||
539 | |||
540 | // Purge all files last accessed prior to this point |
||
541 | DateTime purgeLine = DateTime.Now - m_FileExpiration; |
||
542 | |||
543 | // An asset cache may contain local non-temporary assets that are not in the asset service. Therefore, |
||
544 | // before cleaning up expired files we must scan the objects in the scene to make sure that we retain |
||
545 | // such local assets if they have not been recently accessed. |
||
546 | TouchAllSceneAssets(false); |
||
547 | |||
548 | foreach (string dir in Directory.GetDirectories(m_CacheDirectory)) |
||
549 | { |
||
550 | CleanExpiredFiles(dir, purgeLine); |
||
551 | } |
||
552 | } |
||
553 | |||
554 | /// <summary> |
||
555 | /// Recurses through specified directory checking for asset files last |
||
556 | /// accessed prior to the specified purge line and deletes them. Also |
||
557 | /// removes empty tier directories. |
||
558 | /// </summary> |
||
559 | /// <param name="dir"></param> |
||
560 | /// <param name="purgeLine"></param> |
||
561 | private void CleanExpiredFiles(string dir, DateTime purgeLine) |
||
562 | { |
||
563 | try |
||
564 | { |
||
565 | foreach (string file in Directory.GetFiles(dir)) |
||
566 | { |
||
567 | if (File.GetLastAccessTime(file) < purgeLine) |
||
568 | { |
||
569 | File.Delete(file); |
||
570 | } |
||
571 | } |
||
572 | |||
573 | // Recurse into lower tiers |
||
574 | foreach (string subdir in Directory.GetDirectories(dir)) |
||
575 | { |
||
576 | CleanExpiredFiles(subdir, purgeLine); |
||
577 | } |
||
578 | |||
579 | // Check if a tier directory is empty, if so, delete it |
||
580 | int dirSize = Directory.GetFiles(dir).Length + Directory.GetDirectories(dir).Length; |
||
581 | if (dirSize == 0) |
||
582 | { |
||
583 | Directory.Delete(dir); |
||
584 | } |
||
585 | else if (dirSize >= m_CacheWarnAt) |
||
586 | { |
||
587 | m_log.WarnFormat( |
||
588 | "[FLOTSAM ASSET CACHE]: Cache folder exceeded CacheWarnAt limit {0} {1}. Suggest increasing tiers, tier length, or reducing cache expiration", |
||
589 | dir, dirSize); |
||
590 | } |
||
591 | } |
||
592 | catch (Exception e) |
||
593 | { |
||
594 | m_log.Warn( |
||
595 | string.Format("[FLOTSAM ASSET CACHE]: Could not complete clean of expired files in {0}, exception ", dir), e); |
||
596 | } |
||
597 | } |
||
598 | |||
599 | /// <summary> |
||
600 | /// Determines the filename for an AssetID stored in the file cache |
||
601 | /// </summary> |
||
602 | /// <param name="id"></param> |
||
603 | /// <returns></returns> |
||
604 | private string GetFileName(string id) |
||
605 | { |
||
606 | // Would it be faster to just hash the darn thing? |
||
607 | foreach (char c in m_InvalidChars) |
||
608 | { |
||
609 | id = id.Replace(c, '_'); |
||
610 | } |
||
611 | |||
612 | string path = m_CacheDirectory; |
||
613 | for (int p = 1; p <= m_CacheDirectoryTiers; p++) |
||
614 | { |
||
615 | string pathPart = id.Substring((p - 1) * m_CacheDirectoryTierLen, m_CacheDirectoryTierLen); |
||
616 | path = Path.Combine(path, pathPart); |
||
617 | } |
||
618 | |||
619 | return Path.Combine(path, id); |
||
620 | } |
||
621 | |||
622 | /// <summary> |
||
623 | /// Writes a file to the file cache, creating any nessesary |
||
624 | /// tier directories along the way |
||
625 | /// </summary> |
||
626 | /// <param name="filename"></param> |
||
627 | /// <param name="asset"></param> |
||
628 | private void WriteFileCache(string filename, AssetBase asset) |
||
629 | { |
||
630 | Stream stream = null; |
||
631 | |||
632 | // Make sure the target cache directory exists |
||
633 | string directory = Path.GetDirectoryName(filename); |
||
634 | |||
635 | // Write file first to a temp name, so that it doesn't look |
||
636 | // like it's already cached while it's still writing. |
||
637 | string tempname = Path.Combine(directory, Path.GetRandomFileName()); |
||
638 | |||
639 | try |
||
640 | { |
||
641 | try |
||
642 | { |
||
643 | if (!Directory.Exists(directory)) |
||
644 | { |
||
645 | Directory.CreateDirectory(directory); |
||
646 | } |
||
647 | |||
648 | stream = File.Open(tempname, FileMode.Create); |
||
649 | BinaryFormatter bformatter = new BinaryFormatter(); |
||
650 | bformatter.Serialize(stream, asset); |
||
651 | } |
||
652 | catch (IOException e) |
||
653 | { |
||
654 | m_log.WarnFormat( |
||
655 | "[FLOTSAM ASSET CACHE]: Failed to write asset {0} to temporary location {1} (final {2}) on cache in {3}. Exception {4} {5}.", |
||
656 | asset.ID, tempname, filename, directory, e.Message, e.StackTrace); |
||
657 | |||
658 | return; |
||
659 | } |
||
660 | finally |
||
661 | { |
||
662 | if (stream != null) |
||
663 | stream.Close(); |
||
664 | } |
||
665 | |||
666 | try |
||
667 | { |
||
668 | // Now that it's written, rename it so that it can be found. |
||
669 | // |
||
670 | // File.Copy(tempname, filename, true); |
||
671 | // File.Delete(tempname); |
||
672 | // |
||
673 | // For a brief period, this was done as a separate copy and then temporary file delete operation to |
||
674 | // avoid an IOException caused by move if some competing thread had already written the file. |
||
675 | // However, this causes exceptions on Windows when other threads attempt to read a file |
||
676 | // which is still being copied. So instead, go back to moving the file and swallow any IOException. |
||
677 | // |
||
678 | // This situation occurs fairly rarely anyway. We assume in this that moves are atomic on the |
||
679 | // filesystem. |
||
680 | File.Move(tempname, filename); |
||
681 | |||
682 | if (m_LogLevel >= 2) |
||
683 | m_log.DebugFormat("[FLOTSAM ASSET CACHE]: Cache Stored :: {0}", asset.ID); |
||
684 | } |
||
685 | catch (IOException) |
||
686 | { |
||
687 | // If we see an IOException here it's likely that some other competing thread has written the |
||
688 | // cache file first, so ignore. Other IOException errors (e.g. filesystem full) should be |
||
689 | // signally by the earlier temporary file writing code. |
||
690 | } |
||
691 | } |
||
692 | finally |
||
693 | { |
||
694 | // Even if the write fails with an exception, we need to make sure |
||
695 | // that we release the lock on that file, otherwise it'll never get |
||
696 | // cached |
||
697 | lock (m_CurrentlyWriting) |
||
698 | { |
||
699 | #if WAIT_ON_INPROGRESS_REQUESTS |
||
700 | ManualResetEvent waitEvent; |
||
701 | if (m_CurrentlyWriting.TryGetValue(filename, out waitEvent)) |
||
702 | { |
||
703 | m_CurrentlyWriting.Remove(filename); |
||
704 | waitEvent.Set(); |
||
705 | } |
||
706 | #else |
||
707 | m_CurrentlyWriting.Remove(filename); |
||
708 | #endif |
||
709 | } |
||
710 | } |
||
711 | } |
||
712 | |||
713 | /// <summary> |
||
714 | /// Scan through the file cache, and return number of assets currently cached. |
||
715 | /// </summary> |
||
716 | /// <param name="dir"></param> |
||
717 | /// <returns></returns> |
||
718 | private int GetFileCacheCount(string dir) |
||
719 | { |
||
720 | int count = Directory.GetFiles(dir).Length; |
||
721 | |||
722 | foreach (string subdir in Directory.GetDirectories(dir)) |
||
723 | { |
||
724 | count += GetFileCacheCount(subdir); |
||
725 | } |
||
726 | |||
727 | return count; |
||
728 | } |
||
729 | |||
730 | /// <summary> |
||
731 | /// This notes the last time the Region had a deep asset scan performed on it. |
||
732 | /// </summary> |
||
733 | /// <param name="regionID"></param> |
||
734 | private void StampRegionStatusFile(UUID regionID) |
||
735 | { |
||
736 | string RegionCacheStatusFile = Path.Combine(m_CacheDirectory, "RegionStatus_" + regionID.ToString() + ".fac"); |
||
737 | |||
738 | try |
||
739 | { |
||
740 | if (File.Exists(RegionCacheStatusFile)) |
||
741 | { |
||
742 | File.SetLastWriteTime(RegionCacheStatusFile, DateTime.Now); |
||
743 | } |
||
744 | else |
||
745 | { |
||
746 | File.WriteAllText( |
||
747 | RegionCacheStatusFile, |
||
748 | "Please do not delete this file unless you are manually clearing your Flotsam Asset Cache."); |
||
749 | } |
||
750 | } |
||
751 | catch (Exception e) |
||
752 | { |
||
753 | m_log.Warn( |
||
754 | string.Format( |
||
755 | "[FLOTSAM ASSET CACHE]: Could not stamp region status file for region {0}. Exception ", |
||
756 | regionID), |
||
757 | e); |
||
758 | } |
||
759 | } |
||
760 | |||
761 | /// <summary> |
||
762 | /// Iterates through all Scenes, doing a deep scan through assets |
||
763 | /// to update the access time of all assets present in the scene or referenced by assets |
||
764 | /// in the scene. |
||
765 | /// </summary> |
||
766 | /// <param name="storeUncached"> |
||
767 | /// If true, then assets scanned which are not found in cache are added to the cache. |
||
768 | /// </param> |
||
769 | /// <returns>Number of distinct asset references found in the scene.</returns> |
||
770 | private int TouchAllSceneAssets(bool storeUncached) |
||
771 | { |
||
772 | UuidGatherer gatherer = new UuidGatherer(m_AssetService); |
||
773 | |||
774 | HashSet<UUID> uniqueUuids = new HashSet<UUID>(); |
||
775 | Dictionary<UUID, sbyte> assets = new Dictionary<UUID, sbyte>(); |
||
776 | |||
777 | foreach (Scene s in m_Scenes) |
||
778 | { |
||
779 | StampRegionStatusFile(s.RegionInfo.RegionID); |
||
780 | |||
781 | s.ForEachSOG(delegate(SceneObjectGroup e) |
||
782 | { |
||
783 | gatherer.GatherAssetUuids(e, assets); |
||
784 | |||
785 | foreach (UUID assetID in assets.Keys) |
||
786 | { |
||
787 | uniqueUuids.Add(assetID); |
||
788 | |||
789 | string filename = GetFileName(assetID.ToString()); |
||
790 | |||
791 | if (File.Exists(filename)) |
||
792 | { |
||
793 | File.SetLastAccessTime(filename, DateTime.Now); |
||
794 | } |
||
795 | else if (storeUncached) |
||
796 | { |
||
797 | AssetBase cachedAsset = m_AssetService.Get(assetID.ToString()); |
||
798 | if (cachedAsset == null && assets[assetID] != (sbyte)AssetType.Unknown) |
||
799 | m_log.DebugFormat( |
||
800 | "[FLOTSAM ASSET CACHE]: Could not find asset {0}, type {1} referenced by object {2} at {3} in scene {4} when pre-caching all scene assets", |
||
801 | assetID, assets[assetID], e.Name, e.AbsolutePosition, s.Name); |
||
802 | } |
||
803 | } |
||
804 | |||
805 | assets.Clear(); |
||
806 | }); |
||
807 | } |
||
808 | |||
809 | |||
810 | return uniqueUuids.Count; |
||
811 | } |
||
812 | |||
813 | /// <summary> |
||
814 | /// Deletes all cache contents |
||
815 | /// </summary> |
||
816 | private void ClearFileCache() |
||
817 | { |
||
818 | foreach (string dir in Directory.GetDirectories(m_CacheDirectory)) |
||
819 | { |
||
820 | try |
||
821 | { |
||
822 | Directory.Delete(dir, true); |
||
823 | } |
||
824 | catch (Exception e) |
||
825 | { |
||
826 | m_log.WarnFormat( |
||
827 | "[FLOTSAM ASSET CACHE]: Couldn't clear asset cache directory {0} from {1}. Exception {2} {3}", |
||
828 | dir, m_CacheDirectory, e.Message, e.StackTrace); |
||
829 | } |
||
830 | } |
||
831 | |||
832 | foreach (string file in Directory.GetFiles(m_CacheDirectory)) |
||
833 | { |
||
834 | try |
||
835 | { |
||
836 | File.Delete(file); |
||
837 | } |
||
838 | catch (Exception e) |
||
839 | { |
||
840 | m_log.WarnFormat( |
||
841 | "[FLOTSAM ASSET CACHE]: Couldn't clear asset cache file {0} from {1}. Exception {1} {2}", |
||
842 | file, m_CacheDirectory, e.Message, e.StackTrace); |
||
843 | } |
||
844 | } |
||
845 | } |
||
846 | |||
847 | private List<string> GenerateCacheHitReport() |
||
848 | { |
||
849 | List<string> outputLines = new List<string>(); |
||
850 | |||
851 | double fileHitRate = (double)m_DiskHits / m_Requests * 100.0; |
||
852 | outputLines.Add( |
||
853 | string.Format("File Hit Rate: {0}% for {1} requests", fileHitRate.ToString("0.00"), m_Requests)); |
||
854 | |||
855 | if (m_MemoryCacheEnabled) |
||
856 | { |
||
857 | double memHitRate = (double)m_MemoryHits / m_Requests * 100.0; |
||
858 | |||
859 | outputLines.Add( |
||
860 | string.Format("Memory Hit Rate: {0}% for {1} requests", memHitRate.ToString("0.00"), m_Requests)); |
||
861 | } |
||
862 | |||
863 | outputLines.Add( |
||
864 | string.Format( |
||
865 | "Unnecessary requests due to requests for assets that are currently downloading: {0}", |
||
866 | m_RequestsForInprogress)); |
||
867 | |||
868 | return outputLines; |
||
869 | } |
||
870 | |||
871 | #region Console Commands |
||
872 | private void HandleConsoleCommand(string module, string[] cmdparams) |
||
873 | { |
||
874 | ICommandConsole con = MainConsole.Instance; |
||
875 | |||
876 | if (cmdparams.Length >= 2) |
||
877 | { |
||
878 | string cmd = cmdparams[1]; |
||
879 | |||
880 | switch (cmd) |
||
881 | { |
||
882 | case "status": |
||
883 | if (m_MemoryCacheEnabled) |
||
884 | con.OutputFormat("Memory Cache: {0} assets", m_MemoryCache.Count); |
||
885 | else |
||
886 | con.OutputFormat("Memory cache disabled"); |
||
887 | |||
888 | if (m_FileCacheEnabled) |
||
889 | { |
||
890 | int fileCount = GetFileCacheCount(m_CacheDirectory); |
||
891 | con.OutputFormat("File Cache: {0} assets", fileCount); |
||
892 | } |
||
893 | else |
||
894 | { |
||
895 | con.Output("File cache disabled"); |
||
896 | } |
||
897 | |||
898 | GenerateCacheHitReport().ForEach(l => con.Output(l)); |
||
899 | |||
900 | if (m_FileCacheEnabled) |
||
901 | { |
||
902 | con.Output("Deep scans have previously been performed on the following regions:"); |
||
903 | |||
904 | foreach (string s in Directory.GetFiles(m_CacheDirectory, "*.fac")) |
||
905 | { |
||
906 | string RegionID = s.Remove(0,s.IndexOf("_")).Replace(".fac",""); |
||
907 | DateTime RegionDeepScanTMStamp = File.GetLastWriteTime(s); |
||
908 | con.OutputFormat("Region: {0}, {1}", RegionID, RegionDeepScanTMStamp.ToString("MM/dd/yyyy hh:mm:ss")); |
||
909 | } |
||
910 | } |
||
911 | |||
912 | break; |
||
913 | |||
914 | case "clear": |
||
915 | if (cmdparams.Length < 2) |
||
916 | { |
||
917 | con.Output("Usage is fcache clear [file] [memory]"); |
||
918 | break; |
||
919 | } |
||
920 | |||
921 | bool clearMemory = false, clearFile = false; |
||
922 | |||
923 | if (cmdparams.Length == 2) |
||
924 | { |
||
925 | clearMemory = true; |
||
926 | clearFile = true; |
||
927 | } |
||
928 | foreach (string s in cmdparams) |
||
929 | { |
||
930 | if (s.ToLower() == "memory") |
||
931 | clearMemory = true; |
||
932 | else if (s.ToLower() == "file") |
||
933 | clearFile = true; |
||
934 | } |
||
935 | |||
936 | if (clearMemory) |
||
937 | { |
||
938 | if (m_MemoryCacheEnabled) |
||
939 | { |
||
940 | m_MemoryCache.Clear(); |
||
941 | con.Output("Memory cache cleared."); |
||
942 | } |
||
943 | else |
||
944 | { |
||
945 | con.Output("Memory cache not enabled."); |
||
946 | } |
||
947 | } |
||
948 | |||
949 | if (clearFile) |
||
950 | { |
||
951 | if (m_FileCacheEnabled) |
||
952 | { |
||
953 | ClearFileCache(); |
||
954 | con.Output("File cache cleared."); |
||
955 | } |
||
956 | else |
||
957 | { |
||
958 | con.Output("File cache not enabled."); |
||
959 | } |
||
960 | } |
||
961 | |||
962 | break; |
||
963 | |||
964 | case "assets": |
||
965 | con.Output("Ensuring assets are cached for all scenes."); |
||
966 | |||
967 | Watchdog.RunInThread(delegate |
||
968 | { |
||
969 | int assetReferenceTotal = TouchAllSceneAssets(true); |
||
970 | con.OutputFormat("Completed check with {0} assets.", assetReferenceTotal); |
||
971 | }, "TouchAllSceneAssets", null); |
||
972 | |||
973 | break; |
||
974 | |||
975 | case "expire": |
||
976 | if (cmdparams.Length < 3) |
||
977 | { |
||
978 | con.OutputFormat("Invalid parameters for Expire, please specify a valid date & time", cmd); |
||
979 | break; |
||
980 | } |
||
981 | |||
982 | string s_expirationDate = ""; |
||
983 | DateTime expirationDate; |
||
984 | |||
985 | if (cmdparams.Length > 3) |
||
986 | { |
||
987 | s_expirationDate = string.Join(" ", cmdparams, 2, cmdparams.Length - 2); |
||
988 | } |
||
989 | else |
||
990 | { |
||
991 | s_expirationDate = cmdparams[2]; |
||
992 | } |
||
993 | |||
994 | if (!DateTime.TryParse(s_expirationDate, out expirationDate)) |
||
995 | { |
||
996 | con.OutputFormat("{0} is not a valid date & time", cmd); |
||
997 | break; |
||
998 | } |
||
999 | |||
1000 | if (m_FileCacheEnabled) |
||
1001 | CleanExpiredFiles(m_CacheDirectory, expirationDate); |
||
1002 | else |
||
1003 | con.OutputFormat("File cache not active, not clearing."); |
||
1004 | |||
1005 | break; |
||
1006 | default: |
||
1007 | con.OutputFormat("Unknown command {0}", cmd); |
||
1008 | break; |
||
1009 | } |
||
1010 | } |
||
1011 | else if (cmdparams.Length == 1) |
||
1012 | { |
||
1013 | con.Output("fcache assets - Attempt a deep cache of all assets in all scenes"); |
||
1014 | con.Output("fcache expire <datetime> - Purge assets older then the specified date & time"); |
||
1015 | con.Output("fcache clear [file] [memory] - Remove cached assets"); |
||
1016 | con.Output("fcache status - Display cache status"); |
||
1017 | } |
||
1018 | } |
||
1019 | |||
1020 | #endregion |
||
1021 | |||
1022 | #region IAssetService Members |
||
1023 | |||
1024 | public AssetMetadata GetMetadata(string id) |
||
1025 | { |
||
1026 | AssetBase asset = Get(id); |
||
1027 | return asset.Metadata; |
||
1028 | } |
||
1029 | |||
1030 | public byte[] GetData(string id) |
||
1031 | { |
||
1032 | AssetBase asset = Get(id); |
||
1033 | return asset.Data; |
||
1034 | } |
||
1035 | |||
1036 | public bool Get(string id, object sender, AssetRetrieved handler) |
||
1037 | { |
||
1038 | AssetBase asset = Get(id); |
||
1039 | handler(id, sender, asset); |
||
1040 | return true; |
||
1041 | } |
||
1042 | |||
1043 | public bool[] AssetsExist(string[] ids) |
||
1044 | { |
||
1045 | bool[] exist = new bool[ids.Length]; |
||
1046 | |||
1047 | for (int i = 0; i < ids.Length; i++) |
||
1048 | { |
||
1049 | exist[i] = Check(ids[i]); |
||
1050 | } |
||
1051 | |||
1052 | return exist; |
||
1053 | } |
||
1054 | |||
1055 | public string Store(AssetBase asset) |
||
1056 | { |
||
1057 | if (asset.FullID == UUID.Zero) |
||
1058 | { |
||
1059 | asset.FullID = UUID.Random(); |
||
1060 | } |
||
1061 | |||
1062 | Cache(asset); |
||
1063 | |||
1064 | return asset.ID; |
||
1065 | } |
||
1066 | |||
1067 | public bool UpdateContent(string id, byte[] data) |
||
1068 | { |
||
1069 | AssetBase asset = Get(id); |
||
1070 | asset.Data = data; |
||
1071 | Cache(asset); |
||
1072 | return true; |
||
1073 | } |
||
1074 | |||
1075 | public bool Delete(string id) |
||
1076 | { |
||
1077 | Expire(id); |
||
1078 | return true; |
||
1079 | } |
||
1080 | |||
1081 | #endregion |
||
1082 | } |
||
1083 | } |