clockwerk-opensim-stable – Blame information for rev 1

Subversion Repositories:
Rev:
Rev Author Line No. Line
1 vero 1 /*
2 * Copyright (c) Contributors, http://opensimulator.org/
3 * See CONTRIBUTORS.TXT for a full list of copyright holders.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 * * Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * * Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 * * Neither the name of the OpenSimulator Project nor the
13 * names of its contributors may be used to endorse or promote products
14 * derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY
17 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY
20 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
23 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27  
28 // Uncomment to make asset Get requests for existing
29 // #define WAIT_ON_INPROGRESS_REQUESTS
30  
31 using System;
32 using System.IO;
33 using System.Collections.Generic;
34 using System.Reflection;
35 using System.Runtime.Serialization;
36 using System.Runtime.Serialization.Formatters.Binary;
37 using System.Threading;
38 using System.Timers;
39  
40 using log4net;
41 using Nini.Config;
42 using Mono.Addins;
43 using OpenMetaverse;
44  
45 using OpenSim.Framework;
46 using OpenSim.Framework.Console;
47 using OpenSim.Region.Framework.Interfaces;
48 using OpenSim.Region.Framework.Scenes;
49 using OpenSim.Services.Interfaces;
50  
51  
52 //[assembly: Addin("FlotsamAssetCache", "1.1")]
53 //[assembly: AddinDependency("OpenSim", "0.5")]
54  
55 namespace OpenSim.Region.CoreModules.Asset
56 {
57 [Extension(Path = "/OpenSim/RegionModules", NodeName = "RegionModule", Id = "FlotsamAssetCache")]
58 public class FlotsamAssetCache : ISharedRegionModule, IImprovedAssetCache, IAssetService
59 {
60 private static readonly ILog m_log =
61 LogManager.GetLogger(
62 MethodBase.GetCurrentMethod().DeclaringType);
63  
64 private bool m_Enabled;
65  
66 private const string m_ModuleName = "FlotsamAssetCache";
67 private const string m_DefaultCacheDirectory = "./assetcache";
68 private string m_CacheDirectory = m_DefaultCacheDirectory;
69  
70 private readonly List<char> m_InvalidChars = new List<char>();
71  
72 private int m_LogLevel = 0;
73 private ulong m_HitRateDisplay = 100; // How often to display hit statistics, given in requests
74  
75 private static ulong m_Requests;
76 private static ulong m_RequestsForInprogress;
77 private static ulong m_DiskHits;
78 private static ulong m_MemoryHits;
79 private static double m_HitRateMemory;
80 private static double m_HitRateFile;
81  
82 #if WAIT_ON_INPROGRESS_REQUESTS
83 private Dictionary<string, ManualResetEvent> m_CurrentlyWriting = new Dictionary<string, ManualResetEvent>();
84 private int m_WaitOnInprogressTimeout = 3000;
85 #else
86 private HashSet<string> m_CurrentlyWriting = new HashSet<string>();
87 #endif
88  
89 private bool m_FileCacheEnabled = true;
90  
91 private ExpiringCache<string, AssetBase> m_MemoryCache;
92 private bool m_MemoryCacheEnabled = false;
93  
94 // Expiration is expressed in hours.
95 private const double m_DefaultMemoryExpiration = 2;
96 private const double m_DefaultFileExpiration = 48;
97 private TimeSpan m_MemoryExpiration = TimeSpan.FromHours(m_DefaultMemoryExpiration);
98 private TimeSpan m_FileExpiration = TimeSpan.FromHours(m_DefaultFileExpiration);
99 private TimeSpan m_FileExpirationCleanupTimer = TimeSpan.FromHours(0.166);
100  
101 private static int m_CacheDirectoryTiers = 1;
102 private static int m_CacheDirectoryTierLen = 3;
103 private static int m_CacheWarnAt = 30000;
104  
105 private System.Timers.Timer m_CacheCleanTimer;
106  
107 private IAssetService m_AssetService;
108 private List<Scene> m_Scenes = new List<Scene>();
109  
110 public FlotsamAssetCache()
111 {
112 m_InvalidChars.AddRange(Path.GetInvalidPathChars());
113 m_InvalidChars.AddRange(Path.GetInvalidFileNameChars());
114 }
115  
116 public Type ReplaceableInterface
117 {
118 get { return null; }
119 }
120  
121 public string Name
122 {
123 get { return m_ModuleName; }
124 }
125  
126 public void Initialise(IConfigSource source)
127 {
128 IConfig moduleConfig = source.Configs["Modules"];
129  
130 if (moduleConfig != null)
131 {
132 string name = moduleConfig.GetString("AssetCaching", String.Empty);
133  
134 if (name == Name)
135 {
136 m_MemoryCache = new ExpiringCache<string, AssetBase>();
137 m_Enabled = true;
138  
139 m_log.InfoFormat("[FLOTSAM ASSET CACHE]: {0} enabled", this.Name);
140  
141 IConfig assetConfig = source.Configs["AssetCache"];
142 if (assetConfig == null)
143 {
144 m_log.Debug(
145 "[FLOTSAM ASSET CACHE]: AssetCache section missing from config (not copied config-include/FlotsamCache.ini.example? Using defaults.");
146 }
147 else
148 {
149 m_FileCacheEnabled = assetConfig.GetBoolean("FileCacheEnabled", m_FileCacheEnabled);
150 m_CacheDirectory = assetConfig.GetString("CacheDirectory", m_DefaultCacheDirectory);
151  
152 m_MemoryCacheEnabled = assetConfig.GetBoolean("MemoryCacheEnabled", m_MemoryCacheEnabled);
153 m_MemoryExpiration = TimeSpan.FromHours(assetConfig.GetDouble("MemoryCacheTimeout", m_DefaultMemoryExpiration));
154  
155 #if WAIT_ON_INPROGRESS_REQUESTS
156 m_WaitOnInprogressTimeout = assetConfig.GetInt("WaitOnInprogressTimeout", 3000);
157 #endif
158  
159 m_LogLevel = assetConfig.GetInt("LogLevel", m_LogLevel);
160 m_HitRateDisplay = (ulong)assetConfig.GetLong("HitRateDisplay", (long)m_HitRateDisplay);
161  
162 m_FileExpiration = TimeSpan.FromHours(assetConfig.GetDouble("FileCacheTimeout", m_DefaultFileExpiration));
163 m_FileExpirationCleanupTimer
164 = TimeSpan.FromHours(
165 assetConfig.GetDouble("FileCleanupTimer", m_FileExpirationCleanupTimer.TotalHours));
166  
167 m_CacheDirectoryTiers = assetConfig.GetInt("CacheDirectoryTiers", m_CacheDirectoryTiers);
168 m_CacheDirectoryTierLen = assetConfig.GetInt("CacheDirectoryTierLength", m_CacheDirectoryTierLen);
169  
170 m_CacheWarnAt = assetConfig.GetInt("CacheWarnAt", m_CacheWarnAt);
171 }
172  
173 m_log.InfoFormat("[FLOTSAM ASSET CACHE]: Cache Directory {0}", m_CacheDirectory);
174  
175 if (m_FileCacheEnabled && (m_FileExpiration > TimeSpan.Zero) && (m_FileExpirationCleanupTimer > TimeSpan.Zero))
176 {
177 m_CacheCleanTimer = new System.Timers.Timer(m_FileExpirationCleanupTimer.TotalMilliseconds);
178 m_CacheCleanTimer.AutoReset = true;
179 m_CacheCleanTimer.Elapsed += CleanupExpiredFiles;
180 lock (m_CacheCleanTimer)
181 m_CacheCleanTimer.Start();
182 }
183  
184 if (m_CacheDirectoryTiers < 1)
185 {
186 m_CacheDirectoryTiers = 1;
187 }
188 else if (m_CacheDirectoryTiers > 3)
189 {
190 m_CacheDirectoryTiers = 3;
191 }
192  
193 if (m_CacheDirectoryTierLen < 1)
194 {
195 m_CacheDirectoryTierLen = 1;
196 }
197 else if (m_CacheDirectoryTierLen > 4)
198 {
199 m_CacheDirectoryTierLen = 4;
200 }
201  
202 MainConsole.Instance.Commands.AddCommand("Assets", true, "fcache status", "fcache status", "Display cache status", HandleConsoleCommand);
203 MainConsole.Instance.Commands.AddCommand("Assets", true, "fcache clear", "fcache clear [file] [memory]", "Remove all assets in the cache. If file or memory is specified then only this cache is cleared.", HandleConsoleCommand);
204 MainConsole.Instance.Commands.AddCommand("Assets", true, "fcache assets", "fcache assets", "Attempt a deep scan and cache of all assets in all scenes", HandleConsoleCommand);
205 MainConsole.Instance.Commands.AddCommand("Assets", true, "fcache expire", "fcache expire <datetime>", "Purge cached assets older then the specified date/time", HandleConsoleCommand);
206 }
207 }
208 }
209  
210 public void PostInitialise()
211 {
212 }
213  
214 public void Close()
215 {
216 }
217  
218 public void AddRegion(Scene scene)
219 {
220 if (m_Enabled)
221 {
222 scene.RegisterModuleInterface<IImprovedAssetCache>(this);
223 m_Scenes.Add(scene);
224  
225 }
226 }
227  
228 public void RemoveRegion(Scene scene)
229 {
230 if (m_Enabled)
231 {
232 scene.UnregisterModuleInterface<IImprovedAssetCache>(this);
233 m_Scenes.Remove(scene);
234 }
235 }
236  
237 public void RegionLoaded(Scene scene)
238 {
239 if (m_Enabled && m_AssetService == null)
240 m_AssetService = scene.RequestModuleInterface<IAssetService>();
241 }
242  
243 ////////////////////////////////////////////////////////////
244 // IImprovedAssetCache
245 //
246  
247 private void UpdateMemoryCache(string key, AssetBase asset)
248 {
249 m_MemoryCache.AddOrUpdate(key, asset, m_MemoryExpiration);
250 }
251  
252 private void UpdateFileCache(string key, AssetBase asset)
253 {
254 string filename = GetFileName(asset.ID);
255  
256 try
257 {
258 // If the file is already cached just update access time.
259 if (File.Exists(filename))
260 {
261 lock (m_CurrentlyWriting)
262 {
263 if (!m_CurrentlyWriting.Contains(filename))
264 File.SetLastAccessTime(filename, DateTime.Now);
265 }
266 }
267 else
268 {
269 // Once we start writing, make sure we flag that we're writing
270 // that object to the cache so that we don't try to write the
271 // same file multiple times.
272 lock (m_CurrentlyWriting)
273 {
274 #if WAIT_ON_INPROGRESS_REQUESTS
275 if (m_CurrentlyWriting.ContainsKey(filename))
276 {
277 return;
278 }
279 else
280 {
281 m_CurrentlyWriting.Add(filename, new ManualResetEvent(false));
282 }
283  
284 #else
285 if (m_CurrentlyWriting.Contains(filename))
286 {
287 return;
288 }
289 else
290 {
291 m_CurrentlyWriting.Add(filename);
292 }
293 #endif
294 }
295  
296 Util.FireAndForget(
297 delegate { WriteFileCache(filename, asset); });
298 }
299 }
300 catch (Exception e)
301 {
302 m_log.WarnFormat(
303 "[FLOTSAM ASSET CACHE]: Failed to update cache for asset {0}. Exception {1} {2}",
304 asset.ID, e.Message, e.StackTrace);
305 }
306 }
307  
308 public void Cache(AssetBase asset)
309 {
310 // TODO: Spawn this off to some seperate thread to do the actual writing
311 if (asset != null)
312 {
313 //m_log.DebugFormat("[FLOTSAM ASSET CACHE]: Caching asset with id {0}", asset.ID);
314  
315 if (m_MemoryCacheEnabled)
316 UpdateMemoryCache(asset.ID, asset);
317  
318 if (m_FileCacheEnabled)
319 UpdateFileCache(asset.ID, asset);
320 }
321 }
322  
323 /// <summary>
324 /// Try to get an asset from the in-memory cache.
325 /// </summary>
326 /// <param name="id"></param>
327 /// <returns></returns>
328 private AssetBase GetFromMemoryCache(string id)
329 {
330 AssetBase asset = null;
331  
332 if (m_MemoryCache.TryGetValue(id, out asset))
333 m_MemoryHits++;
334  
335 return asset;
336 }
337  
338 /// <summary>
339 /// Try to get an asset from the file cache.
340 /// </summary>
341 /// <param name="id"></param>
342 /// <returns>An asset retrieved from the file cache. null if there was a problem retrieving an asset.</returns>
343 private AssetBase GetFromFileCache(string id)
344 {
345 string filename = GetFileName(id);
346  
347 #if WAIT_ON_INPROGRESS_REQUESTS
348 // Check if we're already downloading this asset. If so, try to wait for it to
349 // download.
350 if (m_WaitOnInprogressTimeout > 0)
351 {
352 m_RequestsForInprogress++;
353  
354 ManualResetEvent waitEvent;
355 if (m_CurrentlyWriting.TryGetValue(filename, out waitEvent))
356 {
357 waitEvent.WaitOne(m_WaitOnInprogressTimeout);
358 return Get(id);
359 }
360 }
361 #else
362 // Track how often we have the problem that an asset is requested while
363 // it is still being downloaded by a previous request.
364 if (m_CurrentlyWriting.Contains(filename))
365 {
366 m_RequestsForInprogress++;
367 return null;
368 }
369 #endif
370  
371 AssetBase asset = null;
372  
373 if (File.Exists(filename))
374 {
375 FileStream stream = null;
376 try
377 {
378 stream = File.Open(filename, FileMode.Open, FileAccess.Read, FileShare.Read);
379 BinaryFormatter bformatter = new BinaryFormatter();
380  
381 asset = (AssetBase)bformatter.Deserialize(stream);
382  
383 m_DiskHits++;
384 }
385 catch (System.Runtime.Serialization.SerializationException e)
386 {
387 m_log.WarnFormat(
388 "[FLOTSAM ASSET CACHE]: Failed to get file {0} for asset {1}. Exception {2} {3}",
389 filename, id, e.Message, e.StackTrace);
390  
391 // If there was a problem deserializing the asset, the asset may
392 // either be corrupted OR was serialized under an old format
393 // {different version of AssetBase} -- we should attempt to
394 // delete it and re-cache
395 File.Delete(filename);
396 }
397 catch (Exception e)
398 {
399 m_log.WarnFormat(
400 "[FLOTSAM ASSET CACHE]: Failed to get file {0} for asset {1}. Exception {2} {3}",
401 filename, id, e.Message, e.StackTrace);
402 }
403 finally
404 {
405 if (stream != null)
406 stream.Close();
407 }
408 }
409  
410 return asset;
411 }
412  
413 public AssetBase Get(string id)
414 {
415 m_Requests++;
416  
417 AssetBase asset = null;
418  
419 if (m_MemoryCacheEnabled)
420 asset = GetFromMemoryCache(id);
421  
422 if (asset == null && m_FileCacheEnabled)
423 {
424 asset = GetFromFileCache(id);
425  
426 if (m_MemoryCacheEnabled && asset != null)
427 UpdateMemoryCache(id, asset);
428 }
429  
430 if (((m_LogLevel >= 1)) && (m_HitRateDisplay != 0) && (m_Requests % m_HitRateDisplay == 0))
431 {
432 m_HitRateFile = (double)m_DiskHits / m_Requests * 100.0;
433  
434 m_log.InfoFormat("[FLOTSAM ASSET CACHE]: Cache Get :: {0} :: {1}", id, asset == null ? "Miss" : "Hit");
435 m_log.InfoFormat("[FLOTSAM ASSET CACHE]: File Hit Rate {0}% for {1} requests", m_HitRateFile.ToString("0.00"), m_Requests);
436  
437 if (m_MemoryCacheEnabled)
438 {
439 m_HitRateMemory = (double)m_MemoryHits / m_Requests * 100.0;
440 m_log.InfoFormat("[FLOTSAM ASSET CACHE]: Memory Hit Rate {0}% for {1} requests", m_HitRateMemory.ToString("0.00"), m_Requests);
441 }
442  
443 m_log.InfoFormat("[FLOTSAM ASSET CACHE]: {0} unnessesary requests due to requests for assets that are currently downloading.", m_RequestsForInprogress);
444 }
445  
446 return asset;
447 }
448  
449 public AssetBase GetCached(string id)
450 {
451 return Get(id);
452 }
453  
454 public void Expire(string id)
455 {
456 if (m_LogLevel >= 2)
457 m_log.DebugFormat("[FLOTSAM ASSET CACHE]: Expiring Asset {0}", id);
458  
459 try
460 {
461 if (m_FileCacheEnabled)
462 {
463 string filename = GetFileName(id);
464 if (File.Exists(filename))
465 {
466 File.Delete(filename);
467 }
468 }
469  
470 if (m_MemoryCacheEnabled)
471 m_MemoryCache.Remove(id);
472 }
473 catch (Exception e)
474 {
475 m_log.WarnFormat(
476 "[FLOTSAM ASSET CACHE]: Failed to expire cached file {0}. Exception {1} {2}",
477 id, e.Message, e.StackTrace);
478 }
479 }
480  
481 public void Clear()
482 {
483 if (m_LogLevel >= 2)
484 m_log.Debug("[FLOTSAM ASSET CACHE]: Clearing caches.");
485  
486 if (m_FileCacheEnabled)
487 {
488 foreach (string dir in Directory.GetDirectories(m_CacheDirectory))
489 {
490 Directory.Delete(dir);
491 }
492 }
493  
494 if (m_MemoryCacheEnabled)
495 m_MemoryCache.Clear();
496 }
497  
498 private void CleanupExpiredFiles(object source, ElapsedEventArgs e)
499 {
500 if (m_LogLevel >= 2)
501 m_log.DebugFormat("[FLOTSAM ASSET CACHE]: Checking for expired files older then {0}.", m_FileExpiration);
502  
503 // Purge all files last accessed prior to this point
504 DateTime purgeLine = DateTime.Now - m_FileExpiration;
505  
506 // An asset cache may contain local non-temporary assets that are not in the asset service. Therefore,
507 // before cleaning up expired files we must scan the objects in the scene to make sure that we retain
508 // such local assets if they have not been recently accessed.
509 TouchAllSceneAssets(false);
510  
511 foreach (string dir in Directory.GetDirectories(m_CacheDirectory))
512 {
513 CleanExpiredFiles(dir, purgeLine);
514 }
515 }
516  
517 /// <summary>
518 /// Recurses through specified directory checking for asset files last
519 /// accessed prior to the specified purge line and deletes them. Also
520 /// removes empty tier directories.
521 /// </summary>
522 /// <param name="dir"></param>
523 /// <param name="purgeLine"></param>
524 private void CleanExpiredFiles(string dir, DateTime purgeLine)
525 {
526 try
527 {
528 foreach (string file in Directory.GetFiles(dir))
529 {
530 if (File.GetLastAccessTime(file) < purgeLine)
531 {
532 File.Delete(file);
533 }
534 }
535  
536 // Recurse into lower tiers
537 foreach (string subdir in Directory.GetDirectories(dir))
538 {
539 CleanExpiredFiles(subdir, purgeLine);
540 }
541  
542 // Check if a tier directory is empty, if so, delete it
543 int dirSize = Directory.GetFiles(dir).Length + Directory.GetDirectories(dir).Length;
544 if (dirSize == 0)
545 {
546 Directory.Delete(dir);
547 }
548 else if (dirSize >= m_CacheWarnAt)
549 {
550 m_log.WarnFormat(
551 "[FLOTSAM ASSET CACHE]: Cache folder exceeded CacheWarnAt limit {0} {1}. Suggest increasing tiers, tier length, or reducing cache expiration",
552 dir, dirSize);
553 }
554 }
555 catch (Exception e)
556 {
557 m_log.Warn(
558 string.Format("[FLOTSAM ASSET CACHE]: Could not complete clean of expired files in {0}, exception ", dir), e);
559 }
560 }
561  
562 /// <summary>
563 /// Determines the filename for an AssetID stored in the file cache
564 /// </summary>
565 /// <param name="id"></param>
566 /// <returns></returns>
567 private string GetFileName(string id)
568 {
569 // Would it be faster to just hash the darn thing?
570 foreach (char c in m_InvalidChars)
571 {
572 id = id.Replace(c, '_');
573 }
574  
575 string path = m_CacheDirectory;
576 for (int p = 1; p <= m_CacheDirectoryTiers; p++)
577 {
578 string pathPart = id.Substring((p - 1) * m_CacheDirectoryTierLen, m_CacheDirectoryTierLen);
579 path = Path.Combine(path, pathPart);
580 }
581  
582 return Path.Combine(path, id);
583 }
584  
585 /// <summary>
586 /// Writes a file to the file cache, creating any nessesary
587 /// tier directories along the way
588 /// </summary>
589 /// <param name="filename"></param>
590 /// <param name="asset"></param>
591 private void WriteFileCache(string filename, AssetBase asset)
592 {
593 Stream stream = null;
594  
595 // Make sure the target cache directory exists
596 string directory = Path.GetDirectoryName(filename);
597  
598 // Write file first to a temp name, so that it doesn't look
599 // like it's already cached while it's still writing.
600 string tempname = Path.Combine(directory, Path.GetRandomFileName());
601  
602 try
603 {
604 try
605 {
606 if (!Directory.Exists(directory))
607 {
608 Directory.CreateDirectory(directory);
609 }
610  
611 stream = File.Open(tempname, FileMode.Create);
612 BinaryFormatter bformatter = new BinaryFormatter();
613 bformatter.Serialize(stream, asset);
614 }
615 catch (IOException e)
616 {
617 m_log.WarnFormat(
618 "[FLOTSAM ASSET CACHE]: Failed to write asset {0} to temporary location {1} (final {2}) on cache in {3}. Exception {4} {5}.",
619 asset.ID, tempname, filename, directory, e.Message, e.StackTrace);
620  
621 return;
622 }
623 finally
624 {
625 if (stream != null)
626 stream.Close();
627 }
628  
629 try
630 {
631 // Now that it's written, rename it so that it can be found.
632 //
633 // File.Copy(tempname, filename, true);
634 // File.Delete(tempname);
635 //
636 // For a brief period, this was done as a separate copy and then temporary file delete operation to
637 // avoid an IOException caused by move if some competing thread had already written the file.
638 // However, this causes exceptions on Windows when other threads attempt to read a file
639 // which is still being copied. So instead, go back to moving the file and swallow any IOException.
640 //
641 // This situation occurs fairly rarely anyway. We assume in this that moves are atomic on the
642 // filesystem.
643 File.Move(tempname, filename);
644  
645 if (m_LogLevel >= 2)
646 m_log.DebugFormat("[FLOTSAM ASSET CACHE]: Cache Stored :: {0}", asset.ID);
647 }
648 catch (IOException)
649 {
650 // If we see an IOException here it's likely that some other competing thread has written the
651 // cache file first, so ignore. Other IOException errors (e.g. filesystem full) should be
652 // signally by the earlier temporary file writing code.
653 }
654 }
655 finally
656 {
657 // Even if the write fails with an exception, we need to make sure
658 // that we release the lock on that file, otherwise it'll never get
659 // cached
660 lock (m_CurrentlyWriting)
661 {
662 #if WAIT_ON_INPROGRESS_REQUESTS
663 ManualResetEvent waitEvent;
664 if (m_CurrentlyWriting.TryGetValue(filename, out waitEvent))
665 {
666 m_CurrentlyWriting.Remove(filename);
667 waitEvent.Set();
668 }
669 #else
670 m_CurrentlyWriting.Remove(filename);
671 #endif
672 }
673 }
674 }
675  
676 /// <summary>
677 /// Scan through the file cache, and return number of assets currently cached.
678 /// </summary>
679 /// <param name="dir"></param>
680 /// <returns></returns>
681 private int GetFileCacheCount(string dir)
682 {
683 int count = Directory.GetFiles(dir).Length;
684  
685 foreach (string subdir in Directory.GetDirectories(dir))
686 {
687 count += GetFileCacheCount(subdir);
688 }
689  
690 return count;
691 }
692  
693 /// <summary>
694 /// This notes the last time the Region had a deep asset scan performed on it.
695 /// </summary>
696 /// <param name="regionID"></param>
697 private void StampRegionStatusFile(UUID regionID)
698 {
699 string RegionCacheStatusFile = Path.Combine(m_CacheDirectory, "RegionStatus_" + regionID.ToString() + ".fac");
700  
701 try
702 {
703 if (File.Exists(RegionCacheStatusFile))
704 {
705 File.SetLastWriteTime(RegionCacheStatusFile, DateTime.Now);
706 }
707 else
708 {
709 File.WriteAllText(
710 RegionCacheStatusFile,
711 "Please do not delete this file unless you are manually clearing your Flotsam Asset Cache.");
712 }
713 }
714 catch (Exception e)
715 {
716 m_log.Warn(
717 string.Format(
718 "[FLOTSAM ASSET CACHE]: Could not stamp region status file for region {0}. Exception ",
719 regionID),
720 e);
721 }
722 }
723  
724 /// <summary>
725 /// Iterates through all Scenes, doing a deep scan through assets
726 /// to update the access time of all assets present in the scene or referenced by assets
727 /// in the scene.
728 /// </summary>
729 /// <param name="storeUncached">
730 /// If true, then assets scanned which are not found in cache are added to the cache.
731 /// </param>
732 /// <returns>Number of distinct asset references found in the scene.</returns>
733 private int TouchAllSceneAssets(bool storeUncached)
734 {
735 UuidGatherer gatherer = new UuidGatherer(m_AssetService);
736  
737 HashSet<UUID> uniqueUuids = new HashSet<UUID>();
738 Dictionary<UUID, AssetType> assets = new Dictionary<UUID, AssetType>();
739  
740 foreach (Scene s in m_Scenes)
741 {
742 StampRegionStatusFile(s.RegionInfo.RegionID);
743  
744 s.ForEachSOG(delegate(SceneObjectGroup e)
745 {
746 gatherer.GatherAssetUuids(e, assets);
747  
748 foreach (UUID assetID in assets.Keys)
749 {
750 uniqueUuids.Add(assetID);
751  
752 string filename = GetFileName(assetID.ToString());
753  
754 if (File.Exists(filename))
755 {
756 File.SetLastAccessTime(filename, DateTime.Now);
757 }
758 else if (storeUncached)
759 {
760 AssetBase cachedAsset = m_AssetService.Get(assetID.ToString());
761 if (cachedAsset == null && assets[assetID] != AssetType.Unknown)
762 m_log.DebugFormat(
763 "[FLOTSAM ASSET CACHE]: Could not find asset {0}, type {1} referenced by object {2} at {3} in scene {4} when pre-caching all scene assets",
764 assetID, assets[assetID], e.Name, e.AbsolutePosition, s.Name);
765 }
766 }
767  
768 assets.Clear();
769 });
770 }
771  
772  
773 return uniqueUuids.Count;
774 }
775  
776 /// <summary>
777 /// Deletes all cache contents
778 /// </summary>
779 private void ClearFileCache()
780 {
781 foreach (string dir in Directory.GetDirectories(m_CacheDirectory))
782 {
783 try
784 {
785 Directory.Delete(dir, true);
786 }
787 catch (Exception e)
788 {
789 m_log.WarnFormat(
790 "[FLOTSAM ASSET CACHE]: Couldn't clear asset cache directory {0} from {1}. Exception {2} {3}",
791 dir, m_CacheDirectory, e.Message, e.StackTrace);
792 }
793 }
794  
795 foreach (string file in Directory.GetFiles(m_CacheDirectory))
796 {
797 try
798 {
799 File.Delete(file);
800 }
801 catch (Exception e)
802 {
803 m_log.WarnFormat(
804 "[FLOTSAM ASSET CACHE]: Couldn't clear asset cache file {0} from {1}. Exception {1} {2}",
805 file, m_CacheDirectory, e.Message, e.StackTrace);
806 }
807 }
808 }
809  
810 #region Console Commands
811 private void HandleConsoleCommand(string module, string[] cmdparams)
812 {
813 if (cmdparams.Length >= 2)
814 {
815 string cmd = cmdparams[1];
816 switch (cmd)
817 {
818 case "status":
819 if (m_MemoryCacheEnabled)
820 m_log.InfoFormat("[FLOTSAM ASSET CACHE]: Memory Cache : {0} assets", m_MemoryCache.Count);
821 else
822 m_log.InfoFormat("[FLOTSAM ASSET CACHE]: Memory cache disabled");
823  
824 if (m_FileCacheEnabled)
825 {
826 int fileCount = GetFileCacheCount(m_CacheDirectory);
827 m_log.InfoFormat("[FLOTSAM ASSET CACHE]: File Cache : {0} assets", fileCount);
828  
829 foreach (string s in Directory.GetFiles(m_CacheDirectory, "*.fac"))
830 {
831 m_log.Info("[FLOTSAM ASSET CACHE]: Deep scans have previously been performed on the following regions:");
832  
833 string RegionID = s.Remove(0,s.IndexOf("_")).Replace(".fac","");
834 DateTime RegionDeepScanTMStamp = File.GetLastWriteTime(s);
835 m_log.InfoFormat("[FLOTSAM ASSET CACHE]: Region: {0}, {1}", RegionID, RegionDeepScanTMStamp.ToString("MM/dd/yyyy hh:mm:ss"));
836 }
837 }
838 else
839 {
840 m_log.InfoFormat("[FLOTSAM ASSET CACHE]: File cache disabled");
841 }
842  
843 break;
844  
845 case "clear":
846 if (cmdparams.Length < 2)
847 {
848 m_log.Warn("[FLOTSAM ASSET CACHE]: Usage is fcache clear [file] [memory]");
849 break;
850 }
851  
852 bool clearMemory = false, clearFile = false;
853  
854 if (cmdparams.Length == 2)
855 {
856 clearMemory = true;
857 clearFile = true;
858 }
859 foreach (string s in cmdparams)
860 {
861 if (s.ToLower() == "memory")
862 clearMemory = true;
863 else if (s.ToLower() == "file")
864 clearFile = true;
865 }
866  
867 if (clearMemory)
868 {
869 if (m_MemoryCacheEnabled)
870 {
871 m_MemoryCache.Clear();
872 m_log.Info("[FLOTSAM ASSET CACHE]: Memory cache cleared.");
873 }
874 else
875 {
876 m_log.Info("[FLOTSAM ASSET CACHE]: Memory cache not enabled.");
877 }
878 }
879  
880 if (clearFile)
881 {
882 if (m_FileCacheEnabled)
883 {
884 ClearFileCache();
885 m_log.Info("[FLOTSAM ASSET CACHE]: File cache cleared.");
886 }
887 else
888 {
889 m_log.Info("[FLOTSAM ASSET CACHE]: File cache not enabled.");
890 }
891 }
892  
893 break;
894  
895 case "assets":
896 m_log.Info("[FLOTSAM ASSET CACHE]: Ensuring assets are cached for all scenes.");
897  
898 Util.FireAndForget(delegate {
899 int assetReferenceTotal = TouchAllSceneAssets(true);
900 m_log.InfoFormat(
901 "[FLOTSAM ASSET CACHE]: Completed check with {0} assets.",
902 assetReferenceTotal);
903 });
904  
905 break;
906  
907 case "expire":
908 if (cmdparams.Length < 3)
909 {
910 m_log.InfoFormat("[FLOTSAM ASSET CACHE]: Invalid parameters for Expire, please specify a valid date & time", cmd);
911 break;
912 }
913  
914 string s_expirationDate = "";
915 DateTime expirationDate;
916  
917 if (cmdparams.Length > 3)
918 {
919 s_expirationDate = string.Join(" ", cmdparams, 2, cmdparams.Length - 2);
920 }
921 else
922 {
923 s_expirationDate = cmdparams[2];
924 }
925  
926 if (!DateTime.TryParse(s_expirationDate, out expirationDate))
927 {
928 m_log.InfoFormat("[FLOTSAM ASSET CACHE]: {0} is not a valid date & time", cmd);
929 break;
930 }
931  
932 if (m_FileCacheEnabled)
933 CleanExpiredFiles(m_CacheDirectory, expirationDate);
934 else
935 m_log.InfoFormat("[FLOTSAM ASSET CACHE]: File cache not active, not clearing.");
936  
937 break;
938 default:
939 m_log.InfoFormat("[FLOTSAM ASSET CACHE]: Unknown command {0}", cmd);
940 break;
941 }
942 }
943 else if (cmdparams.Length == 1)
944 {
945 m_log.InfoFormat("[FLOTSAM ASSET CACHE]: fcache status - Display cache status");
946 m_log.InfoFormat("[FLOTSAM ASSET CACHE]: fcache clearmem - Remove all assets cached in memory");
947 m_log.InfoFormat("[FLOTSAM ASSET CACHE]: fcache clearfile - Remove all assets cached on disk");
948 m_log.InfoFormat("[FLOTSAM ASSET CACHE]: fcache cachescenes - Attempt a deep cache of all assets in all scenes");
949 m_log.InfoFormat("[FLOTSAM ASSET CACHE]: fcache <datetime> - Purge assets older then the specified date & time");
950 }
951 }
952  
953 #endregion
954  
955 #region IAssetService Members
956  
957 public AssetMetadata GetMetadata(string id)
958 {
959 AssetBase asset = Get(id);
960 return asset.Metadata;
961 }
962  
963 public byte[] GetData(string id)
964 {
965 AssetBase asset = Get(id);
966 return asset.Data;
967 }
968  
969 public bool Get(string id, object sender, AssetRetrieved handler)
970 {
971 AssetBase asset = Get(id);
972 handler(id, sender, asset);
973 return true;
974 }
975  
976 public string Store(AssetBase asset)
977 {
978 if (asset.FullID == UUID.Zero)
979 {
980 asset.FullID = UUID.Random();
981 }
982  
983 Cache(asset);
984  
985 return asset.ID;
986 }
987  
988 public bool UpdateContent(string id, byte[] data)
989 {
990 AssetBase asset = Get(id);
991 asset.Data = data;
992 Cache(asset);
993 return true;
994 }
995  
996 public bool Delete(string id)
997 {
998 Expire(id);
999 return true;
1000 }
1001  
1002 #endregion
1003 }
1004 }