clockwerk-opensim – Blame information for rev 1

Subversion Repositories:
Rev:
Rev Author Line No. Line
1 vero 1 /*
2 * Copyright (c) Contributors, http://opensimulator.org/
3 * See CONTRIBUTORS.TXT for a full list of copyright holders.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 * * Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * * Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 * * Neither the name of the OpenSimulator Project nor the
13 * names of its contributors may be used to endorse or promote products
14 * derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY
17 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY
20 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
23 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27  
28 // Uncomment to make asset Get requests for existing
29 // #define WAIT_ON_INPROGRESS_REQUESTS
30  
31 using System;
32 using System.IO;
33 using System.Collections.Generic;
34 using System.Linq;
35 using System.Reflection;
36 using System.Runtime.Serialization;
37 using System.Runtime.Serialization.Formatters.Binary;
38 using System.Threading;
39 using System.Timers;
40 using log4net;
41 using Nini.Config;
42 using Mono.Addins;
43 using OpenMetaverse;
44 using OpenSim.Framework;
45 using OpenSim.Framework.Console;
46 using OpenSim.Framework.Monitoring;
47 using OpenSim.Region.Framework.Interfaces;
48 using OpenSim.Region.Framework.Scenes;
49 using OpenSim.Services.Interfaces;
50  
51  
52 //[assembly: Addin("FlotsamAssetCache", "1.1")]
53 //[assembly: AddinDependency("OpenSim", "0.5")]
54  
55 namespace OpenSim.Region.CoreModules.Asset
56 {
57 [Extension(Path = "/OpenSim/RegionModules", NodeName = "RegionModule", Id = "FlotsamAssetCache")]
58 public class FlotsamAssetCache : ISharedRegionModule, IImprovedAssetCache, IAssetService
59 {
60 private static readonly ILog m_log =
61 LogManager.GetLogger(
62 MethodBase.GetCurrentMethod().DeclaringType);
63  
64 private bool m_Enabled;
65  
66 private const string m_ModuleName = "FlotsamAssetCache";
67 private const string m_DefaultCacheDirectory = "./assetcache";
68 private string m_CacheDirectory = m_DefaultCacheDirectory;
69  
70 private readonly List<char> m_InvalidChars = new List<char>();
71  
72 private int m_LogLevel = 0;
73 private ulong m_HitRateDisplay = 100; // How often to display hit statistics, given in requests
74  
75 private static ulong m_Requests;
76 private static ulong m_RequestsForInprogress;
77 private static ulong m_DiskHits;
78 private static ulong m_MemoryHits;
79  
80 #if WAIT_ON_INPROGRESS_REQUESTS
81 private Dictionary<string, ManualResetEvent> m_CurrentlyWriting = new Dictionary<string, ManualResetEvent>();
82 private int m_WaitOnInprogressTimeout = 3000;
83 #else
84 private HashSet<string> m_CurrentlyWriting = new HashSet<string>();
85 #endif
86  
87 private bool m_FileCacheEnabled = true;
88  
89 private ExpiringCache<string, AssetBase> m_MemoryCache;
90 private bool m_MemoryCacheEnabled = false;
91  
92 // Expiration is expressed in hours.
93 private const double m_DefaultMemoryExpiration = 2;
94 private const double m_DefaultFileExpiration = 48;
95 private TimeSpan m_MemoryExpiration = TimeSpan.FromHours(m_DefaultMemoryExpiration);
96 private TimeSpan m_FileExpiration = TimeSpan.FromHours(m_DefaultFileExpiration);
97 private TimeSpan m_FileExpirationCleanupTimer = TimeSpan.FromHours(0.166);
98  
99 private static int m_CacheDirectoryTiers = 1;
100 private static int m_CacheDirectoryTierLen = 3;
101 private static int m_CacheWarnAt = 30000;
102  
103 private System.Timers.Timer m_CacheCleanTimer;
104  
105 private IAssetService m_AssetService;
106 private List<Scene> m_Scenes = new List<Scene>();
107  
108 public FlotsamAssetCache()
109 {
110 m_InvalidChars.AddRange(Path.GetInvalidPathChars());
111 m_InvalidChars.AddRange(Path.GetInvalidFileNameChars());
112 }
113  
114 public Type ReplaceableInterface
115 {
116 get { return null; }
117 }
118  
119 public string Name
120 {
121 get { return m_ModuleName; }
122 }
123  
124 public void Initialise(IConfigSource source)
125 {
126 IConfig moduleConfig = source.Configs["Modules"];
127  
128 if (moduleConfig != null)
129 {
130 string name = moduleConfig.GetString("AssetCaching", String.Empty);
131  
132 if (name == Name)
133 {
134 m_MemoryCache = new ExpiringCache<string, AssetBase>();
135 m_Enabled = true;
136  
137 m_log.InfoFormat("[FLOTSAM ASSET CACHE]: {0} enabled", this.Name);
138  
139 IConfig assetConfig = source.Configs["AssetCache"];
140 if (assetConfig == null)
141 {
142 m_log.Debug(
143 "[FLOTSAM ASSET CACHE]: AssetCache section missing from config (not copied config-include/FlotsamCache.ini.example? Using defaults.");
144 }
145 else
146 {
147 m_FileCacheEnabled = assetConfig.GetBoolean("FileCacheEnabled", m_FileCacheEnabled);
148 m_CacheDirectory = assetConfig.GetString("CacheDirectory", m_DefaultCacheDirectory);
149  
150 m_MemoryCacheEnabled = assetConfig.GetBoolean("MemoryCacheEnabled", m_MemoryCacheEnabled);
151 m_MemoryExpiration = TimeSpan.FromHours(assetConfig.GetDouble("MemoryCacheTimeout", m_DefaultMemoryExpiration));
152  
153 #if WAIT_ON_INPROGRESS_REQUESTS
154 m_WaitOnInprogressTimeout = assetConfig.GetInt("WaitOnInprogressTimeout", 3000);
155 #endif
156  
157 m_LogLevel = assetConfig.GetInt("LogLevel", m_LogLevel);
158 m_HitRateDisplay = (ulong)assetConfig.GetLong("HitRateDisplay", (long)m_HitRateDisplay);
159  
160 m_FileExpiration = TimeSpan.FromHours(assetConfig.GetDouble("FileCacheTimeout", m_DefaultFileExpiration));
161 m_FileExpirationCleanupTimer
162 = TimeSpan.FromHours(
163 assetConfig.GetDouble("FileCleanupTimer", m_FileExpirationCleanupTimer.TotalHours));
164  
165 m_CacheDirectoryTiers = assetConfig.GetInt("CacheDirectoryTiers", m_CacheDirectoryTiers);
166 m_CacheDirectoryTierLen = assetConfig.GetInt("CacheDirectoryTierLength", m_CacheDirectoryTierLen);
167  
168 m_CacheWarnAt = assetConfig.GetInt("CacheWarnAt", m_CacheWarnAt);
169 }
170  
171 m_log.InfoFormat("[FLOTSAM ASSET CACHE]: Cache Directory {0}", m_CacheDirectory);
172  
173 if (m_FileCacheEnabled && (m_FileExpiration > TimeSpan.Zero) && (m_FileExpirationCleanupTimer > TimeSpan.Zero))
174 {
175 m_CacheCleanTimer = new System.Timers.Timer(m_FileExpirationCleanupTimer.TotalMilliseconds);
176 m_CacheCleanTimer.AutoReset = true;
177 m_CacheCleanTimer.Elapsed += CleanupExpiredFiles;
178 lock (m_CacheCleanTimer)
179 m_CacheCleanTimer.Start();
180 }
181  
182 if (m_CacheDirectoryTiers < 1)
183 {
184 m_CacheDirectoryTiers = 1;
185 }
186 else if (m_CacheDirectoryTiers > 3)
187 {
188 m_CacheDirectoryTiers = 3;
189 }
190  
191 if (m_CacheDirectoryTierLen < 1)
192 {
193 m_CacheDirectoryTierLen = 1;
194 }
195 else if (m_CacheDirectoryTierLen > 4)
196 {
197 m_CacheDirectoryTierLen = 4;
198 }
199  
200 MainConsole.Instance.Commands.AddCommand("Assets", true, "fcache status", "fcache status", "Display cache status", HandleConsoleCommand);
201 MainConsole.Instance.Commands.AddCommand("Assets", true, "fcache clear", "fcache clear [file] [memory]", "Remove all assets in the cache. If file or memory is specified then only this cache is cleared.", HandleConsoleCommand);
202 MainConsole.Instance.Commands.AddCommand("Assets", true, "fcache assets", "fcache assets", "Attempt a deep scan and cache of all assets in all scenes", HandleConsoleCommand);
203 MainConsole.Instance.Commands.AddCommand("Assets", true, "fcache expire", "fcache expire <datetime>", "Purge cached assets older then the specified date/time", HandleConsoleCommand);
204 }
205 }
206 }
207  
208 public void PostInitialise()
209 {
210 }
211  
212 public void Close()
213 {
214 }
215  
216 public void AddRegion(Scene scene)
217 {
218 if (m_Enabled)
219 {
220 scene.RegisterModuleInterface<IImprovedAssetCache>(this);
221 m_Scenes.Add(scene);
222  
223 }
224 }
225  
226 public void RemoveRegion(Scene scene)
227 {
228 if (m_Enabled)
229 {
230 scene.UnregisterModuleInterface<IImprovedAssetCache>(this);
231 m_Scenes.Remove(scene);
232 }
233 }
234  
235 public void RegionLoaded(Scene scene)
236 {
237 if (m_Enabled && m_AssetService == null)
238 m_AssetService = scene.RequestModuleInterface<IAssetService>();
239 }
240  
241 ////////////////////////////////////////////////////////////
242 // IImprovedAssetCache
243 //
244  
245 private void UpdateMemoryCache(string key, AssetBase asset)
246 {
247 m_MemoryCache.AddOrUpdate(key, asset, m_MemoryExpiration);
248 }
249  
250 private void UpdateFileCache(string key, AssetBase asset)
251 {
252 string filename = GetFileName(key);
253  
254 try
255 {
256 // If the file is already cached, don't cache it, just touch it so access time is updated
257 if (File.Exists(filename))
258 {
259 // We don't really want to know about sharing
260 // violations here. If the file is locked, then
261 // the other thread has updated the time for us.
262 try
263 {
264 lock (m_CurrentlyWriting)
265 {
266 if (!m_CurrentlyWriting.Contains(filename))
267 File.SetLastAccessTime(filename, DateTime.Now);
268 }
269 }
270 catch
271 {
272 }
273 }
274 else
275 {
276 // Once we start writing, make sure we flag that we're writing
277 // that object to the cache so that we don't try to write the
278 // same file multiple times.
279 lock (m_CurrentlyWriting)
280 {
281 #if WAIT_ON_INPROGRESS_REQUESTS
282 if (m_CurrentlyWriting.ContainsKey(filename))
283 {
284 return;
285 }
286 else
287 {
288 m_CurrentlyWriting.Add(filename, new ManualResetEvent(false));
289 }
290  
291 #else
292 if (m_CurrentlyWriting.Contains(filename))
293 {
294 return;
295 }
296 else
297 {
298 m_CurrentlyWriting.Add(filename);
299 }
300 #endif
301  
302 }
303  
304 Util.FireAndForget(
305 delegate { WriteFileCache(filename, asset); });
306 }
307 }
308 catch (Exception e)
309 {
310 m_log.ErrorFormat(
311 "[FLOTSAM ASSET CACHE]: Failed to update cache for asset {0}. Exception {1} {2}",
312 asset.ID, e.Message, e.StackTrace);
313 }
314 }
315  
316 public void Cache(AssetBase asset)
317 {
318 // TODO: Spawn this off to some seperate thread to do the actual writing
319 if (asset != null)
320 {
321 //m_log.DebugFormat("[FLOTSAM ASSET CACHE]: Caching asset with id {0}", asset.ID);
322  
323 if (m_MemoryCacheEnabled)
324 UpdateMemoryCache(asset.ID, asset);
325  
326 if (m_FileCacheEnabled)
327 UpdateFileCache(asset.ID, asset);
328 }
329 }
330  
331 /// <summary>
332 /// Try to get an asset from the in-memory cache.
333 /// </summary>
334 /// <param name="id"></param>
335 /// <returns></returns>
336 private AssetBase GetFromMemoryCache(string id)
337 {
338 AssetBase asset = null;
339  
340 if (m_MemoryCache.TryGetValue(id, out asset))
341 m_MemoryHits++;
342  
343 return asset;
344 }
345  
346 private bool CheckFromMemoryCache(string id)
347 {
348 return m_MemoryCache.Contains(id);
349 }
350  
351 /// <summary>
352 /// Try to get an asset from the file cache.
353 /// </summary>
354 /// <param name="id"></param>
355 /// <returns>An asset retrieved from the file cache. null if there was a problem retrieving an asset.</returns>
356 private AssetBase GetFromFileCache(string id)
357 {
358 string filename = GetFileName(id);
359  
360 #if WAIT_ON_INPROGRESS_REQUESTS
361 // Check if we're already downloading this asset. If so, try to wait for it to
362 // download.
363 if (m_WaitOnInprogressTimeout > 0)
364 {
365 m_RequestsForInprogress++;
366  
367 ManualResetEvent waitEvent;
368 if (m_CurrentlyWriting.TryGetValue(filename, out waitEvent))
369 {
370 waitEvent.WaitOne(m_WaitOnInprogressTimeout);
371 return Get(id);
372 }
373 }
374 #else
375 // Track how often we have the problem that an asset is requested while
376 // it is still being downloaded by a previous request.
377 if (m_CurrentlyWriting.Contains(filename))
378 {
379 m_RequestsForInprogress++;
380 return null;
381 }
382 #endif
383  
384 AssetBase asset = null;
385  
386 if (File.Exists(filename))
387 {
388 try
389 {
390 using (FileStream stream = File.Open(filename, FileMode.Open, FileAccess.Read, FileShare.Read))
391 {
392 BinaryFormatter bformatter = new BinaryFormatter();
393  
394 asset = (AssetBase)bformatter.Deserialize(stream);
395  
396 m_DiskHits++;
397 }
398 }
399 catch (System.Runtime.Serialization.SerializationException e)
400 {
401 m_log.WarnFormat(
402 "[FLOTSAM ASSET CACHE]: Failed to get file {0} for asset {1}. Exception {2} {3}",
403 filename, id, e.Message, e.StackTrace);
404  
405 // If there was a problem deserializing the asset, the asset may
406 // either be corrupted OR was serialized under an old format
407 // {different version of AssetBase} -- we should attempt to
408 // delete it and re-cache
409 File.Delete(filename);
410 }
411 catch (Exception e)
412 {
413 m_log.WarnFormat(
414 "[FLOTSAM ASSET CACHE]: Failed to get file {0} for asset {1}. Exception {2} {3}",
415 filename, id, e.Message, e.StackTrace);
416 }
417 }
418  
419 return asset;
420 }
421  
422 private bool CheckFromFileCache(string id)
423 {
424 bool found = false;
425  
426 string filename = GetFileName(id);
427  
428 if (File.Exists(filename))
429 {
430 try
431 {
432 using (FileStream stream = File.Open(filename, FileMode.Open, FileAccess.Read, FileShare.Read))
433 {
434 if (stream != null)
435 found = true;
436 }
437 }
438 catch (Exception e)
439 {
440 m_log.ErrorFormat(
441 "[FLOTSAM ASSET CACHE]: Failed to check file {0} for asset {1}. Exception {2} {3}",
442 filename, id, e.Message, e.StackTrace);
443 }
444 }
445  
446 return found;
447 }
448  
449 public AssetBase Get(string id)
450 {
451 m_Requests++;
452  
453 AssetBase asset = null;
454  
455 if (m_MemoryCacheEnabled)
456 asset = GetFromMemoryCache(id);
457  
458 if (asset == null && m_FileCacheEnabled)
459 {
460 asset = GetFromFileCache(id);
461  
462 if (m_MemoryCacheEnabled && asset != null)
463 UpdateMemoryCache(id, asset);
464 }
465  
466 if (((m_LogLevel >= 1)) && (m_HitRateDisplay != 0) && (m_Requests % m_HitRateDisplay == 0))
467 {
468 m_log.InfoFormat("[FLOTSAM ASSET CACHE]: Cache Get :: {0} :: {1}", id, asset == null ? "Miss" : "Hit");
469  
470 GenerateCacheHitReport().ForEach(l => m_log.InfoFormat("[FLOTSAM ASSET CACHE]: {0}", l));
471 }
472  
473 return asset;
474 }
475  
476 public bool Check(string id)
477 {
478 if (m_MemoryCacheEnabled && CheckFromMemoryCache(id))
479 return true;
480  
481 if (m_FileCacheEnabled && CheckFromFileCache(id))
482 return true;
483 return false;
484 }
485  
486 public AssetBase GetCached(string id)
487 {
488 return Get(id);
489 }
490  
491 public void Expire(string id)
492 {
493 if (m_LogLevel >= 2)
494 m_log.DebugFormat("[FLOTSAM ASSET CACHE]: Expiring Asset {0}", id);
495  
496 try
497 {
498 if (m_FileCacheEnabled)
499 {
500 string filename = GetFileName(id);
501 if (File.Exists(filename))
502 {
503 File.Delete(filename);
504 }
505 }
506  
507 if (m_MemoryCacheEnabled)
508 m_MemoryCache.Remove(id);
509 }
510 catch (Exception e)
511 {
512 m_log.WarnFormat(
513 "[FLOTSAM ASSET CACHE]: Failed to expire cached file {0}. Exception {1} {2}",
514 id, e.Message, e.StackTrace);
515 }
516 }
517  
518 public void Clear()
519 {
520 if (m_LogLevel >= 2)
521 m_log.Debug("[FLOTSAM ASSET CACHE]: Clearing caches.");
522  
523 if (m_FileCacheEnabled)
524 {
525 foreach (string dir in Directory.GetDirectories(m_CacheDirectory))
526 {
527 Directory.Delete(dir);
528 }
529 }
530  
531 if (m_MemoryCacheEnabled)
532 m_MemoryCache.Clear();
533 }
534  
535 private void CleanupExpiredFiles(object source, ElapsedEventArgs e)
536 {
537 if (m_LogLevel >= 2)
538 m_log.DebugFormat("[FLOTSAM ASSET CACHE]: Checking for expired files older then {0}.", m_FileExpiration);
539  
540 // Purge all files last accessed prior to this point
541 DateTime purgeLine = DateTime.Now - m_FileExpiration;
542  
543 // An asset cache may contain local non-temporary assets that are not in the asset service. Therefore,
544 // before cleaning up expired files we must scan the objects in the scene to make sure that we retain
545 // such local assets if they have not been recently accessed.
546 TouchAllSceneAssets(false);
547  
548 foreach (string dir in Directory.GetDirectories(m_CacheDirectory))
549 {
550 CleanExpiredFiles(dir, purgeLine);
551 }
552 }
553  
554 /// <summary>
555 /// Recurses through specified directory checking for asset files last
556 /// accessed prior to the specified purge line and deletes them. Also
557 /// removes empty tier directories.
558 /// </summary>
559 /// <param name="dir"></param>
560 /// <param name="purgeLine"></param>
561 private void CleanExpiredFiles(string dir, DateTime purgeLine)
562 {
563 try
564 {
565 foreach (string file in Directory.GetFiles(dir))
566 {
567 if (File.GetLastAccessTime(file) < purgeLine)
568 {
569 File.Delete(file);
570 }
571 }
572  
573 // Recurse into lower tiers
574 foreach (string subdir in Directory.GetDirectories(dir))
575 {
576 CleanExpiredFiles(subdir, purgeLine);
577 }
578  
579 // Check if a tier directory is empty, if so, delete it
580 int dirSize = Directory.GetFiles(dir).Length + Directory.GetDirectories(dir).Length;
581 if (dirSize == 0)
582 {
583 Directory.Delete(dir);
584 }
585 else if (dirSize >= m_CacheWarnAt)
586 {
587 m_log.WarnFormat(
588 "[FLOTSAM ASSET CACHE]: Cache folder exceeded CacheWarnAt limit {0} {1}. Suggest increasing tiers, tier length, or reducing cache expiration",
589 dir, dirSize);
590 }
591 }
592 catch (Exception e)
593 {
594 m_log.Warn(
595 string.Format("[FLOTSAM ASSET CACHE]: Could not complete clean of expired files in {0}, exception ", dir), e);
596 }
597 }
598  
599 /// <summary>
600 /// Determines the filename for an AssetID stored in the file cache
601 /// </summary>
602 /// <param name="id"></param>
603 /// <returns></returns>
604 private string GetFileName(string id)
605 {
606 // Would it be faster to just hash the darn thing?
607 foreach (char c in m_InvalidChars)
608 {
609 id = id.Replace(c, '_');
610 }
611  
612 string path = m_CacheDirectory;
613 for (int p = 1; p <= m_CacheDirectoryTiers; p++)
614 {
615 string pathPart = id.Substring((p - 1) * m_CacheDirectoryTierLen, m_CacheDirectoryTierLen);
616 path = Path.Combine(path, pathPart);
617 }
618  
619 return Path.Combine(path, id);
620 }
621  
622 /// <summary>
623 /// Writes a file to the file cache, creating any nessesary
624 /// tier directories along the way
625 /// </summary>
626 /// <param name="filename"></param>
627 /// <param name="asset"></param>
628 private void WriteFileCache(string filename, AssetBase asset)
629 {
630 Stream stream = null;
631  
632 // Make sure the target cache directory exists
633 string directory = Path.GetDirectoryName(filename);
634  
635 // Write file first to a temp name, so that it doesn't look
636 // like it's already cached while it's still writing.
637 string tempname = Path.Combine(directory, Path.GetRandomFileName());
638  
639 try
640 {
641 try
642 {
643 if (!Directory.Exists(directory))
644 {
645 Directory.CreateDirectory(directory);
646 }
647  
648 stream = File.Open(tempname, FileMode.Create);
649 BinaryFormatter bformatter = new BinaryFormatter();
650 bformatter.Serialize(stream, asset);
651 }
652 catch (IOException e)
653 {
654 m_log.WarnFormat(
655 "[FLOTSAM ASSET CACHE]: Failed to write asset {0} to temporary location {1} (final {2}) on cache in {3}. Exception {4} {5}.",
656 asset.ID, tempname, filename, directory, e.Message, e.StackTrace);
657  
658 return;
659 }
660 finally
661 {
662 if (stream != null)
663 stream.Close();
664 }
665  
666 try
667 {
668 // Now that it's written, rename it so that it can be found.
669 //
670 // File.Copy(tempname, filename, true);
671 // File.Delete(tempname);
672 //
673 // For a brief period, this was done as a separate copy and then temporary file delete operation to
674 // avoid an IOException caused by move if some competing thread had already written the file.
675 // However, this causes exceptions on Windows when other threads attempt to read a file
676 // which is still being copied. So instead, go back to moving the file and swallow any IOException.
677 //
678 // This situation occurs fairly rarely anyway. We assume in this that moves are atomic on the
679 // filesystem.
680 File.Move(tempname, filename);
681  
682 if (m_LogLevel >= 2)
683 m_log.DebugFormat("[FLOTSAM ASSET CACHE]: Cache Stored :: {0}", asset.ID);
684 }
685 catch (IOException)
686 {
687 // If we see an IOException here it's likely that some other competing thread has written the
688 // cache file first, so ignore. Other IOException errors (e.g. filesystem full) should be
689 // signally by the earlier temporary file writing code.
690 }
691 }
692 finally
693 {
694 // Even if the write fails with an exception, we need to make sure
695 // that we release the lock on that file, otherwise it'll never get
696 // cached
697 lock (m_CurrentlyWriting)
698 {
699 #if WAIT_ON_INPROGRESS_REQUESTS
700 ManualResetEvent waitEvent;
701 if (m_CurrentlyWriting.TryGetValue(filename, out waitEvent))
702 {
703 m_CurrentlyWriting.Remove(filename);
704 waitEvent.Set();
705 }
706 #else
707 m_CurrentlyWriting.Remove(filename);
708 #endif
709 }
710 }
711 }
712  
713 /// <summary>
714 /// Scan through the file cache, and return number of assets currently cached.
715 /// </summary>
716 /// <param name="dir"></param>
717 /// <returns></returns>
718 private int GetFileCacheCount(string dir)
719 {
720 int count = Directory.GetFiles(dir).Length;
721  
722 foreach (string subdir in Directory.GetDirectories(dir))
723 {
724 count += GetFileCacheCount(subdir);
725 }
726  
727 return count;
728 }
729  
730 /// <summary>
731 /// This notes the last time the Region had a deep asset scan performed on it.
732 /// </summary>
733 /// <param name="regionID"></param>
734 private void StampRegionStatusFile(UUID regionID)
735 {
736 string RegionCacheStatusFile = Path.Combine(m_CacheDirectory, "RegionStatus_" + regionID.ToString() + ".fac");
737  
738 try
739 {
740 if (File.Exists(RegionCacheStatusFile))
741 {
742 File.SetLastWriteTime(RegionCacheStatusFile, DateTime.Now);
743 }
744 else
745 {
746 File.WriteAllText(
747 RegionCacheStatusFile,
748 "Please do not delete this file unless you are manually clearing your Flotsam Asset Cache.");
749 }
750 }
751 catch (Exception e)
752 {
753 m_log.Warn(
754 string.Format(
755 "[FLOTSAM ASSET CACHE]: Could not stamp region status file for region {0}. Exception ",
756 regionID),
757 e);
758 }
759 }
760  
761 /// <summary>
762 /// Iterates through all Scenes, doing a deep scan through assets
763 /// to update the access time of all assets present in the scene or referenced by assets
764 /// in the scene.
765 /// </summary>
766 /// <param name="storeUncached">
767 /// If true, then assets scanned which are not found in cache are added to the cache.
768 /// </param>
769 /// <returns>Number of distinct asset references found in the scene.</returns>
770 private int TouchAllSceneAssets(bool storeUncached)
771 {
772 UuidGatherer gatherer = new UuidGatherer(m_AssetService);
773  
774 HashSet<UUID> uniqueUuids = new HashSet<UUID>();
775 Dictionary<UUID, sbyte> assets = new Dictionary<UUID, sbyte>();
776  
777 foreach (Scene s in m_Scenes)
778 {
779 StampRegionStatusFile(s.RegionInfo.RegionID);
780  
781 s.ForEachSOG(delegate(SceneObjectGroup e)
782 {
783 gatherer.GatherAssetUuids(e, assets);
784  
785 foreach (UUID assetID in assets.Keys)
786 {
787 uniqueUuids.Add(assetID);
788  
789 string filename = GetFileName(assetID.ToString());
790  
791 if (File.Exists(filename))
792 {
793 File.SetLastAccessTime(filename, DateTime.Now);
794 }
795 else if (storeUncached)
796 {
797 AssetBase cachedAsset = m_AssetService.Get(assetID.ToString());
798 if (cachedAsset == null && assets[assetID] != (sbyte)AssetType.Unknown)
799 m_log.DebugFormat(
800 "[FLOTSAM ASSET CACHE]: Could not find asset {0}, type {1} referenced by object {2} at {3} in scene {4} when pre-caching all scene assets",
801 assetID, assets[assetID], e.Name, e.AbsolutePosition, s.Name);
802 }
803 }
804  
805 assets.Clear();
806 });
807 }
808  
809  
810 return uniqueUuids.Count;
811 }
812  
813 /// <summary>
814 /// Deletes all cache contents
815 /// </summary>
816 private void ClearFileCache()
817 {
818 foreach (string dir in Directory.GetDirectories(m_CacheDirectory))
819 {
820 try
821 {
822 Directory.Delete(dir, true);
823 }
824 catch (Exception e)
825 {
826 m_log.WarnFormat(
827 "[FLOTSAM ASSET CACHE]: Couldn't clear asset cache directory {0} from {1}. Exception {2} {3}",
828 dir, m_CacheDirectory, e.Message, e.StackTrace);
829 }
830 }
831  
832 foreach (string file in Directory.GetFiles(m_CacheDirectory))
833 {
834 try
835 {
836 File.Delete(file);
837 }
838 catch (Exception e)
839 {
840 m_log.WarnFormat(
841 "[FLOTSAM ASSET CACHE]: Couldn't clear asset cache file {0} from {1}. Exception {1} {2}",
842 file, m_CacheDirectory, e.Message, e.StackTrace);
843 }
844 }
845 }
846  
847 private List<string> GenerateCacheHitReport()
848 {
849 List<string> outputLines = new List<string>();
850  
851 double fileHitRate = (double)m_DiskHits / m_Requests * 100.0;
852 outputLines.Add(
853 string.Format("File Hit Rate: {0}% for {1} requests", fileHitRate.ToString("0.00"), m_Requests));
854  
855 if (m_MemoryCacheEnabled)
856 {
857 double memHitRate = (double)m_MemoryHits / m_Requests * 100.0;
858  
859 outputLines.Add(
860 string.Format("Memory Hit Rate: {0}% for {1} requests", memHitRate.ToString("0.00"), m_Requests));
861 }
862  
863 outputLines.Add(
864 string.Format(
865 "Unnecessary requests due to requests for assets that are currently downloading: {0}",
866 m_RequestsForInprogress));
867  
868 return outputLines;
869 }
870  
871 #region Console Commands
872 private void HandleConsoleCommand(string module, string[] cmdparams)
873 {
874 ICommandConsole con = MainConsole.Instance;
875  
876 if (cmdparams.Length >= 2)
877 {
878 string cmd = cmdparams[1];
879  
880 switch (cmd)
881 {
882 case "status":
883 if (m_MemoryCacheEnabled)
884 con.OutputFormat("Memory Cache: {0} assets", m_MemoryCache.Count);
885 else
886 con.OutputFormat("Memory cache disabled");
887  
888 if (m_FileCacheEnabled)
889 {
890 int fileCount = GetFileCacheCount(m_CacheDirectory);
891 con.OutputFormat("File Cache: {0} assets", fileCount);
892 }
893 else
894 {
895 con.Output("File cache disabled");
896 }
897  
898 GenerateCacheHitReport().ForEach(l => con.Output(l));
899  
900 if (m_FileCacheEnabled)
901 {
902 con.Output("Deep scans have previously been performed on the following regions:");
903  
904 foreach (string s in Directory.GetFiles(m_CacheDirectory, "*.fac"))
905 {
906 string RegionID = s.Remove(0,s.IndexOf("_")).Replace(".fac","");
907 DateTime RegionDeepScanTMStamp = File.GetLastWriteTime(s);
908 con.OutputFormat("Region: {0}, {1}", RegionID, RegionDeepScanTMStamp.ToString("MM/dd/yyyy hh:mm:ss"));
909 }
910 }
911  
912 break;
913  
914 case "clear":
915 if (cmdparams.Length < 2)
916 {
917 con.Output("Usage is fcache clear [file] [memory]");
918 break;
919 }
920  
921 bool clearMemory = false, clearFile = false;
922  
923 if (cmdparams.Length == 2)
924 {
925 clearMemory = true;
926 clearFile = true;
927 }
928 foreach (string s in cmdparams)
929 {
930 if (s.ToLower() == "memory")
931 clearMemory = true;
932 else if (s.ToLower() == "file")
933 clearFile = true;
934 }
935  
936 if (clearMemory)
937 {
938 if (m_MemoryCacheEnabled)
939 {
940 m_MemoryCache.Clear();
941 con.Output("Memory cache cleared.");
942 }
943 else
944 {
945 con.Output("Memory cache not enabled.");
946 }
947 }
948  
949 if (clearFile)
950 {
951 if (m_FileCacheEnabled)
952 {
953 ClearFileCache();
954 con.Output("File cache cleared.");
955 }
956 else
957 {
958 con.Output("File cache not enabled.");
959 }
960 }
961  
962 break;
963  
964 case "assets":
965 con.Output("Ensuring assets are cached for all scenes.");
966  
967 Watchdog.RunInThread(delegate
968 {
969 int assetReferenceTotal = TouchAllSceneAssets(true);
970 con.OutputFormat("Completed check with {0} assets.", assetReferenceTotal);
971 }, "TouchAllSceneAssets", null);
972  
973 break;
974  
975 case "expire":
976 if (cmdparams.Length < 3)
977 {
978 con.OutputFormat("Invalid parameters for Expire, please specify a valid date & time", cmd);
979 break;
980 }
981  
982 string s_expirationDate = "";
983 DateTime expirationDate;
984  
985 if (cmdparams.Length > 3)
986 {
987 s_expirationDate = string.Join(" ", cmdparams, 2, cmdparams.Length - 2);
988 }
989 else
990 {
991 s_expirationDate = cmdparams[2];
992 }
993  
994 if (!DateTime.TryParse(s_expirationDate, out expirationDate))
995 {
996 con.OutputFormat("{0} is not a valid date & time", cmd);
997 break;
998 }
999  
1000 if (m_FileCacheEnabled)
1001 CleanExpiredFiles(m_CacheDirectory, expirationDate);
1002 else
1003 con.OutputFormat("File cache not active, not clearing.");
1004  
1005 break;
1006 default:
1007 con.OutputFormat("Unknown command {0}", cmd);
1008 break;
1009 }
1010 }
1011 else if (cmdparams.Length == 1)
1012 {
1013 con.Output("fcache assets - Attempt a deep cache of all assets in all scenes");
1014 con.Output("fcache expire <datetime> - Purge assets older then the specified date & time");
1015 con.Output("fcache clear [file] [memory] - Remove cached assets");
1016 con.Output("fcache status - Display cache status");
1017 }
1018 }
1019  
1020 #endregion
1021  
1022 #region IAssetService Members
1023  
1024 public AssetMetadata GetMetadata(string id)
1025 {
1026 AssetBase asset = Get(id);
1027 return asset.Metadata;
1028 }
1029  
1030 public byte[] GetData(string id)
1031 {
1032 AssetBase asset = Get(id);
1033 return asset.Data;
1034 }
1035  
1036 public bool Get(string id, object sender, AssetRetrieved handler)
1037 {
1038 AssetBase asset = Get(id);
1039 handler(id, sender, asset);
1040 return true;
1041 }
1042  
1043 public bool[] AssetsExist(string[] ids)
1044 {
1045 bool[] exist = new bool[ids.Length];
1046  
1047 for (int i = 0; i < ids.Length; i++)
1048 {
1049 exist[i] = Check(ids[i]);
1050 }
1051  
1052 return exist;
1053 }
1054  
1055 public string Store(AssetBase asset)
1056 {
1057 if (asset.FullID == UUID.Zero)
1058 {
1059 asset.FullID = UUID.Random();
1060 }
1061  
1062 Cache(asset);
1063  
1064 return asset.ID;
1065 }
1066  
1067 public bool UpdateContent(string id, byte[] data)
1068 {
1069 AssetBase asset = Get(id);
1070 asset.Data = data;
1071 Cache(asset);
1072 return true;
1073 }
1074  
1075 public bool Delete(string id)
1076 {
1077 Expire(id);
1078 return true;
1079 }
1080  
1081 #endregion
1082 }
1083 }