opensim – Blame information for rev 1

Subversion Repositories:
Rev:
Rev Author Line No. Line
1 eva 1 /*
2 * Copyright (c) Contributors, http://opensimulator.org/
3 * See CONTRIBUTORS.TXT for a full list of copyright holders.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 * * Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * * Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 * * Neither the name of the OpenSimulator Project nor the
13 * names of its contributors may be used to endorse or promote products
14 * derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY
17 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY
20 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
23 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27  
28 // Uncomment to make asset Get requests for existing
29 // #define WAIT_ON_INPROGRESS_REQUESTS
30  
31 using System;
32 using System.IO;
33 using System.Collections.Generic;
34 using System.Linq;
35 using System.Reflection;
36 using System.Runtime.Serialization;
37 using System.Runtime.Serialization.Formatters.Binary;
38 using System.Threading;
39 using System.Timers;
40 using log4net;
41 using Nini.Config;
42 using Mono.Addins;
43 using OpenMetaverse;
44 using OpenSim.Framework;
45 using OpenSim.Framework.Console;
46 using OpenSim.Region.Framework.Interfaces;
47 using OpenSim.Region.Framework.Scenes;
48 using OpenSim.Services.Interfaces;
49  
50  
51 //[assembly: Addin("FlotsamAssetCache", "1.1")]
52 //[assembly: AddinDependency("OpenSim", "0.5")]
53  
54 namespace OpenSim.Region.CoreModules.Asset
55 {
56 [Extension(Path = "/OpenSim/RegionModules", NodeName = "RegionModule", Id = "FlotsamAssetCache")]
57 public class FlotsamAssetCache : ISharedRegionModule, IImprovedAssetCache, IAssetService
58 {
59 private static readonly ILog m_log =
60 LogManager.GetLogger(
61 MethodBase.GetCurrentMethod().DeclaringType);
62  
63 private bool m_Enabled;
64  
65 private const string m_ModuleName = "FlotsamAssetCache";
66 private const string m_DefaultCacheDirectory = "./assetcache";
67 private string m_CacheDirectory = m_DefaultCacheDirectory;
68  
69 private readonly List<char> m_InvalidChars = new List<char>();
70  
71 private int m_LogLevel = 0;
72 private ulong m_HitRateDisplay = 100; // How often to display hit statistics, given in requests
73  
74 private static ulong m_Requests;
75 private static ulong m_RequestsForInprogress;
76 private static ulong m_DiskHits;
77 private static ulong m_MemoryHits;
78  
79 #if WAIT_ON_INPROGRESS_REQUESTS
80 private Dictionary<string, ManualResetEvent> m_CurrentlyWriting = new Dictionary<string, ManualResetEvent>();
81 private int m_WaitOnInprogressTimeout = 3000;
82 #else
83 private HashSet<string> m_CurrentlyWriting = new HashSet<string>();
84 #endif
85  
86 private bool m_FileCacheEnabled = true;
87  
88 private ExpiringCache<string, AssetBase> m_MemoryCache;
89 private bool m_MemoryCacheEnabled = false;
90  
91 // Expiration is expressed in hours.
92 private const double m_DefaultMemoryExpiration = 2;
93 private const double m_DefaultFileExpiration = 48;
94 private TimeSpan m_MemoryExpiration = TimeSpan.FromHours(m_DefaultMemoryExpiration);
95 private TimeSpan m_FileExpiration = TimeSpan.FromHours(m_DefaultFileExpiration);
96 private TimeSpan m_FileExpirationCleanupTimer = TimeSpan.FromHours(0.166);
97  
98 private static int m_CacheDirectoryTiers = 1;
99 private static int m_CacheDirectoryTierLen = 3;
100 private static int m_CacheWarnAt = 30000;
101  
102 private System.Timers.Timer m_CacheCleanTimer;
103  
104 private IAssetService m_AssetService;
105 private List<Scene> m_Scenes = new List<Scene>();
106  
107 public FlotsamAssetCache()
108 {
109 m_InvalidChars.AddRange(Path.GetInvalidPathChars());
110 m_InvalidChars.AddRange(Path.GetInvalidFileNameChars());
111 }
112  
113 public Type ReplaceableInterface
114 {
115 get { return null; }
116 }
117  
118 public string Name
119 {
120 get { return m_ModuleName; }
121 }
122  
123 public void Initialise(IConfigSource source)
124 {
125 IConfig moduleConfig = source.Configs["Modules"];
126  
127 if (moduleConfig != null)
128 {
129 string name = moduleConfig.GetString("AssetCaching", String.Empty);
130  
131 if (name == Name)
132 {
133 m_MemoryCache = new ExpiringCache<string, AssetBase>();
134 m_Enabled = true;
135  
136 m_log.InfoFormat("[FLOTSAM ASSET CACHE]: {0} enabled", this.Name);
137  
138 IConfig assetConfig = source.Configs["AssetCache"];
139 if (assetConfig == null)
140 {
141 m_log.Debug(
142 "[FLOTSAM ASSET CACHE]: AssetCache section missing from config (not copied config-include/FlotsamCache.ini.example? Using defaults.");
143 }
144 else
145 {
146 m_FileCacheEnabled = assetConfig.GetBoolean("FileCacheEnabled", m_FileCacheEnabled);
147 m_CacheDirectory = assetConfig.GetString("CacheDirectory", m_DefaultCacheDirectory);
148  
149 m_MemoryCacheEnabled = assetConfig.GetBoolean("MemoryCacheEnabled", m_MemoryCacheEnabled);
150 m_MemoryExpiration = TimeSpan.FromHours(assetConfig.GetDouble("MemoryCacheTimeout", m_DefaultMemoryExpiration));
151  
152 #if WAIT_ON_INPROGRESS_REQUESTS
153 m_WaitOnInprogressTimeout = assetConfig.GetInt("WaitOnInprogressTimeout", 3000);
154 #endif
155  
156 m_LogLevel = assetConfig.GetInt("LogLevel", m_LogLevel);
157 m_HitRateDisplay = (ulong)assetConfig.GetLong("HitRateDisplay", (long)m_HitRateDisplay);
158  
159 m_FileExpiration = TimeSpan.FromHours(assetConfig.GetDouble("FileCacheTimeout", m_DefaultFileExpiration));
160 m_FileExpirationCleanupTimer
161 = TimeSpan.FromHours(
162 assetConfig.GetDouble("FileCleanupTimer", m_FileExpirationCleanupTimer.TotalHours));
163  
164 m_CacheDirectoryTiers = assetConfig.GetInt("CacheDirectoryTiers", m_CacheDirectoryTiers);
165 m_CacheDirectoryTierLen = assetConfig.GetInt("CacheDirectoryTierLength", m_CacheDirectoryTierLen);
166  
167 m_CacheWarnAt = assetConfig.GetInt("CacheWarnAt", m_CacheWarnAt);
168 }
169  
170 m_log.InfoFormat("[FLOTSAM ASSET CACHE]: Cache Directory {0}", m_CacheDirectory);
171  
172 if (m_FileCacheEnabled && (m_FileExpiration > TimeSpan.Zero) && (m_FileExpirationCleanupTimer > TimeSpan.Zero))
173 {
174 m_CacheCleanTimer = new System.Timers.Timer(m_FileExpirationCleanupTimer.TotalMilliseconds);
175 m_CacheCleanTimer.AutoReset = true;
176 m_CacheCleanTimer.Elapsed += CleanupExpiredFiles;
177 lock (m_CacheCleanTimer)
178 m_CacheCleanTimer.Start();
179 }
180  
181 if (m_CacheDirectoryTiers < 1)
182 {
183 m_CacheDirectoryTiers = 1;
184 }
185 else if (m_CacheDirectoryTiers > 3)
186 {
187 m_CacheDirectoryTiers = 3;
188 }
189  
190 if (m_CacheDirectoryTierLen < 1)
191 {
192 m_CacheDirectoryTierLen = 1;
193 }
194 else if (m_CacheDirectoryTierLen > 4)
195 {
196 m_CacheDirectoryTierLen = 4;
197 }
198  
199 MainConsole.Instance.Commands.AddCommand("Assets", true, "fcache status", "fcache status", "Display cache status", HandleConsoleCommand);
200 MainConsole.Instance.Commands.AddCommand("Assets", true, "fcache clear", "fcache clear [file] [memory]", "Remove all assets in the cache. If file or memory is specified then only this cache is cleared.", HandleConsoleCommand);
201 MainConsole.Instance.Commands.AddCommand("Assets", true, "fcache assets", "fcache assets", "Attempt a deep scan and cache of all assets in all scenes", HandleConsoleCommand);
202 MainConsole.Instance.Commands.AddCommand("Assets", true, "fcache expire", "fcache expire <datetime>", "Purge cached assets older then the specified date/time", HandleConsoleCommand);
203 }
204 }
205 }
206  
207 public void PostInitialise()
208 {
209 }
210  
211 public void Close()
212 {
213 }
214  
215 public void AddRegion(Scene scene)
216 {
217 if (m_Enabled)
218 {
219 scene.RegisterModuleInterface<IImprovedAssetCache>(this);
220 m_Scenes.Add(scene);
221  
222 }
223 }
224  
225 public void RemoveRegion(Scene scene)
226 {
227 if (m_Enabled)
228 {
229 scene.UnregisterModuleInterface<IImprovedAssetCache>(this);
230 m_Scenes.Remove(scene);
231 }
232 }
233  
234 public void RegionLoaded(Scene scene)
235 {
236 if (m_Enabled && m_AssetService == null)
237 m_AssetService = scene.RequestModuleInterface<IAssetService>();
238 }
239  
240 ////////////////////////////////////////////////////////////
241 // IImprovedAssetCache
242 //
243  
244 private void UpdateMemoryCache(string key, AssetBase asset)
245 {
246 m_MemoryCache.AddOrUpdate(key, asset, m_MemoryExpiration);
247 }
248  
249 private void UpdateFileCache(string key, AssetBase asset)
250 {
251 string filename = GetFileName(key);
252  
253 try
254 {
255 // If the file is already cached, don't cache it, just touch it so access time is updated
256 if (File.Exists(filename))
257 {
258 // We don't really want to know about sharing
259 // violations here. If the file is locked, then
260 // the other thread has updated the time for us.
261 try
262 {
263 lock (m_CurrentlyWriting)
264 {
265 if (!m_CurrentlyWriting.Contains(filename))
266 File.SetLastAccessTime(filename, DateTime.Now);
267 }
268 }
269 catch
270 {
271 }
272 }
273 else
274 {
275 // Once we start writing, make sure we flag that we're writing
276 // that object to the cache so that we don't try to write the
277 // same file multiple times.
278 lock (m_CurrentlyWriting)
279 {
280 #if WAIT_ON_INPROGRESS_REQUESTS
281 if (m_CurrentlyWriting.ContainsKey(filename))
282 {
283 return;
284 }
285 else
286 {
287 m_CurrentlyWriting.Add(filename, new ManualResetEvent(false));
288 }
289  
290 #else
291 if (m_CurrentlyWriting.Contains(filename))
292 {
293 return;
294 }
295 else
296 {
297 m_CurrentlyWriting.Add(filename);
298 }
299 #endif
300  
301 }
302  
303 Util.FireAndForget(
304 delegate { WriteFileCache(filename, asset); });
305 }
306 }
307 catch (Exception e)
308 {
309 m_log.ErrorFormat(
310 "[FLOTSAM ASSET CACHE]: Failed to update cache for asset {0}. Exception {1} {2}",
311 asset.ID, e.Message, e.StackTrace);
312 }
313 }
314  
315 public void Cache(AssetBase asset)
316 {
317 // TODO: Spawn this off to some seperate thread to do the actual writing
318 if (asset != null)
319 {
320 //m_log.DebugFormat("[FLOTSAM ASSET CACHE]: Caching asset with id {0}", asset.ID);
321  
322 if (m_MemoryCacheEnabled)
323 UpdateMemoryCache(asset.ID, asset);
324  
325 if (m_FileCacheEnabled)
326 UpdateFileCache(asset.ID, asset);
327 }
328 }
329  
330 /// <summary>
331 /// Try to get an asset from the in-memory cache.
332 /// </summary>
333 /// <param name="id"></param>
334 /// <returns></returns>
335 private AssetBase GetFromMemoryCache(string id)
336 {
337 AssetBase asset = null;
338  
339 if (m_MemoryCache.TryGetValue(id, out asset))
340 m_MemoryHits++;
341  
342 return asset;
343 }
344  
345 private bool CheckFromMemoryCache(string id)
346 {
347 return m_MemoryCache.Contains(id);
348 }
349  
350 /// <summary>
351 /// Try to get an asset from the file cache.
352 /// </summary>
353 /// <param name="id"></param>
354 /// <returns>An asset retrieved from the file cache. null if there was a problem retrieving an asset.</returns>
355 private AssetBase GetFromFileCache(string id)
356 {
357 string filename = GetFileName(id);
358  
359 #if WAIT_ON_INPROGRESS_REQUESTS
360 // Check if we're already downloading this asset. If so, try to wait for it to
361 // download.
362 if (m_WaitOnInprogressTimeout > 0)
363 {
364 m_RequestsForInprogress++;
365  
366 ManualResetEvent waitEvent;
367 if (m_CurrentlyWriting.TryGetValue(filename, out waitEvent))
368 {
369 waitEvent.WaitOne(m_WaitOnInprogressTimeout);
370 return Get(id);
371 }
372 }
373 #else
374 // Track how often we have the problem that an asset is requested while
375 // it is still being downloaded by a previous request.
376 if (m_CurrentlyWriting.Contains(filename))
377 {
378 m_RequestsForInprogress++;
379 return null;
380 }
381 #endif
382  
383 AssetBase asset = null;
384  
385 if (File.Exists(filename))
386 {
387 try
388 {
389 using (FileStream stream = File.Open(filename, FileMode.Open, FileAccess.Read, FileShare.Read))
390 {
391 BinaryFormatter bformatter = new BinaryFormatter();
392  
393 asset = (AssetBase)bformatter.Deserialize(stream);
394  
395 m_DiskHits++;
396 }
397 }
398 catch (System.Runtime.Serialization.SerializationException e)
399 {
400 m_log.WarnFormat(
401 "[FLOTSAM ASSET CACHE]: Failed to get file {0} for asset {1}. Exception {2} {3}",
402 filename, id, e.Message, e.StackTrace);
403  
404 // If there was a problem deserializing the asset, the asset may
405 // either be corrupted OR was serialized under an old format
406 // {different version of AssetBase} -- we should attempt to
407 // delete it and re-cache
408 File.Delete(filename);
409 }
410 catch (Exception e)
411 {
412 m_log.WarnFormat(
413 "[FLOTSAM ASSET CACHE]: Failed to get file {0} for asset {1}. Exception {2} {3}",
414 filename, id, e.Message, e.StackTrace);
415 }
416 }
417  
418 return asset;
419 }
420  
421 private bool CheckFromFileCache(string id)
422 {
423 bool found = false;
424  
425 string filename = GetFileName(id);
426  
427 if (File.Exists(filename))
428 {
429 try
430 {
431 using (FileStream stream = File.Open(filename, FileMode.Open, FileAccess.Read, FileShare.Read))
432 {
433 if (stream != null)
434 found = true;
435 }
436 }
437 catch (Exception e)
438 {
439 m_log.ErrorFormat(
440 "[FLOTSAM ASSET CACHE]: Failed to check file {0} for asset {1}. Exception {2} {3}",
441 filename, id, e.Message, e.StackTrace);
442 }
443 }
444  
445 return found;
446 }
447  
448 public AssetBase Get(string id)
449 {
450 m_Requests++;
451  
452 AssetBase asset = null;
453  
454 if (m_MemoryCacheEnabled)
455 asset = GetFromMemoryCache(id);
456  
457 if (asset == null && m_FileCacheEnabled)
458 {
459 asset = GetFromFileCache(id);
460  
461 if (m_MemoryCacheEnabled && asset != null)
462 UpdateMemoryCache(id, asset);
463 }
464  
465 if (((m_LogLevel >= 1)) && (m_HitRateDisplay != 0) && (m_Requests % m_HitRateDisplay == 0))
466 {
467 m_log.InfoFormat("[FLOTSAM ASSET CACHE]: Cache Get :: {0} :: {1}", id, asset == null ? "Miss" : "Hit");
468  
469 GenerateCacheHitReport().ForEach(l => m_log.InfoFormat("[FLOTSAM ASSET CACHE]: {0}", l));
470 }
471  
472 return asset;
473 }
474  
475 public bool Check(string id)
476 {
477 if (m_MemoryCacheEnabled && CheckFromMemoryCache(id))
478 return true;
479  
480 if (m_FileCacheEnabled && CheckFromFileCache(id))
481 return true;
482 return false;
483 }
484  
485 public AssetBase GetCached(string id)
486 {
487 return Get(id);
488 }
489  
490 public void Expire(string id)
491 {
492 if (m_LogLevel >= 2)
493 m_log.DebugFormat("[FLOTSAM ASSET CACHE]: Expiring Asset {0}", id);
494  
495 try
496 {
497 if (m_FileCacheEnabled)
498 {
499 string filename = GetFileName(id);
500 if (File.Exists(filename))
501 {
502 File.Delete(filename);
503 }
504 }
505  
506 if (m_MemoryCacheEnabled)
507 m_MemoryCache.Remove(id);
508 }
509 catch (Exception e)
510 {
511 m_log.WarnFormat(
512 "[FLOTSAM ASSET CACHE]: Failed to expire cached file {0}. Exception {1} {2}",
513 id, e.Message, e.StackTrace);
514 }
515 }
516  
517 public void Clear()
518 {
519 if (m_LogLevel >= 2)
520 m_log.Debug("[FLOTSAM ASSET CACHE]: Clearing caches.");
521  
522 if (m_FileCacheEnabled)
523 {
524 foreach (string dir in Directory.GetDirectories(m_CacheDirectory))
525 {
526 Directory.Delete(dir);
527 }
528 }
529  
530 if (m_MemoryCacheEnabled)
531 m_MemoryCache.Clear();
532 }
533  
534 private void CleanupExpiredFiles(object source, ElapsedEventArgs e)
535 {
536 if (m_LogLevel >= 2)
537 m_log.DebugFormat("[FLOTSAM ASSET CACHE]: Checking for expired files older then {0}.", m_FileExpiration);
538  
539 // Purge all files last accessed prior to this point
540 DateTime purgeLine = DateTime.Now - m_FileExpiration;
541  
542 // An asset cache may contain local non-temporary assets that are not in the asset service. Therefore,
543 // before cleaning up expired files we must scan the objects in the scene to make sure that we retain
544 // such local assets if they have not been recently accessed.
545 TouchAllSceneAssets(false);
546  
547 foreach (string dir in Directory.GetDirectories(m_CacheDirectory))
548 {
549 CleanExpiredFiles(dir, purgeLine);
550 }
551 }
552  
553 /// <summary>
554 /// Recurses through specified directory checking for asset files last
555 /// accessed prior to the specified purge line and deletes them. Also
556 /// removes empty tier directories.
557 /// </summary>
558 /// <param name="dir"></param>
559 /// <param name="purgeLine"></param>
560 private void CleanExpiredFiles(string dir, DateTime purgeLine)
561 {
562 try
563 {
564 foreach (string file in Directory.GetFiles(dir))
565 {
566 if (File.GetLastAccessTime(file) < purgeLine)
567 {
568 File.Delete(file);
569 }
570 }
571  
572 // Recurse into lower tiers
573 foreach (string subdir in Directory.GetDirectories(dir))
574 {
575 CleanExpiredFiles(subdir, purgeLine);
576 }
577  
578 // Check if a tier directory is empty, if so, delete it
579 int dirSize = Directory.GetFiles(dir).Length + Directory.GetDirectories(dir).Length;
580 if (dirSize == 0)
581 {
582 Directory.Delete(dir);
583 }
584 else if (dirSize >= m_CacheWarnAt)
585 {
586 m_log.WarnFormat(
587 "[FLOTSAM ASSET CACHE]: Cache folder exceeded CacheWarnAt limit {0} {1}. Suggest increasing tiers, tier length, or reducing cache expiration",
588 dir, dirSize);
589 }
590 }
591 catch (Exception e)
592 {
593 m_log.Warn(
594 string.Format("[FLOTSAM ASSET CACHE]: Could not complete clean of expired files in {0}, exception ", dir), e);
595 }
596 }
597  
598 /// <summary>
599 /// Determines the filename for an AssetID stored in the file cache
600 /// </summary>
601 /// <param name="id"></param>
602 /// <returns></returns>
603 private string GetFileName(string id)
604 {
605 // Would it be faster to just hash the darn thing?
606 foreach (char c in m_InvalidChars)
607 {
608 id = id.Replace(c, '_');
609 }
610  
611 string path = m_CacheDirectory;
612 for (int p = 1; p <= m_CacheDirectoryTiers; p++)
613 {
614 string pathPart = id.Substring((p - 1) * m_CacheDirectoryTierLen, m_CacheDirectoryTierLen);
615 path = Path.Combine(path, pathPart);
616 }
617  
618 return Path.Combine(path, id);
619 }
620  
621 /// <summary>
622 /// Writes a file to the file cache, creating any nessesary
623 /// tier directories along the way
624 /// </summary>
625 /// <param name="filename"></param>
626 /// <param name="asset"></param>
627 private void WriteFileCache(string filename, AssetBase asset)
628 {
629 Stream stream = null;
630  
631 // Make sure the target cache directory exists
632 string directory = Path.GetDirectoryName(filename);
633  
634 // Write file first to a temp name, so that it doesn't look
635 // like it's already cached while it's still writing.
636 string tempname = Path.Combine(directory, Path.GetRandomFileName());
637  
638 try
639 {
640 try
641 {
642 if (!Directory.Exists(directory))
643 {
644 Directory.CreateDirectory(directory);
645 }
646  
647 stream = File.Open(tempname, FileMode.Create);
648 BinaryFormatter bformatter = new BinaryFormatter();
649 bformatter.Serialize(stream, asset);
650 }
651 catch (IOException e)
652 {
653 m_log.WarnFormat(
654 "[FLOTSAM ASSET CACHE]: Failed to write asset {0} to temporary location {1} (final {2}) on cache in {3}. Exception {4} {5}.",
655 asset.ID, tempname, filename, directory, e.Message, e.StackTrace);
656  
657 return;
658 }
659 finally
660 {
661 if (stream != null)
662 stream.Close();
663 }
664  
665 try
666 {
667 // Now that it's written, rename it so that it can be found.
668 //
669 // File.Copy(tempname, filename, true);
670 // File.Delete(tempname);
671 //
672 // For a brief period, this was done as a separate copy and then temporary file delete operation to
673 // avoid an IOException caused by move if some competing thread had already written the file.
674 // However, this causes exceptions on Windows when other threads attempt to read a file
675 // which is still being copied. So instead, go back to moving the file and swallow any IOException.
676 //
677 // This situation occurs fairly rarely anyway. We assume in this that moves are atomic on the
678 // filesystem.
679 File.Move(tempname, filename);
680  
681 if (m_LogLevel >= 2)
682 m_log.DebugFormat("[FLOTSAM ASSET CACHE]: Cache Stored :: {0}", asset.ID);
683 }
684 catch (IOException)
685 {
686 // If we see an IOException here it's likely that some other competing thread has written the
687 // cache file first, so ignore. Other IOException errors (e.g. filesystem full) should be
688 // signally by the earlier temporary file writing code.
689 }
690 }
691 finally
692 {
693 // Even if the write fails with an exception, we need to make sure
694 // that we release the lock on that file, otherwise it'll never get
695 // cached
696 lock (m_CurrentlyWriting)
697 {
698 #if WAIT_ON_INPROGRESS_REQUESTS
699 ManualResetEvent waitEvent;
700 if (m_CurrentlyWriting.TryGetValue(filename, out waitEvent))
701 {
702 m_CurrentlyWriting.Remove(filename);
703 waitEvent.Set();
704 }
705 #else
706 m_CurrentlyWriting.Remove(filename);
707 #endif
708 }
709 }
710 }
711  
712 /// <summary>
713 /// Scan through the file cache, and return number of assets currently cached.
714 /// </summary>
715 /// <param name="dir"></param>
716 /// <returns></returns>
717 private int GetFileCacheCount(string dir)
718 {
719 int count = Directory.GetFiles(dir).Length;
720  
721 foreach (string subdir in Directory.GetDirectories(dir))
722 {
723 count += GetFileCacheCount(subdir);
724 }
725  
726 return count;
727 }
728  
729 /// <summary>
730 /// This notes the last time the Region had a deep asset scan performed on it.
731 /// </summary>
732 /// <param name="regionID"></param>
733 private void StampRegionStatusFile(UUID regionID)
734 {
735 string RegionCacheStatusFile = Path.Combine(m_CacheDirectory, "RegionStatus_" + regionID.ToString() + ".fac");
736  
737 try
738 {
739 if (File.Exists(RegionCacheStatusFile))
740 {
741 File.SetLastWriteTime(RegionCacheStatusFile, DateTime.Now);
742 }
743 else
744 {
745 File.WriteAllText(
746 RegionCacheStatusFile,
747 "Please do not delete this file unless you are manually clearing your Flotsam Asset Cache.");
748 }
749 }
750 catch (Exception e)
751 {
752 m_log.Warn(
753 string.Format(
754 "[FLOTSAM ASSET CACHE]: Could not stamp region status file for region {0}. Exception ",
755 regionID),
756 e);
757 }
758 }
759  
760 /// <summary>
761 /// Iterates through all Scenes, doing a deep scan through assets
762 /// to update the access time of all assets present in the scene or referenced by assets
763 /// in the scene.
764 /// </summary>
765 /// <param name="storeUncached">
766 /// If true, then assets scanned which are not found in cache are added to the cache.
767 /// </param>
768 /// <returns>Number of distinct asset references found in the scene.</returns>
769 private int TouchAllSceneAssets(bool storeUncached)
770 {
771 UuidGatherer gatherer = new UuidGatherer(m_AssetService);
772  
773 HashSet<UUID> uniqueUuids = new HashSet<UUID>();
774 Dictionary<UUID, sbyte> assets = new Dictionary<UUID, sbyte>();
775  
776 foreach (Scene s in m_Scenes)
777 {
778 StampRegionStatusFile(s.RegionInfo.RegionID);
779  
780 s.ForEachSOG(delegate(SceneObjectGroup e)
781 {
782 gatherer.GatherAssetUuids(e, assets);
783  
784 foreach (UUID assetID in assets.Keys)
785 {
786 uniqueUuids.Add(assetID);
787  
788 string filename = GetFileName(assetID.ToString());
789  
790 if (File.Exists(filename))
791 {
792 File.SetLastAccessTime(filename, DateTime.Now);
793 }
794 else if (storeUncached)
795 {
796 AssetBase cachedAsset = m_AssetService.Get(assetID.ToString());
797 if (cachedAsset == null && assets[assetID] != (sbyte)AssetType.Unknown)
798 m_log.DebugFormat(
799 "[FLOTSAM ASSET CACHE]: Could not find asset {0}, type {1} referenced by object {2} at {3} in scene {4} when pre-caching all scene assets",
800 assetID, assets[assetID], e.Name, e.AbsolutePosition, s.Name);
801 }
802 }
803  
804 assets.Clear();
805 });
806 }
807  
808  
809 return uniqueUuids.Count;
810 }
811  
812 /// <summary>
813 /// Deletes all cache contents
814 /// </summary>
815 private void ClearFileCache()
816 {
817 foreach (string dir in Directory.GetDirectories(m_CacheDirectory))
818 {
819 try
820 {
821 Directory.Delete(dir, true);
822 }
823 catch (Exception e)
824 {
825 m_log.WarnFormat(
826 "[FLOTSAM ASSET CACHE]: Couldn't clear asset cache directory {0} from {1}. Exception {2} {3}",
827 dir, m_CacheDirectory, e.Message, e.StackTrace);
828 }
829 }
830  
831 foreach (string file in Directory.GetFiles(m_CacheDirectory))
832 {
833 try
834 {
835 File.Delete(file);
836 }
837 catch (Exception e)
838 {
839 m_log.WarnFormat(
840 "[FLOTSAM ASSET CACHE]: Couldn't clear asset cache file {0} from {1}. Exception {1} {2}",
841 file, m_CacheDirectory, e.Message, e.StackTrace);
842 }
843 }
844 }
845  
846 private List<string> GenerateCacheHitReport()
847 {
848 List<string> outputLines = new List<string>();
849  
850 double fileHitRate = (double)m_DiskHits / m_Requests * 100.0;
851 outputLines.Add(
852 string.Format("File Hit Rate: {0}% for {1} requests", fileHitRate.ToString("0.00"), m_Requests));
853  
854 if (m_MemoryCacheEnabled)
855 {
856 double memHitRate = (double)m_MemoryHits / m_Requests * 100.0;
857  
858 outputLines.Add(
859 string.Format("Memory Hit Rate: {0}% for {1} requests", memHitRate.ToString("0.00"), m_Requests));
860 }
861  
862 outputLines.Add(
863 string.Format(
864 "Unnecessary requests due to requests for assets that are currently downloading: {0}",
865 m_RequestsForInprogress));
866  
867 return outputLines;
868 }
869  
870 #region Console Commands
871 private void HandleConsoleCommand(string module, string[] cmdparams)
872 {
873 ICommandConsole con = MainConsole.Instance;
874  
875 if (cmdparams.Length >= 2)
876 {
877 string cmd = cmdparams[1];
878  
879 switch (cmd)
880 {
881 case "status":
882 if (m_MemoryCacheEnabled)
883 con.OutputFormat("Memory Cache: {0} assets", m_MemoryCache.Count);
884 else
885 con.OutputFormat("Memory cache disabled");
886  
887 if (m_FileCacheEnabled)
888 {
889 int fileCount = GetFileCacheCount(m_CacheDirectory);
890 con.OutputFormat("File Cache: {0} assets", fileCount);
891 }
892 else
893 {
894 con.Output("File cache disabled");
895 }
896  
897 GenerateCacheHitReport().ForEach(l => con.Output(l));
898  
899 if (m_FileCacheEnabled)
900 {
901 con.Output("Deep scans have previously been performed on the following regions:");
902  
903 foreach (string s in Directory.GetFiles(m_CacheDirectory, "*.fac"))
904 {
905 string RegionID = s.Remove(0,s.IndexOf("_")).Replace(".fac","");
906 DateTime RegionDeepScanTMStamp = File.GetLastWriteTime(s);
907 con.OutputFormat("Region: {0}, {1}", RegionID, RegionDeepScanTMStamp.ToString("MM/dd/yyyy hh:mm:ss"));
908 }
909 }
910  
911 break;
912  
913 case "clear":
914 if (cmdparams.Length < 2)
915 {
916 con.Output("Usage is fcache clear [file] [memory]");
917 break;
918 }
919  
920 bool clearMemory = false, clearFile = false;
921  
922 if (cmdparams.Length == 2)
923 {
924 clearMemory = true;
925 clearFile = true;
926 }
927 foreach (string s in cmdparams)
928 {
929 if (s.ToLower() == "memory")
930 clearMemory = true;
931 else if (s.ToLower() == "file")
932 clearFile = true;
933 }
934  
935 if (clearMemory)
936 {
937 if (m_MemoryCacheEnabled)
938 {
939 m_MemoryCache.Clear();
940 con.Output("Memory cache cleared.");
941 }
942 else
943 {
944 con.Output("Memory cache not enabled.");
945 }
946 }
947  
948 if (clearFile)
949 {
950 if (m_FileCacheEnabled)
951 {
952 ClearFileCache();
953 con.Output("File cache cleared.");
954 }
955 else
956 {
957 con.Output("File cache not enabled.");
958 }
959 }
960  
961 break;
962  
963 case "assets":
964 con.Output("Ensuring assets are cached for all scenes.");
965  
966 Util.FireAndForget(delegate {
967 int assetReferenceTotal = TouchAllSceneAssets(true);
968 con.OutputFormat("Completed check with {0} assets.", assetReferenceTotal);
969 });
970  
971 break;
972  
973 case "expire":
974 if (cmdparams.Length < 3)
975 {
976 con.OutputFormat("Invalid parameters for Expire, please specify a valid date & time", cmd);
977 break;
978 }
979  
980 string s_expirationDate = "";
981 DateTime expirationDate;
982  
983 if (cmdparams.Length > 3)
984 {
985 s_expirationDate = string.Join(" ", cmdparams, 2, cmdparams.Length - 2);
986 }
987 else
988 {
989 s_expirationDate = cmdparams[2];
990 }
991  
992 if (!DateTime.TryParse(s_expirationDate, out expirationDate))
993 {
994 con.OutputFormat("{0} is not a valid date & time", cmd);
995 break;
996 }
997  
998 if (m_FileCacheEnabled)
999 CleanExpiredFiles(m_CacheDirectory, expirationDate);
1000 else
1001 con.OutputFormat("File cache not active, not clearing.");
1002  
1003 break;
1004 default:
1005 con.OutputFormat("Unknown command {0}", cmd);
1006 break;
1007 }
1008 }
1009 else if (cmdparams.Length == 1)
1010 {
1011 con.Output("fcache assets - Attempt a deep cache of all assets in all scenes");
1012 con.Output("fcache expire <datetime> - Purge assets older then the specified date & time");
1013 con.Output("fcache clear [file] [memory] - Remove cached assets");
1014 con.Output("fcache status - Display cache status");
1015 }
1016 }
1017  
1018 #endregion
1019  
1020 #region IAssetService Members
1021  
1022 public AssetMetadata GetMetadata(string id)
1023 {
1024 AssetBase asset = Get(id);
1025 return asset.Metadata;
1026 }
1027  
1028 public byte[] GetData(string id)
1029 {
1030 AssetBase asset = Get(id);
1031 return asset.Data;
1032 }
1033  
1034 public bool Get(string id, object sender, AssetRetrieved handler)
1035 {
1036 AssetBase asset = Get(id);
1037 handler(id, sender, asset);
1038 return true;
1039 }
1040  
1041 public string Store(AssetBase asset)
1042 {
1043 if (asset.FullID == UUID.Zero)
1044 {
1045 asset.FullID = UUID.Random();
1046 }
1047  
1048 Cache(asset);
1049  
1050 return asset.ID;
1051 }
1052  
1053 public bool UpdateContent(string id, byte[] data)
1054 {
1055 AssetBase asset = Get(id);
1056 asset.Data = data;
1057 Cache(asset);
1058 return true;
1059 }
1060  
1061 public bool Delete(string id)
1062 {
1063 Expire(id);
1064 return true;
1065 }
1066  
1067 #endregion
1068 }
1069 }