clockwerk-opensim-stable – Blame information for rev 1
?pathlinks?
Rev | Author | Line No. | Line |
---|---|---|---|
1 | vero | 1 | /* |
2 | * Copyright (c) Contributors, http://opensimulator.org/ |
||
3 | * See CONTRIBUTORS.TXT for a full list of copyright holders. |
||
4 | * |
||
5 | * Redistribution and use in source and binary forms, with or without |
||
6 | * modification, are permitted provided that the following conditions are met: |
||
7 | * * Redistributions of source code must retain the above copyright |
||
8 | * notice, this list of conditions and the following disclaimer. |
||
9 | * * Redistributions in binary form must reproduce the above copyright |
||
10 | * notice, this list of conditions and the following disclaimer in the |
||
11 | * documentation and/or other materials provided with the distribution. |
||
12 | * * Neither the name of the OpenSimulator Project nor the |
||
13 | * names of its contributors may be used to endorse or promote products |
||
14 | * derived from this software without specific prior written permission. |
||
15 | * |
||
16 | * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY |
||
17 | * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
||
18 | * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE |
||
19 | * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY |
||
20 | * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES |
||
21 | * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
||
22 | * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND |
||
23 | * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
||
24 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS |
||
25 | * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
||
26 | */ |
||
27 | |||
28 | // Uncomment to make asset Get requests for existing |
||
29 | // #define WAIT_ON_INPROGRESS_REQUESTS |
||
30 | |||
31 | using System; |
||
32 | using System.IO; |
||
33 | using System.Collections.Generic; |
||
34 | using System.Reflection; |
||
35 | using System.Runtime.Serialization; |
||
36 | using System.Runtime.Serialization.Formatters.Binary; |
||
37 | using System.Threading; |
||
38 | using System.Timers; |
||
39 | |||
40 | using log4net; |
||
41 | using Nini.Config; |
||
42 | using Mono.Addins; |
||
43 | using OpenMetaverse; |
||
44 | |||
45 | using OpenSim.Framework; |
||
46 | using OpenSim.Framework.Console; |
||
47 | using OpenSim.Region.Framework.Interfaces; |
||
48 | using OpenSim.Region.Framework.Scenes; |
||
49 | using OpenSim.Services.Interfaces; |
||
50 | |||
51 | |||
52 | //[assembly: Addin("FlotsamAssetCache", "1.1")] |
||
53 | //[assembly: AddinDependency("OpenSim", "0.5")] |
||
54 | |||
55 | namespace OpenSim.Region.CoreModules.Asset |
||
56 | { |
||
57 | [Extension(Path = "/OpenSim/RegionModules", NodeName = "RegionModule", Id = "FlotsamAssetCache")] |
||
58 | public class FlotsamAssetCache : ISharedRegionModule, IImprovedAssetCache, IAssetService |
||
59 | { |
||
60 | private static readonly ILog m_log = |
||
61 | LogManager.GetLogger( |
||
62 | MethodBase.GetCurrentMethod().DeclaringType); |
||
63 | |||
64 | private bool m_Enabled; |
||
65 | |||
66 | private const string m_ModuleName = "FlotsamAssetCache"; |
||
67 | private const string m_DefaultCacheDirectory = "./assetcache"; |
||
68 | private string m_CacheDirectory = m_DefaultCacheDirectory; |
||
69 | |||
70 | private readonly List<char> m_InvalidChars = new List<char>(); |
||
71 | |||
72 | private int m_LogLevel = 0; |
||
73 | private ulong m_HitRateDisplay = 100; // How often to display hit statistics, given in requests |
||
74 | |||
75 | private static ulong m_Requests; |
||
76 | private static ulong m_RequestsForInprogress; |
||
77 | private static ulong m_DiskHits; |
||
78 | private static ulong m_MemoryHits; |
||
79 | private static double m_HitRateMemory; |
||
80 | private static double m_HitRateFile; |
||
81 | |||
82 | #if WAIT_ON_INPROGRESS_REQUESTS |
||
83 | private Dictionary<string, ManualResetEvent> m_CurrentlyWriting = new Dictionary<string, ManualResetEvent>(); |
||
84 | private int m_WaitOnInprogressTimeout = 3000; |
||
85 | #else |
||
86 | private HashSet<string> m_CurrentlyWriting = new HashSet<string>(); |
||
87 | #endif |
||
88 | |||
89 | private bool m_FileCacheEnabled = true; |
||
90 | |||
91 | private ExpiringCache<string, AssetBase> m_MemoryCache; |
||
92 | private bool m_MemoryCacheEnabled = false; |
||
93 | |||
94 | // Expiration is expressed in hours. |
||
95 | private const double m_DefaultMemoryExpiration = 2; |
||
96 | private const double m_DefaultFileExpiration = 48; |
||
97 | private TimeSpan m_MemoryExpiration = TimeSpan.FromHours(m_DefaultMemoryExpiration); |
||
98 | private TimeSpan m_FileExpiration = TimeSpan.FromHours(m_DefaultFileExpiration); |
||
99 | private TimeSpan m_FileExpirationCleanupTimer = TimeSpan.FromHours(0.166); |
||
100 | |||
101 | private static int m_CacheDirectoryTiers = 1; |
||
102 | private static int m_CacheDirectoryTierLen = 3; |
||
103 | private static int m_CacheWarnAt = 30000; |
||
104 | |||
105 | private System.Timers.Timer m_CacheCleanTimer; |
||
106 | |||
107 | private IAssetService m_AssetService; |
||
108 | private List<Scene> m_Scenes = new List<Scene>(); |
||
109 | |||
110 | public FlotsamAssetCache() |
||
111 | { |
||
112 | m_InvalidChars.AddRange(Path.GetInvalidPathChars()); |
||
113 | m_InvalidChars.AddRange(Path.GetInvalidFileNameChars()); |
||
114 | } |
||
115 | |||
116 | public Type ReplaceableInterface |
||
117 | { |
||
118 | get { return null; } |
||
119 | } |
||
120 | |||
121 | public string Name |
||
122 | { |
||
123 | get { return m_ModuleName; } |
||
124 | } |
||
125 | |||
126 | public void Initialise(IConfigSource source) |
||
127 | { |
||
128 | IConfig moduleConfig = source.Configs["Modules"]; |
||
129 | |||
130 | if (moduleConfig != null) |
||
131 | { |
||
132 | string name = moduleConfig.GetString("AssetCaching", String.Empty); |
||
133 | |||
134 | if (name == Name) |
||
135 | { |
||
136 | m_MemoryCache = new ExpiringCache<string, AssetBase>(); |
||
137 | m_Enabled = true; |
||
138 | |||
139 | m_log.InfoFormat("[FLOTSAM ASSET CACHE]: {0} enabled", this.Name); |
||
140 | |||
141 | IConfig assetConfig = source.Configs["AssetCache"]; |
||
142 | if (assetConfig == null) |
||
143 | { |
||
144 | m_log.Debug( |
||
145 | "[FLOTSAM ASSET CACHE]: AssetCache section missing from config (not copied config-include/FlotsamCache.ini.example? Using defaults."); |
||
146 | } |
||
147 | else |
||
148 | { |
||
149 | m_FileCacheEnabled = assetConfig.GetBoolean("FileCacheEnabled", m_FileCacheEnabled); |
||
150 | m_CacheDirectory = assetConfig.GetString("CacheDirectory", m_DefaultCacheDirectory); |
||
151 | |||
152 | m_MemoryCacheEnabled = assetConfig.GetBoolean("MemoryCacheEnabled", m_MemoryCacheEnabled); |
||
153 | m_MemoryExpiration = TimeSpan.FromHours(assetConfig.GetDouble("MemoryCacheTimeout", m_DefaultMemoryExpiration)); |
||
154 | |||
155 | #if WAIT_ON_INPROGRESS_REQUESTS |
||
156 | m_WaitOnInprogressTimeout = assetConfig.GetInt("WaitOnInprogressTimeout", 3000); |
||
157 | #endif |
||
158 | |||
159 | m_LogLevel = assetConfig.GetInt("LogLevel", m_LogLevel); |
||
160 | m_HitRateDisplay = (ulong)assetConfig.GetLong("HitRateDisplay", (long)m_HitRateDisplay); |
||
161 | |||
162 | m_FileExpiration = TimeSpan.FromHours(assetConfig.GetDouble("FileCacheTimeout", m_DefaultFileExpiration)); |
||
163 | m_FileExpirationCleanupTimer |
||
164 | = TimeSpan.FromHours( |
||
165 | assetConfig.GetDouble("FileCleanupTimer", m_FileExpirationCleanupTimer.TotalHours)); |
||
166 | |||
167 | m_CacheDirectoryTiers = assetConfig.GetInt("CacheDirectoryTiers", m_CacheDirectoryTiers); |
||
168 | m_CacheDirectoryTierLen = assetConfig.GetInt("CacheDirectoryTierLength", m_CacheDirectoryTierLen); |
||
169 | |||
170 | m_CacheWarnAt = assetConfig.GetInt("CacheWarnAt", m_CacheWarnAt); |
||
171 | } |
||
172 | |||
173 | m_log.InfoFormat("[FLOTSAM ASSET CACHE]: Cache Directory {0}", m_CacheDirectory); |
||
174 | |||
175 | if (m_FileCacheEnabled && (m_FileExpiration > TimeSpan.Zero) && (m_FileExpirationCleanupTimer > TimeSpan.Zero)) |
||
176 | { |
||
177 | m_CacheCleanTimer = new System.Timers.Timer(m_FileExpirationCleanupTimer.TotalMilliseconds); |
||
178 | m_CacheCleanTimer.AutoReset = true; |
||
179 | m_CacheCleanTimer.Elapsed += CleanupExpiredFiles; |
||
180 | lock (m_CacheCleanTimer) |
||
181 | m_CacheCleanTimer.Start(); |
||
182 | } |
||
183 | |||
184 | if (m_CacheDirectoryTiers < 1) |
||
185 | { |
||
186 | m_CacheDirectoryTiers = 1; |
||
187 | } |
||
188 | else if (m_CacheDirectoryTiers > 3) |
||
189 | { |
||
190 | m_CacheDirectoryTiers = 3; |
||
191 | } |
||
192 | |||
193 | if (m_CacheDirectoryTierLen < 1) |
||
194 | { |
||
195 | m_CacheDirectoryTierLen = 1; |
||
196 | } |
||
197 | else if (m_CacheDirectoryTierLen > 4) |
||
198 | { |
||
199 | m_CacheDirectoryTierLen = 4; |
||
200 | } |
||
201 | |||
202 | MainConsole.Instance.Commands.AddCommand("Assets", true, "fcache status", "fcache status", "Display cache status", HandleConsoleCommand); |
||
203 | MainConsole.Instance.Commands.AddCommand("Assets", true, "fcache clear", "fcache clear [file] [memory]", "Remove all assets in the cache. If file or memory is specified then only this cache is cleared.", HandleConsoleCommand); |
||
204 | MainConsole.Instance.Commands.AddCommand("Assets", true, "fcache assets", "fcache assets", "Attempt a deep scan and cache of all assets in all scenes", HandleConsoleCommand); |
||
205 | MainConsole.Instance.Commands.AddCommand("Assets", true, "fcache expire", "fcache expire <datetime>", "Purge cached assets older then the specified date/time", HandleConsoleCommand); |
||
206 | } |
||
207 | } |
||
208 | } |
||
209 | |||
210 | public void PostInitialise() |
||
211 | { |
||
212 | } |
||
213 | |||
214 | public void Close() |
||
215 | { |
||
216 | } |
||
217 | |||
218 | public void AddRegion(Scene scene) |
||
219 | { |
||
220 | if (m_Enabled) |
||
221 | { |
||
222 | scene.RegisterModuleInterface<IImprovedAssetCache>(this); |
||
223 | m_Scenes.Add(scene); |
||
224 | |||
225 | } |
||
226 | } |
||
227 | |||
228 | public void RemoveRegion(Scene scene) |
||
229 | { |
||
230 | if (m_Enabled) |
||
231 | { |
||
232 | scene.UnregisterModuleInterface<IImprovedAssetCache>(this); |
||
233 | m_Scenes.Remove(scene); |
||
234 | } |
||
235 | } |
||
236 | |||
237 | public void RegionLoaded(Scene scene) |
||
238 | { |
||
239 | if (m_Enabled && m_AssetService == null) |
||
240 | m_AssetService = scene.RequestModuleInterface<IAssetService>(); |
||
241 | } |
||
242 | |||
243 | //////////////////////////////////////////////////////////// |
||
244 | // IImprovedAssetCache |
||
245 | // |
||
246 | |||
247 | private void UpdateMemoryCache(string key, AssetBase asset) |
||
248 | { |
||
249 | m_MemoryCache.AddOrUpdate(key, asset, m_MemoryExpiration); |
||
250 | } |
||
251 | |||
252 | private void UpdateFileCache(string key, AssetBase asset) |
||
253 | { |
||
254 | string filename = GetFileName(asset.ID); |
||
255 | |||
256 | try |
||
257 | { |
||
258 | // If the file is already cached just update access time. |
||
259 | if (File.Exists(filename)) |
||
260 | { |
||
261 | lock (m_CurrentlyWriting) |
||
262 | { |
||
263 | if (!m_CurrentlyWriting.Contains(filename)) |
||
264 | File.SetLastAccessTime(filename, DateTime.Now); |
||
265 | } |
||
266 | } |
||
267 | else |
||
268 | { |
||
269 | // Once we start writing, make sure we flag that we're writing |
||
270 | // that object to the cache so that we don't try to write the |
||
271 | // same file multiple times. |
||
272 | lock (m_CurrentlyWriting) |
||
273 | { |
||
274 | #if WAIT_ON_INPROGRESS_REQUESTS |
||
275 | if (m_CurrentlyWriting.ContainsKey(filename)) |
||
276 | { |
||
277 | return; |
||
278 | } |
||
279 | else |
||
280 | { |
||
281 | m_CurrentlyWriting.Add(filename, new ManualResetEvent(false)); |
||
282 | } |
||
283 | |||
284 | #else |
||
285 | if (m_CurrentlyWriting.Contains(filename)) |
||
286 | { |
||
287 | return; |
||
288 | } |
||
289 | else |
||
290 | { |
||
291 | m_CurrentlyWriting.Add(filename); |
||
292 | } |
||
293 | #endif |
||
294 | } |
||
295 | |||
296 | Util.FireAndForget( |
||
297 | delegate { WriteFileCache(filename, asset); }); |
||
298 | } |
||
299 | } |
||
300 | catch (Exception e) |
||
301 | { |
||
302 | m_log.WarnFormat( |
||
303 | "[FLOTSAM ASSET CACHE]: Failed to update cache for asset {0}. Exception {1} {2}", |
||
304 | asset.ID, e.Message, e.StackTrace); |
||
305 | } |
||
306 | } |
||
307 | |||
308 | public void Cache(AssetBase asset) |
||
309 | { |
||
310 | // TODO: Spawn this off to some seperate thread to do the actual writing |
||
311 | if (asset != null) |
||
312 | { |
||
313 | //m_log.DebugFormat("[FLOTSAM ASSET CACHE]: Caching asset with id {0}", asset.ID); |
||
314 | |||
315 | if (m_MemoryCacheEnabled) |
||
316 | UpdateMemoryCache(asset.ID, asset); |
||
317 | |||
318 | if (m_FileCacheEnabled) |
||
319 | UpdateFileCache(asset.ID, asset); |
||
320 | } |
||
321 | } |
||
322 | |||
323 | /// <summary> |
||
324 | /// Try to get an asset from the in-memory cache. |
||
325 | /// </summary> |
||
326 | /// <param name="id"></param> |
||
327 | /// <returns></returns> |
||
328 | private AssetBase GetFromMemoryCache(string id) |
||
329 | { |
||
330 | AssetBase asset = null; |
||
331 | |||
332 | if (m_MemoryCache.TryGetValue(id, out asset)) |
||
333 | m_MemoryHits++; |
||
334 | |||
335 | return asset; |
||
336 | } |
||
337 | |||
338 | /// <summary> |
||
339 | /// Try to get an asset from the file cache. |
||
340 | /// </summary> |
||
341 | /// <param name="id"></param> |
||
342 | /// <returns>An asset retrieved from the file cache. null if there was a problem retrieving an asset.</returns> |
||
343 | private AssetBase GetFromFileCache(string id) |
||
344 | { |
||
345 | string filename = GetFileName(id); |
||
346 | |||
347 | #if WAIT_ON_INPROGRESS_REQUESTS |
||
348 | // Check if we're already downloading this asset. If so, try to wait for it to |
||
349 | // download. |
||
350 | if (m_WaitOnInprogressTimeout > 0) |
||
351 | { |
||
352 | m_RequestsForInprogress++; |
||
353 | |||
354 | ManualResetEvent waitEvent; |
||
355 | if (m_CurrentlyWriting.TryGetValue(filename, out waitEvent)) |
||
356 | { |
||
357 | waitEvent.WaitOne(m_WaitOnInprogressTimeout); |
||
358 | return Get(id); |
||
359 | } |
||
360 | } |
||
361 | #else |
||
362 | // Track how often we have the problem that an asset is requested while |
||
363 | // it is still being downloaded by a previous request. |
||
364 | if (m_CurrentlyWriting.Contains(filename)) |
||
365 | { |
||
366 | m_RequestsForInprogress++; |
||
367 | return null; |
||
368 | } |
||
369 | #endif |
||
370 | |||
371 | AssetBase asset = null; |
||
372 | |||
373 | if (File.Exists(filename)) |
||
374 | { |
||
375 | FileStream stream = null; |
||
376 | try |
||
377 | { |
||
378 | stream = File.Open(filename, FileMode.Open, FileAccess.Read, FileShare.Read); |
||
379 | BinaryFormatter bformatter = new BinaryFormatter(); |
||
380 | |||
381 | asset = (AssetBase)bformatter.Deserialize(stream); |
||
382 | |||
383 | m_DiskHits++; |
||
384 | } |
||
385 | catch (System.Runtime.Serialization.SerializationException e) |
||
386 | { |
||
387 | m_log.WarnFormat( |
||
388 | "[FLOTSAM ASSET CACHE]: Failed to get file {0} for asset {1}. Exception {2} {3}", |
||
389 | filename, id, e.Message, e.StackTrace); |
||
390 | |||
391 | // If there was a problem deserializing the asset, the asset may |
||
392 | // either be corrupted OR was serialized under an old format |
||
393 | // {different version of AssetBase} -- we should attempt to |
||
394 | // delete it and re-cache |
||
395 | File.Delete(filename); |
||
396 | } |
||
397 | catch (Exception e) |
||
398 | { |
||
399 | m_log.WarnFormat( |
||
400 | "[FLOTSAM ASSET CACHE]: Failed to get file {0} for asset {1}. Exception {2} {3}", |
||
401 | filename, id, e.Message, e.StackTrace); |
||
402 | } |
||
403 | finally |
||
404 | { |
||
405 | if (stream != null) |
||
406 | stream.Close(); |
||
407 | } |
||
408 | } |
||
409 | |||
410 | return asset; |
||
411 | } |
||
412 | |||
413 | public AssetBase Get(string id) |
||
414 | { |
||
415 | m_Requests++; |
||
416 | |||
417 | AssetBase asset = null; |
||
418 | |||
419 | if (m_MemoryCacheEnabled) |
||
420 | asset = GetFromMemoryCache(id); |
||
421 | |||
422 | if (asset == null && m_FileCacheEnabled) |
||
423 | { |
||
424 | asset = GetFromFileCache(id); |
||
425 | |||
426 | if (m_MemoryCacheEnabled && asset != null) |
||
427 | UpdateMemoryCache(id, asset); |
||
428 | } |
||
429 | |||
430 | if (((m_LogLevel >= 1)) && (m_HitRateDisplay != 0) && (m_Requests % m_HitRateDisplay == 0)) |
||
431 | { |
||
432 | m_HitRateFile = (double)m_DiskHits / m_Requests * 100.0; |
||
433 | |||
434 | m_log.InfoFormat("[FLOTSAM ASSET CACHE]: Cache Get :: {0} :: {1}", id, asset == null ? "Miss" : "Hit"); |
||
435 | m_log.InfoFormat("[FLOTSAM ASSET CACHE]: File Hit Rate {0}% for {1} requests", m_HitRateFile.ToString("0.00"), m_Requests); |
||
436 | |||
437 | if (m_MemoryCacheEnabled) |
||
438 | { |
||
439 | m_HitRateMemory = (double)m_MemoryHits / m_Requests * 100.0; |
||
440 | m_log.InfoFormat("[FLOTSAM ASSET CACHE]: Memory Hit Rate {0}% for {1} requests", m_HitRateMemory.ToString("0.00"), m_Requests); |
||
441 | } |
||
442 | |||
443 | m_log.InfoFormat("[FLOTSAM ASSET CACHE]: {0} unnessesary requests due to requests for assets that are currently downloading.", m_RequestsForInprogress); |
||
444 | } |
||
445 | |||
446 | return asset; |
||
447 | } |
||
448 | |||
449 | public AssetBase GetCached(string id) |
||
450 | { |
||
451 | return Get(id); |
||
452 | } |
||
453 | |||
454 | public void Expire(string id) |
||
455 | { |
||
456 | if (m_LogLevel >= 2) |
||
457 | m_log.DebugFormat("[FLOTSAM ASSET CACHE]: Expiring Asset {0}", id); |
||
458 | |||
459 | try |
||
460 | { |
||
461 | if (m_FileCacheEnabled) |
||
462 | { |
||
463 | string filename = GetFileName(id); |
||
464 | if (File.Exists(filename)) |
||
465 | { |
||
466 | File.Delete(filename); |
||
467 | } |
||
468 | } |
||
469 | |||
470 | if (m_MemoryCacheEnabled) |
||
471 | m_MemoryCache.Remove(id); |
||
472 | } |
||
473 | catch (Exception e) |
||
474 | { |
||
475 | m_log.WarnFormat( |
||
476 | "[FLOTSAM ASSET CACHE]: Failed to expire cached file {0}. Exception {1} {2}", |
||
477 | id, e.Message, e.StackTrace); |
||
478 | } |
||
479 | } |
||
480 | |||
481 | public void Clear() |
||
482 | { |
||
483 | if (m_LogLevel >= 2) |
||
484 | m_log.Debug("[FLOTSAM ASSET CACHE]: Clearing caches."); |
||
485 | |||
486 | if (m_FileCacheEnabled) |
||
487 | { |
||
488 | foreach (string dir in Directory.GetDirectories(m_CacheDirectory)) |
||
489 | { |
||
490 | Directory.Delete(dir); |
||
491 | } |
||
492 | } |
||
493 | |||
494 | if (m_MemoryCacheEnabled) |
||
495 | m_MemoryCache.Clear(); |
||
496 | } |
||
497 | |||
498 | private void CleanupExpiredFiles(object source, ElapsedEventArgs e) |
||
499 | { |
||
500 | if (m_LogLevel >= 2) |
||
501 | m_log.DebugFormat("[FLOTSAM ASSET CACHE]: Checking for expired files older then {0}.", m_FileExpiration); |
||
502 | |||
503 | // Purge all files last accessed prior to this point |
||
504 | DateTime purgeLine = DateTime.Now - m_FileExpiration; |
||
505 | |||
506 | // An asset cache may contain local non-temporary assets that are not in the asset service. Therefore, |
||
507 | // before cleaning up expired files we must scan the objects in the scene to make sure that we retain |
||
508 | // such local assets if they have not been recently accessed. |
||
509 | TouchAllSceneAssets(false); |
||
510 | |||
511 | foreach (string dir in Directory.GetDirectories(m_CacheDirectory)) |
||
512 | { |
||
513 | CleanExpiredFiles(dir, purgeLine); |
||
514 | } |
||
515 | } |
||
516 | |||
517 | /// <summary> |
||
518 | /// Recurses through specified directory checking for asset files last |
||
519 | /// accessed prior to the specified purge line and deletes them. Also |
||
520 | /// removes empty tier directories. |
||
521 | /// </summary> |
||
522 | /// <param name="dir"></param> |
||
523 | /// <param name="purgeLine"></param> |
||
524 | private void CleanExpiredFiles(string dir, DateTime purgeLine) |
||
525 | { |
||
526 | try |
||
527 | { |
||
528 | foreach (string file in Directory.GetFiles(dir)) |
||
529 | { |
||
530 | if (File.GetLastAccessTime(file) < purgeLine) |
||
531 | { |
||
532 | File.Delete(file); |
||
533 | } |
||
534 | } |
||
535 | |||
536 | // Recurse into lower tiers |
||
537 | foreach (string subdir in Directory.GetDirectories(dir)) |
||
538 | { |
||
539 | CleanExpiredFiles(subdir, purgeLine); |
||
540 | } |
||
541 | |||
542 | // Check if a tier directory is empty, if so, delete it |
||
543 | int dirSize = Directory.GetFiles(dir).Length + Directory.GetDirectories(dir).Length; |
||
544 | if (dirSize == 0) |
||
545 | { |
||
546 | Directory.Delete(dir); |
||
547 | } |
||
548 | else if (dirSize >= m_CacheWarnAt) |
||
549 | { |
||
550 | m_log.WarnFormat( |
||
551 | "[FLOTSAM ASSET CACHE]: Cache folder exceeded CacheWarnAt limit {0} {1}. Suggest increasing tiers, tier length, or reducing cache expiration", |
||
552 | dir, dirSize); |
||
553 | } |
||
554 | } |
||
555 | catch (Exception e) |
||
556 | { |
||
557 | m_log.Warn( |
||
558 | string.Format("[FLOTSAM ASSET CACHE]: Could not complete clean of expired files in {0}, exception ", dir), e); |
||
559 | } |
||
560 | } |
||
561 | |||
562 | /// <summary> |
||
563 | /// Determines the filename for an AssetID stored in the file cache |
||
564 | /// </summary> |
||
565 | /// <param name="id"></param> |
||
566 | /// <returns></returns> |
||
567 | private string GetFileName(string id) |
||
568 | { |
||
569 | // Would it be faster to just hash the darn thing? |
||
570 | foreach (char c in m_InvalidChars) |
||
571 | { |
||
572 | id = id.Replace(c, '_'); |
||
573 | } |
||
574 | |||
575 | string path = m_CacheDirectory; |
||
576 | for (int p = 1; p <= m_CacheDirectoryTiers; p++) |
||
577 | { |
||
578 | string pathPart = id.Substring((p - 1) * m_CacheDirectoryTierLen, m_CacheDirectoryTierLen); |
||
579 | path = Path.Combine(path, pathPart); |
||
580 | } |
||
581 | |||
582 | return Path.Combine(path, id); |
||
583 | } |
||
584 | |||
585 | /// <summary> |
||
586 | /// Writes a file to the file cache, creating any nessesary |
||
587 | /// tier directories along the way |
||
588 | /// </summary> |
||
589 | /// <param name="filename"></param> |
||
590 | /// <param name="asset"></param> |
||
591 | private void WriteFileCache(string filename, AssetBase asset) |
||
592 | { |
||
593 | Stream stream = null; |
||
594 | |||
595 | // Make sure the target cache directory exists |
||
596 | string directory = Path.GetDirectoryName(filename); |
||
597 | |||
598 | // Write file first to a temp name, so that it doesn't look |
||
599 | // like it's already cached while it's still writing. |
||
600 | string tempname = Path.Combine(directory, Path.GetRandomFileName()); |
||
601 | |||
602 | try |
||
603 | { |
||
604 | try |
||
605 | { |
||
606 | if (!Directory.Exists(directory)) |
||
607 | { |
||
608 | Directory.CreateDirectory(directory); |
||
609 | } |
||
610 | |||
611 | stream = File.Open(tempname, FileMode.Create); |
||
612 | BinaryFormatter bformatter = new BinaryFormatter(); |
||
613 | bformatter.Serialize(stream, asset); |
||
614 | } |
||
615 | catch (IOException e) |
||
616 | { |
||
617 | m_log.WarnFormat( |
||
618 | "[FLOTSAM ASSET CACHE]: Failed to write asset {0} to temporary location {1} (final {2}) on cache in {3}. Exception {4} {5}.", |
||
619 | asset.ID, tempname, filename, directory, e.Message, e.StackTrace); |
||
620 | |||
621 | return; |
||
622 | } |
||
623 | finally |
||
624 | { |
||
625 | if (stream != null) |
||
626 | stream.Close(); |
||
627 | } |
||
628 | |||
629 | try |
||
630 | { |
||
631 | // Now that it's written, rename it so that it can be found. |
||
632 | // |
||
633 | // File.Copy(tempname, filename, true); |
||
634 | // File.Delete(tempname); |
||
635 | // |
||
636 | // For a brief period, this was done as a separate copy and then temporary file delete operation to |
||
637 | // avoid an IOException caused by move if some competing thread had already written the file. |
||
638 | // However, this causes exceptions on Windows when other threads attempt to read a file |
||
639 | // which is still being copied. So instead, go back to moving the file and swallow any IOException. |
||
640 | // |
||
641 | // This situation occurs fairly rarely anyway. We assume in this that moves are atomic on the |
||
642 | // filesystem. |
||
643 | File.Move(tempname, filename); |
||
644 | |||
645 | if (m_LogLevel >= 2) |
||
646 | m_log.DebugFormat("[FLOTSAM ASSET CACHE]: Cache Stored :: {0}", asset.ID); |
||
647 | } |
||
648 | catch (IOException) |
||
649 | { |
||
650 | // If we see an IOException here it's likely that some other competing thread has written the |
||
651 | // cache file first, so ignore. Other IOException errors (e.g. filesystem full) should be |
||
652 | // signally by the earlier temporary file writing code. |
||
653 | } |
||
654 | } |
||
655 | finally |
||
656 | { |
||
657 | // Even if the write fails with an exception, we need to make sure |
||
658 | // that we release the lock on that file, otherwise it'll never get |
||
659 | // cached |
||
660 | lock (m_CurrentlyWriting) |
||
661 | { |
||
662 | #if WAIT_ON_INPROGRESS_REQUESTS |
||
663 | ManualResetEvent waitEvent; |
||
664 | if (m_CurrentlyWriting.TryGetValue(filename, out waitEvent)) |
||
665 | { |
||
666 | m_CurrentlyWriting.Remove(filename); |
||
667 | waitEvent.Set(); |
||
668 | } |
||
669 | #else |
||
670 | m_CurrentlyWriting.Remove(filename); |
||
671 | #endif |
||
672 | } |
||
673 | } |
||
674 | } |
||
675 | |||
676 | /// <summary> |
||
677 | /// Scan through the file cache, and return number of assets currently cached. |
||
678 | /// </summary> |
||
679 | /// <param name="dir"></param> |
||
680 | /// <returns></returns> |
||
681 | private int GetFileCacheCount(string dir) |
||
682 | { |
||
683 | int count = Directory.GetFiles(dir).Length; |
||
684 | |||
685 | foreach (string subdir in Directory.GetDirectories(dir)) |
||
686 | { |
||
687 | count += GetFileCacheCount(subdir); |
||
688 | } |
||
689 | |||
690 | return count; |
||
691 | } |
||
692 | |||
693 | /// <summary> |
||
694 | /// This notes the last time the Region had a deep asset scan performed on it. |
||
695 | /// </summary> |
||
696 | /// <param name="regionID"></param> |
||
697 | private void StampRegionStatusFile(UUID regionID) |
||
698 | { |
||
699 | string RegionCacheStatusFile = Path.Combine(m_CacheDirectory, "RegionStatus_" + regionID.ToString() + ".fac"); |
||
700 | |||
701 | try |
||
702 | { |
||
703 | if (File.Exists(RegionCacheStatusFile)) |
||
704 | { |
||
705 | File.SetLastWriteTime(RegionCacheStatusFile, DateTime.Now); |
||
706 | } |
||
707 | else |
||
708 | { |
||
709 | File.WriteAllText( |
||
710 | RegionCacheStatusFile, |
||
711 | "Please do not delete this file unless you are manually clearing your Flotsam Asset Cache."); |
||
712 | } |
||
713 | } |
||
714 | catch (Exception e) |
||
715 | { |
||
716 | m_log.Warn( |
||
717 | string.Format( |
||
718 | "[FLOTSAM ASSET CACHE]: Could not stamp region status file for region {0}. Exception ", |
||
719 | regionID), |
||
720 | e); |
||
721 | } |
||
722 | } |
||
723 | |||
724 | /// <summary> |
||
725 | /// Iterates through all Scenes, doing a deep scan through assets |
||
726 | /// to update the access time of all assets present in the scene or referenced by assets |
||
727 | /// in the scene. |
||
728 | /// </summary> |
||
729 | /// <param name="storeUncached"> |
||
730 | /// If true, then assets scanned which are not found in cache are added to the cache. |
||
731 | /// </param> |
||
732 | /// <returns>Number of distinct asset references found in the scene.</returns> |
||
733 | private int TouchAllSceneAssets(bool storeUncached) |
||
734 | { |
||
735 | UuidGatherer gatherer = new UuidGatherer(m_AssetService); |
||
736 | |||
737 | HashSet<UUID> uniqueUuids = new HashSet<UUID>(); |
||
738 | Dictionary<UUID, AssetType> assets = new Dictionary<UUID, AssetType>(); |
||
739 | |||
740 | foreach (Scene s in m_Scenes) |
||
741 | { |
||
742 | StampRegionStatusFile(s.RegionInfo.RegionID); |
||
743 | |||
744 | s.ForEachSOG(delegate(SceneObjectGroup e) |
||
745 | { |
||
746 | gatherer.GatherAssetUuids(e, assets); |
||
747 | |||
748 | foreach (UUID assetID in assets.Keys) |
||
749 | { |
||
750 | uniqueUuids.Add(assetID); |
||
751 | |||
752 | string filename = GetFileName(assetID.ToString()); |
||
753 | |||
754 | if (File.Exists(filename)) |
||
755 | { |
||
756 | File.SetLastAccessTime(filename, DateTime.Now); |
||
757 | } |
||
758 | else if (storeUncached) |
||
759 | { |
||
760 | AssetBase cachedAsset = m_AssetService.Get(assetID.ToString()); |
||
761 | if (cachedAsset == null && assets[assetID] != AssetType.Unknown) |
||
762 | m_log.DebugFormat( |
||
763 | "[FLOTSAM ASSET CACHE]: Could not find asset {0}, type {1} referenced by object {2} at {3} in scene {4} when pre-caching all scene assets", |
||
764 | assetID, assets[assetID], e.Name, e.AbsolutePosition, s.Name); |
||
765 | } |
||
766 | } |
||
767 | |||
768 | assets.Clear(); |
||
769 | }); |
||
770 | } |
||
771 | |||
772 | |||
773 | return uniqueUuids.Count; |
||
774 | } |
||
775 | |||
776 | /// <summary> |
||
777 | /// Deletes all cache contents |
||
778 | /// </summary> |
||
779 | private void ClearFileCache() |
||
780 | { |
||
781 | foreach (string dir in Directory.GetDirectories(m_CacheDirectory)) |
||
782 | { |
||
783 | try |
||
784 | { |
||
785 | Directory.Delete(dir, true); |
||
786 | } |
||
787 | catch (Exception e) |
||
788 | { |
||
789 | m_log.WarnFormat( |
||
790 | "[FLOTSAM ASSET CACHE]: Couldn't clear asset cache directory {0} from {1}. Exception {2} {3}", |
||
791 | dir, m_CacheDirectory, e.Message, e.StackTrace); |
||
792 | } |
||
793 | } |
||
794 | |||
795 | foreach (string file in Directory.GetFiles(m_CacheDirectory)) |
||
796 | { |
||
797 | try |
||
798 | { |
||
799 | File.Delete(file); |
||
800 | } |
||
801 | catch (Exception e) |
||
802 | { |
||
803 | m_log.WarnFormat( |
||
804 | "[FLOTSAM ASSET CACHE]: Couldn't clear asset cache file {0} from {1}. Exception {1} {2}", |
||
805 | file, m_CacheDirectory, e.Message, e.StackTrace); |
||
806 | } |
||
807 | } |
||
808 | } |
||
809 | |||
810 | #region Console Commands |
||
811 | private void HandleConsoleCommand(string module, string[] cmdparams) |
||
812 | { |
||
813 | if (cmdparams.Length >= 2) |
||
814 | { |
||
815 | string cmd = cmdparams[1]; |
||
816 | switch (cmd) |
||
817 | { |
||
818 | case "status": |
||
819 | if (m_MemoryCacheEnabled) |
||
820 | m_log.InfoFormat("[FLOTSAM ASSET CACHE]: Memory Cache : {0} assets", m_MemoryCache.Count); |
||
821 | else |
||
822 | m_log.InfoFormat("[FLOTSAM ASSET CACHE]: Memory cache disabled"); |
||
823 | |||
824 | if (m_FileCacheEnabled) |
||
825 | { |
||
826 | int fileCount = GetFileCacheCount(m_CacheDirectory); |
||
827 | m_log.InfoFormat("[FLOTSAM ASSET CACHE]: File Cache : {0} assets", fileCount); |
||
828 | |||
829 | foreach (string s in Directory.GetFiles(m_CacheDirectory, "*.fac")) |
||
830 | { |
||
831 | m_log.Info("[FLOTSAM ASSET CACHE]: Deep scans have previously been performed on the following regions:"); |
||
832 | |||
833 | string RegionID = s.Remove(0,s.IndexOf("_")).Replace(".fac",""); |
||
834 | DateTime RegionDeepScanTMStamp = File.GetLastWriteTime(s); |
||
835 | m_log.InfoFormat("[FLOTSAM ASSET CACHE]: Region: {0}, {1}", RegionID, RegionDeepScanTMStamp.ToString("MM/dd/yyyy hh:mm:ss")); |
||
836 | } |
||
837 | } |
||
838 | else |
||
839 | { |
||
840 | m_log.InfoFormat("[FLOTSAM ASSET CACHE]: File cache disabled"); |
||
841 | } |
||
842 | |||
843 | break; |
||
844 | |||
845 | case "clear": |
||
846 | if (cmdparams.Length < 2) |
||
847 | { |
||
848 | m_log.Warn("[FLOTSAM ASSET CACHE]: Usage is fcache clear [file] [memory]"); |
||
849 | break; |
||
850 | } |
||
851 | |||
852 | bool clearMemory = false, clearFile = false; |
||
853 | |||
854 | if (cmdparams.Length == 2) |
||
855 | { |
||
856 | clearMemory = true; |
||
857 | clearFile = true; |
||
858 | } |
||
859 | foreach (string s in cmdparams) |
||
860 | { |
||
861 | if (s.ToLower() == "memory") |
||
862 | clearMemory = true; |
||
863 | else if (s.ToLower() == "file") |
||
864 | clearFile = true; |
||
865 | } |
||
866 | |||
867 | if (clearMemory) |
||
868 | { |
||
869 | if (m_MemoryCacheEnabled) |
||
870 | { |
||
871 | m_MemoryCache.Clear(); |
||
872 | m_log.Info("[FLOTSAM ASSET CACHE]: Memory cache cleared."); |
||
873 | } |
||
874 | else |
||
875 | { |
||
876 | m_log.Info("[FLOTSAM ASSET CACHE]: Memory cache not enabled."); |
||
877 | } |
||
878 | } |
||
879 | |||
880 | if (clearFile) |
||
881 | { |
||
882 | if (m_FileCacheEnabled) |
||
883 | { |
||
884 | ClearFileCache(); |
||
885 | m_log.Info("[FLOTSAM ASSET CACHE]: File cache cleared."); |
||
886 | } |
||
887 | else |
||
888 | { |
||
889 | m_log.Info("[FLOTSAM ASSET CACHE]: File cache not enabled."); |
||
890 | } |
||
891 | } |
||
892 | |||
893 | break; |
||
894 | |||
895 | case "assets": |
||
896 | m_log.Info("[FLOTSAM ASSET CACHE]: Ensuring assets are cached for all scenes."); |
||
897 | |||
898 | Util.FireAndForget(delegate { |
||
899 | int assetReferenceTotal = TouchAllSceneAssets(true); |
||
900 | m_log.InfoFormat( |
||
901 | "[FLOTSAM ASSET CACHE]: Completed check with {0} assets.", |
||
902 | assetReferenceTotal); |
||
903 | }); |
||
904 | |||
905 | break; |
||
906 | |||
907 | case "expire": |
||
908 | if (cmdparams.Length < 3) |
||
909 | { |
||
910 | m_log.InfoFormat("[FLOTSAM ASSET CACHE]: Invalid parameters for Expire, please specify a valid date & time", cmd); |
||
911 | break; |
||
912 | } |
||
913 | |||
914 | string s_expirationDate = ""; |
||
915 | DateTime expirationDate; |
||
916 | |||
917 | if (cmdparams.Length > 3) |
||
918 | { |
||
919 | s_expirationDate = string.Join(" ", cmdparams, 2, cmdparams.Length - 2); |
||
920 | } |
||
921 | else |
||
922 | { |
||
923 | s_expirationDate = cmdparams[2]; |
||
924 | } |
||
925 | |||
926 | if (!DateTime.TryParse(s_expirationDate, out expirationDate)) |
||
927 | { |
||
928 | m_log.InfoFormat("[FLOTSAM ASSET CACHE]: {0} is not a valid date & time", cmd); |
||
929 | break; |
||
930 | } |
||
931 | |||
932 | if (m_FileCacheEnabled) |
||
933 | CleanExpiredFiles(m_CacheDirectory, expirationDate); |
||
934 | else |
||
935 | m_log.InfoFormat("[FLOTSAM ASSET CACHE]: File cache not active, not clearing."); |
||
936 | |||
937 | break; |
||
938 | default: |
||
939 | m_log.InfoFormat("[FLOTSAM ASSET CACHE]: Unknown command {0}", cmd); |
||
940 | break; |
||
941 | } |
||
942 | } |
||
943 | else if (cmdparams.Length == 1) |
||
944 | { |
||
945 | m_log.InfoFormat("[FLOTSAM ASSET CACHE]: fcache status - Display cache status"); |
||
946 | m_log.InfoFormat("[FLOTSAM ASSET CACHE]: fcache clearmem - Remove all assets cached in memory"); |
||
947 | m_log.InfoFormat("[FLOTSAM ASSET CACHE]: fcache clearfile - Remove all assets cached on disk"); |
||
948 | m_log.InfoFormat("[FLOTSAM ASSET CACHE]: fcache cachescenes - Attempt a deep cache of all assets in all scenes"); |
||
949 | m_log.InfoFormat("[FLOTSAM ASSET CACHE]: fcache <datetime> - Purge assets older then the specified date & time"); |
||
950 | } |
||
951 | } |
||
952 | |||
953 | #endregion |
||
954 | |||
955 | #region IAssetService Members |
||
956 | |||
957 | public AssetMetadata GetMetadata(string id) |
||
958 | { |
||
959 | AssetBase asset = Get(id); |
||
960 | return asset.Metadata; |
||
961 | } |
||
962 | |||
963 | public byte[] GetData(string id) |
||
964 | { |
||
965 | AssetBase asset = Get(id); |
||
966 | return asset.Data; |
||
967 | } |
||
968 | |||
969 | public bool Get(string id, object sender, AssetRetrieved handler) |
||
970 | { |
||
971 | AssetBase asset = Get(id); |
||
972 | handler(id, sender, asset); |
||
973 | return true; |
||
974 | } |
||
975 | |||
976 | public string Store(AssetBase asset) |
||
977 | { |
||
978 | if (asset.FullID == UUID.Zero) |
||
979 | { |
||
980 | asset.FullID = UUID.Random(); |
||
981 | } |
||
982 | |||
983 | Cache(asset); |
||
984 | |||
985 | return asset.ID; |
||
986 | } |
||
987 | |||
988 | public bool UpdateContent(string id, byte[] data) |
||
989 | { |
||
990 | AssetBase asset = Get(id); |
||
991 | asset.Data = data; |
||
992 | Cache(asset); |
||
993 | return true; |
||
994 | } |
||
995 | |||
996 | public bool Delete(string id) |
||
997 | { |
||
998 | Expire(id); |
||
999 | return true; |
||
1000 | } |
||
1001 | |||
1002 | #endregion |
||
1003 | } |
||
1004 | } |