opensim – Blame information for rev 1
?pathlinks?
Rev | Author | Line No. | Line |
---|---|---|---|
1 | eva | 1 | /* |
2 | * Copyright (c) Contributors, http://opensimulator.org/ |
||
3 | * See CONTRIBUTORS.TXT for a full list of copyright holders. |
||
4 | * |
||
5 | * Redistribution and use in source and binary forms, with or without |
||
6 | * modification, are permitted provided that the following conditions are met: |
||
7 | * * Redistributions of source code must retain the above copyright |
||
8 | * notice, this list of conditions and the following disclaimer. |
||
9 | * * Redistributions in binary form must reproduce the above copyright |
||
10 | * notice, this list of conditions and the following disclaimer in the |
||
11 | * documentation and/or other materials provided with the distribution. |
||
12 | * * Neither the name of the OpenSimulator Project nor the |
||
13 | * names of its contributors may be used to endorse or promote products |
||
14 | * derived from this software without specific prior written permission. |
||
15 | * |
||
16 | * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY |
||
17 | * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
||
18 | * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE |
||
19 | * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY |
||
20 | * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES |
||
21 | * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
||
22 | * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND |
||
23 | * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
||
24 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS |
||
25 | * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
||
26 | */ |
||
27 | |||
28 | // Uncomment to make asset Get requests for existing |
||
29 | // #define WAIT_ON_INPROGRESS_REQUESTS |
||
30 | |||
31 | using System; |
||
32 | using System.IO; |
||
33 | using System.Collections.Generic; |
||
34 | using System.Linq; |
||
35 | using System.Reflection; |
||
36 | using System.Runtime.Serialization; |
||
37 | using System.Runtime.Serialization.Formatters.Binary; |
||
38 | using System.Threading; |
||
39 | using System.Timers; |
||
40 | using log4net; |
||
41 | using Nini.Config; |
||
42 | using Mono.Addins; |
||
43 | using OpenMetaverse; |
||
44 | using OpenSim.Framework; |
||
45 | using OpenSim.Framework.Console; |
||
46 | using OpenSim.Region.Framework.Interfaces; |
||
47 | using OpenSim.Region.Framework.Scenes; |
||
48 | using OpenSim.Services.Interfaces; |
||
49 | |||
50 | |||
51 | //[assembly: Addin("FlotsamAssetCache", "1.1")] |
||
52 | //[assembly: AddinDependency("OpenSim", "0.5")] |
||
53 | |||
54 | namespace OpenSim.Region.CoreModules.Asset |
||
55 | { |
||
56 | [Extension(Path = "/OpenSim/RegionModules", NodeName = "RegionModule", Id = "FlotsamAssetCache")] |
||
57 | public class FlotsamAssetCache : ISharedRegionModule, IImprovedAssetCache, IAssetService |
||
58 | { |
||
59 | private static readonly ILog m_log = |
||
60 | LogManager.GetLogger( |
||
61 | MethodBase.GetCurrentMethod().DeclaringType); |
||
62 | |||
63 | private bool m_Enabled; |
||
64 | |||
65 | private const string m_ModuleName = "FlotsamAssetCache"; |
||
66 | private const string m_DefaultCacheDirectory = "./assetcache"; |
||
67 | private string m_CacheDirectory = m_DefaultCacheDirectory; |
||
68 | |||
69 | private readonly List<char> m_InvalidChars = new List<char>(); |
||
70 | |||
71 | private int m_LogLevel = 0; |
||
72 | private ulong m_HitRateDisplay = 100; // How often to display hit statistics, given in requests |
||
73 | |||
74 | private static ulong m_Requests; |
||
75 | private static ulong m_RequestsForInprogress; |
||
76 | private static ulong m_DiskHits; |
||
77 | private static ulong m_MemoryHits; |
||
78 | |||
79 | #if WAIT_ON_INPROGRESS_REQUESTS |
||
80 | private Dictionary<string, ManualResetEvent> m_CurrentlyWriting = new Dictionary<string, ManualResetEvent>(); |
||
81 | private int m_WaitOnInprogressTimeout = 3000; |
||
82 | #else |
||
83 | private HashSet<string> m_CurrentlyWriting = new HashSet<string>(); |
||
84 | #endif |
||
85 | |||
86 | private bool m_FileCacheEnabled = true; |
||
87 | |||
88 | private ExpiringCache<string, AssetBase> m_MemoryCache; |
||
89 | private bool m_MemoryCacheEnabled = false; |
||
90 | |||
91 | // Expiration is expressed in hours. |
||
92 | private const double m_DefaultMemoryExpiration = 2; |
||
93 | private const double m_DefaultFileExpiration = 48; |
||
94 | private TimeSpan m_MemoryExpiration = TimeSpan.FromHours(m_DefaultMemoryExpiration); |
||
95 | private TimeSpan m_FileExpiration = TimeSpan.FromHours(m_DefaultFileExpiration); |
||
96 | private TimeSpan m_FileExpirationCleanupTimer = TimeSpan.FromHours(0.166); |
||
97 | |||
98 | private static int m_CacheDirectoryTiers = 1; |
||
99 | private static int m_CacheDirectoryTierLen = 3; |
||
100 | private static int m_CacheWarnAt = 30000; |
||
101 | |||
102 | private System.Timers.Timer m_CacheCleanTimer; |
||
103 | |||
104 | private IAssetService m_AssetService; |
||
105 | private List<Scene> m_Scenes = new List<Scene>(); |
||
106 | |||
107 | public FlotsamAssetCache() |
||
108 | { |
||
109 | m_InvalidChars.AddRange(Path.GetInvalidPathChars()); |
||
110 | m_InvalidChars.AddRange(Path.GetInvalidFileNameChars()); |
||
111 | } |
||
112 | |||
113 | public Type ReplaceableInterface |
||
114 | { |
||
115 | get { return null; } |
||
116 | } |
||
117 | |||
118 | public string Name |
||
119 | { |
||
120 | get { return m_ModuleName; } |
||
121 | } |
||
122 | |||
123 | public void Initialise(IConfigSource source) |
||
124 | { |
||
125 | IConfig moduleConfig = source.Configs["Modules"]; |
||
126 | |||
127 | if (moduleConfig != null) |
||
128 | { |
||
129 | string name = moduleConfig.GetString("AssetCaching", String.Empty); |
||
130 | |||
131 | if (name == Name) |
||
132 | { |
||
133 | m_MemoryCache = new ExpiringCache<string, AssetBase>(); |
||
134 | m_Enabled = true; |
||
135 | |||
136 | m_log.InfoFormat("[FLOTSAM ASSET CACHE]: {0} enabled", this.Name); |
||
137 | |||
138 | IConfig assetConfig = source.Configs["AssetCache"]; |
||
139 | if (assetConfig == null) |
||
140 | { |
||
141 | m_log.Debug( |
||
142 | "[FLOTSAM ASSET CACHE]: AssetCache section missing from config (not copied config-include/FlotsamCache.ini.example? Using defaults."); |
||
143 | } |
||
144 | else |
||
145 | { |
||
146 | m_FileCacheEnabled = assetConfig.GetBoolean("FileCacheEnabled", m_FileCacheEnabled); |
||
147 | m_CacheDirectory = assetConfig.GetString("CacheDirectory", m_DefaultCacheDirectory); |
||
148 | |||
149 | m_MemoryCacheEnabled = assetConfig.GetBoolean("MemoryCacheEnabled", m_MemoryCacheEnabled); |
||
150 | m_MemoryExpiration = TimeSpan.FromHours(assetConfig.GetDouble("MemoryCacheTimeout", m_DefaultMemoryExpiration)); |
||
151 | |||
152 | #if WAIT_ON_INPROGRESS_REQUESTS |
||
153 | m_WaitOnInprogressTimeout = assetConfig.GetInt("WaitOnInprogressTimeout", 3000); |
||
154 | #endif |
||
155 | |||
156 | m_LogLevel = assetConfig.GetInt("LogLevel", m_LogLevel); |
||
157 | m_HitRateDisplay = (ulong)assetConfig.GetLong("HitRateDisplay", (long)m_HitRateDisplay); |
||
158 | |||
159 | m_FileExpiration = TimeSpan.FromHours(assetConfig.GetDouble("FileCacheTimeout", m_DefaultFileExpiration)); |
||
160 | m_FileExpirationCleanupTimer |
||
161 | = TimeSpan.FromHours( |
||
162 | assetConfig.GetDouble("FileCleanupTimer", m_FileExpirationCleanupTimer.TotalHours)); |
||
163 | |||
164 | m_CacheDirectoryTiers = assetConfig.GetInt("CacheDirectoryTiers", m_CacheDirectoryTiers); |
||
165 | m_CacheDirectoryTierLen = assetConfig.GetInt("CacheDirectoryTierLength", m_CacheDirectoryTierLen); |
||
166 | |||
167 | m_CacheWarnAt = assetConfig.GetInt("CacheWarnAt", m_CacheWarnAt); |
||
168 | } |
||
169 | |||
170 | m_log.InfoFormat("[FLOTSAM ASSET CACHE]: Cache Directory {0}", m_CacheDirectory); |
||
171 | |||
172 | if (m_FileCacheEnabled && (m_FileExpiration > TimeSpan.Zero) && (m_FileExpirationCleanupTimer > TimeSpan.Zero)) |
||
173 | { |
||
174 | m_CacheCleanTimer = new System.Timers.Timer(m_FileExpirationCleanupTimer.TotalMilliseconds); |
||
175 | m_CacheCleanTimer.AutoReset = true; |
||
176 | m_CacheCleanTimer.Elapsed += CleanupExpiredFiles; |
||
177 | lock (m_CacheCleanTimer) |
||
178 | m_CacheCleanTimer.Start(); |
||
179 | } |
||
180 | |||
181 | if (m_CacheDirectoryTiers < 1) |
||
182 | { |
||
183 | m_CacheDirectoryTiers = 1; |
||
184 | } |
||
185 | else if (m_CacheDirectoryTiers > 3) |
||
186 | { |
||
187 | m_CacheDirectoryTiers = 3; |
||
188 | } |
||
189 | |||
190 | if (m_CacheDirectoryTierLen < 1) |
||
191 | { |
||
192 | m_CacheDirectoryTierLen = 1; |
||
193 | } |
||
194 | else if (m_CacheDirectoryTierLen > 4) |
||
195 | { |
||
196 | m_CacheDirectoryTierLen = 4; |
||
197 | } |
||
198 | |||
199 | MainConsole.Instance.Commands.AddCommand("Assets", true, "fcache status", "fcache status", "Display cache status", HandleConsoleCommand); |
||
200 | MainConsole.Instance.Commands.AddCommand("Assets", true, "fcache clear", "fcache clear [file] [memory]", "Remove all assets in the cache. If file or memory is specified then only this cache is cleared.", HandleConsoleCommand); |
||
201 | MainConsole.Instance.Commands.AddCommand("Assets", true, "fcache assets", "fcache assets", "Attempt a deep scan and cache of all assets in all scenes", HandleConsoleCommand); |
||
202 | MainConsole.Instance.Commands.AddCommand("Assets", true, "fcache expire", "fcache expire <datetime>", "Purge cached assets older then the specified date/time", HandleConsoleCommand); |
||
203 | } |
||
204 | } |
||
205 | } |
||
206 | |||
207 | public void PostInitialise() |
||
208 | { |
||
209 | } |
||
210 | |||
211 | public void Close() |
||
212 | { |
||
213 | } |
||
214 | |||
215 | public void AddRegion(Scene scene) |
||
216 | { |
||
217 | if (m_Enabled) |
||
218 | { |
||
219 | scene.RegisterModuleInterface<IImprovedAssetCache>(this); |
||
220 | m_Scenes.Add(scene); |
||
221 | |||
222 | } |
||
223 | } |
||
224 | |||
225 | public void RemoveRegion(Scene scene) |
||
226 | { |
||
227 | if (m_Enabled) |
||
228 | { |
||
229 | scene.UnregisterModuleInterface<IImprovedAssetCache>(this); |
||
230 | m_Scenes.Remove(scene); |
||
231 | } |
||
232 | } |
||
233 | |||
234 | public void RegionLoaded(Scene scene) |
||
235 | { |
||
236 | if (m_Enabled && m_AssetService == null) |
||
237 | m_AssetService = scene.RequestModuleInterface<IAssetService>(); |
||
238 | } |
||
239 | |||
240 | //////////////////////////////////////////////////////////// |
||
241 | // IImprovedAssetCache |
||
242 | // |
||
243 | |||
244 | private void UpdateMemoryCache(string key, AssetBase asset) |
||
245 | { |
||
246 | m_MemoryCache.AddOrUpdate(key, asset, m_MemoryExpiration); |
||
247 | } |
||
248 | |||
249 | private void UpdateFileCache(string key, AssetBase asset) |
||
250 | { |
||
251 | string filename = GetFileName(key); |
||
252 | |||
253 | try |
||
254 | { |
||
255 | // If the file is already cached, don't cache it, just touch it so access time is updated |
||
256 | if (File.Exists(filename)) |
||
257 | { |
||
258 | // We don't really want to know about sharing |
||
259 | // violations here. If the file is locked, then |
||
260 | // the other thread has updated the time for us. |
||
261 | try |
||
262 | { |
||
263 | lock (m_CurrentlyWriting) |
||
264 | { |
||
265 | if (!m_CurrentlyWriting.Contains(filename)) |
||
266 | File.SetLastAccessTime(filename, DateTime.Now); |
||
267 | } |
||
268 | } |
||
269 | catch |
||
270 | { |
||
271 | } |
||
272 | } |
||
273 | else |
||
274 | { |
||
275 | // Once we start writing, make sure we flag that we're writing |
||
276 | // that object to the cache so that we don't try to write the |
||
277 | // same file multiple times. |
||
278 | lock (m_CurrentlyWriting) |
||
279 | { |
||
280 | #if WAIT_ON_INPROGRESS_REQUESTS |
||
281 | if (m_CurrentlyWriting.ContainsKey(filename)) |
||
282 | { |
||
283 | return; |
||
284 | } |
||
285 | else |
||
286 | { |
||
287 | m_CurrentlyWriting.Add(filename, new ManualResetEvent(false)); |
||
288 | } |
||
289 | |||
290 | #else |
||
291 | if (m_CurrentlyWriting.Contains(filename)) |
||
292 | { |
||
293 | return; |
||
294 | } |
||
295 | else |
||
296 | { |
||
297 | m_CurrentlyWriting.Add(filename); |
||
298 | } |
||
299 | #endif |
||
300 | |||
301 | } |
||
302 | |||
303 | Util.FireAndForget( |
||
304 | delegate { WriteFileCache(filename, asset); }); |
||
305 | } |
||
306 | } |
||
307 | catch (Exception e) |
||
308 | { |
||
309 | m_log.ErrorFormat( |
||
310 | "[FLOTSAM ASSET CACHE]: Failed to update cache for asset {0}. Exception {1} {2}", |
||
311 | asset.ID, e.Message, e.StackTrace); |
||
312 | } |
||
313 | } |
||
314 | |||
315 | public void Cache(AssetBase asset) |
||
316 | { |
||
317 | // TODO: Spawn this off to some seperate thread to do the actual writing |
||
318 | if (asset != null) |
||
319 | { |
||
320 | //m_log.DebugFormat("[FLOTSAM ASSET CACHE]: Caching asset with id {0}", asset.ID); |
||
321 | |||
322 | if (m_MemoryCacheEnabled) |
||
323 | UpdateMemoryCache(asset.ID, asset); |
||
324 | |||
325 | if (m_FileCacheEnabled) |
||
326 | UpdateFileCache(asset.ID, asset); |
||
327 | } |
||
328 | } |
||
329 | |||
330 | /// <summary> |
||
331 | /// Try to get an asset from the in-memory cache. |
||
332 | /// </summary> |
||
333 | /// <param name="id"></param> |
||
334 | /// <returns></returns> |
||
335 | private AssetBase GetFromMemoryCache(string id) |
||
336 | { |
||
337 | AssetBase asset = null; |
||
338 | |||
339 | if (m_MemoryCache.TryGetValue(id, out asset)) |
||
340 | m_MemoryHits++; |
||
341 | |||
342 | return asset; |
||
343 | } |
||
344 | |||
345 | private bool CheckFromMemoryCache(string id) |
||
346 | { |
||
347 | return m_MemoryCache.Contains(id); |
||
348 | } |
||
349 | |||
350 | /// <summary> |
||
351 | /// Try to get an asset from the file cache. |
||
352 | /// </summary> |
||
353 | /// <param name="id"></param> |
||
354 | /// <returns>An asset retrieved from the file cache. null if there was a problem retrieving an asset.</returns> |
||
355 | private AssetBase GetFromFileCache(string id) |
||
356 | { |
||
357 | string filename = GetFileName(id); |
||
358 | |||
359 | #if WAIT_ON_INPROGRESS_REQUESTS |
||
360 | // Check if we're already downloading this asset. If so, try to wait for it to |
||
361 | // download. |
||
362 | if (m_WaitOnInprogressTimeout > 0) |
||
363 | { |
||
364 | m_RequestsForInprogress++; |
||
365 | |||
366 | ManualResetEvent waitEvent; |
||
367 | if (m_CurrentlyWriting.TryGetValue(filename, out waitEvent)) |
||
368 | { |
||
369 | waitEvent.WaitOne(m_WaitOnInprogressTimeout); |
||
370 | return Get(id); |
||
371 | } |
||
372 | } |
||
373 | #else |
||
374 | // Track how often we have the problem that an asset is requested while |
||
375 | // it is still being downloaded by a previous request. |
||
376 | if (m_CurrentlyWriting.Contains(filename)) |
||
377 | { |
||
378 | m_RequestsForInprogress++; |
||
379 | return null; |
||
380 | } |
||
381 | #endif |
||
382 | |||
383 | AssetBase asset = null; |
||
384 | |||
385 | if (File.Exists(filename)) |
||
386 | { |
||
387 | try |
||
388 | { |
||
389 | using (FileStream stream = File.Open(filename, FileMode.Open, FileAccess.Read, FileShare.Read)) |
||
390 | { |
||
391 | BinaryFormatter bformatter = new BinaryFormatter(); |
||
392 | |||
393 | asset = (AssetBase)bformatter.Deserialize(stream); |
||
394 | |||
395 | m_DiskHits++; |
||
396 | } |
||
397 | } |
||
398 | catch (System.Runtime.Serialization.SerializationException e) |
||
399 | { |
||
400 | m_log.WarnFormat( |
||
401 | "[FLOTSAM ASSET CACHE]: Failed to get file {0} for asset {1}. Exception {2} {3}", |
||
402 | filename, id, e.Message, e.StackTrace); |
||
403 | |||
404 | // If there was a problem deserializing the asset, the asset may |
||
405 | // either be corrupted OR was serialized under an old format |
||
406 | // {different version of AssetBase} -- we should attempt to |
||
407 | // delete it and re-cache |
||
408 | File.Delete(filename); |
||
409 | } |
||
410 | catch (Exception e) |
||
411 | { |
||
412 | m_log.WarnFormat( |
||
413 | "[FLOTSAM ASSET CACHE]: Failed to get file {0} for asset {1}. Exception {2} {3}", |
||
414 | filename, id, e.Message, e.StackTrace); |
||
415 | } |
||
416 | } |
||
417 | |||
418 | return asset; |
||
419 | } |
||
420 | |||
421 | private bool CheckFromFileCache(string id) |
||
422 | { |
||
423 | bool found = false; |
||
424 | |||
425 | string filename = GetFileName(id); |
||
426 | |||
427 | if (File.Exists(filename)) |
||
428 | { |
||
429 | try |
||
430 | { |
||
431 | using (FileStream stream = File.Open(filename, FileMode.Open, FileAccess.Read, FileShare.Read)) |
||
432 | { |
||
433 | if (stream != null) |
||
434 | found = true; |
||
435 | } |
||
436 | } |
||
437 | catch (Exception e) |
||
438 | { |
||
439 | m_log.ErrorFormat( |
||
440 | "[FLOTSAM ASSET CACHE]: Failed to check file {0} for asset {1}. Exception {2} {3}", |
||
441 | filename, id, e.Message, e.StackTrace); |
||
442 | } |
||
443 | } |
||
444 | |||
445 | return found; |
||
446 | } |
||
447 | |||
448 | public AssetBase Get(string id) |
||
449 | { |
||
450 | m_Requests++; |
||
451 | |||
452 | AssetBase asset = null; |
||
453 | |||
454 | if (m_MemoryCacheEnabled) |
||
455 | asset = GetFromMemoryCache(id); |
||
456 | |||
457 | if (asset == null && m_FileCacheEnabled) |
||
458 | { |
||
459 | asset = GetFromFileCache(id); |
||
460 | |||
461 | if (m_MemoryCacheEnabled && asset != null) |
||
462 | UpdateMemoryCache(id, asset); |
||
463 | } |
||
464 | |||
465 | if (((m_LogLevel >= 1)) && (m_HitRateDisplay != 0) && (m_Requests % m_HitRateDisplay == 0)) |
||
466 | { |
||
467 | m_log.InfoFormat("[FLOTSAM ASSET CACHE]: Cache Get :: {0} :: {1}", id, asset == null ? "Miss" : "Hit"); |
||
468 | |||
469 | GenerateCacheHitReport().ForEach(l => m_log.InfoFormat("[FLOTSAM ASSET CACHE]: {0}", l)); |
||
470 | } |
||
471 | |||
472 | return asset; |
||
473 | } |
||
474 | |||
475 | public bool Check(string id) |
||
476 | { |
||
477 | if (m_MemoryCacheEnabled && CheckFromMemoryCache(id)) |
||
478 | return true; |
||
479 | |||
480 | if (m_FileCacheEnabled && CheckFromFileCache(id)) |
||
481 | return true; |
||
482 | return false; |
||
483 | } |
||
484 | |||
485 | public AssetBase GetCached(string id) |
||
486 | { |
||
487 | return Get(id); |
||
488 | } |
||
489 | |||
490 | public void Expire(string id) |
||
491 | { |
||
492 | if (m_LogLevel >= 2) |
||
493 | m_log.DebugFormat("[FLOTSAM ASSET CACHE]: Expiring Asset {0}", id); |
||
494 | |||
495 | try |
||
496 | { |
||
497 | if (m_FileCacheEnabled) |
||
498 | { |
||
499 | string filename = GetFileName(id); |
||
500 | if (File.Exists(filename)) |
||
501 | { |
||
502 | File.Delete(filename); |
||
503 | } |
||
504 | } |
||
505 | |||
506 | if (m_MemoryCacheEnabled) |
||
507 | m_MemoryCache.Remove(id); |
||
508 | } |
||
509 | catch (Exception e) |
||
510 | { |
||
511 | m_log.WarnFormat( |
||
512 | "[FLOTSAM ASSET CACHE]: Failed to expire cached file {0}. Exception {1} {2}", |
||
513 | id, e.Message, e.StackTrace); |
||
514 | } |
||
515 | } |
||
516 | |||
517 | public void Clear() |
||
518 | { |
||
519 | if (m_LogLevel >= 2) |
||
520 | m_log.Debug("[FLOTSAM ASSET CACHE]: Clearing caches."); |
||
521 | |||
522 | if (m_FileCacheEnabled) |
||
523 | { |
||
524 | foreach (string dir in Directory.GetDirectories(m_CacheDirectory)) |
||
525 | { |
||
526 | Directory.Delete(dir); |
||
527 | } |
||
528 | } |
||
529 | |||
530 | if (m_MemoryCacheEnabled) |
||
531 | m_MemoryCache.Clear(); |
||
532 | } |
||
533 | |||
534 | private void CleanupExpiredFiles(object source, ElapsedEventArgs e) |
||
535 | { |
||
536 | if (m_LogLevel >= 2) |
||
537 | m_log.DebugFormat("[FLOTSAM ASSET CACHE]: Checking for expired files older then {0}.", m_FileExpiration); |
||
538 | |||
539 | // Purge all files last accessed prior to this point |
||
540 | DateTime purgeLine = DateTime.Now - m_FileExpiration; |
||
541 | |||
542 | // An asset cache may contain local non-temporary assets that are not in the asset service. Therefore, |
||
543 | // before cleaning up expired files we must scan the objects in the scene to make sure that we retain |
||
544 | // such local assets if they have not been recently accessed. |
||
545 | TouchAllSceneAssets(false); |
||
546 | |||
547 | foreach (string dir in Directory.GetDirectories(m_CacheDirectory)) |
||
548 | { |
||
549 | CleanExpiredFiles(dir, purgeLine); |
||
550 | } |
||
551 | } |
||
552 | |||
553 | /// <summary> |
||
554 | /// Recurses through specified directory checking for asset files last |
||
555 | /// accessed prior to the specified purge line and deletes them. Also |
||
556 | /// removes empty tier directories. |
||
557 | /// </summary> |
||
558 | /// <param name="dir"></param> |
||
559 | /// <param name="purgeLine"></param> |
||
560 | private void CleanExpiredFiles(string dir, DateTime purgeLine) |
||
561 | { |
||
562 | try |
||
563 | { |
||
564 | foreach (string file in Directory.GetFiles(dir)) |
||
565 | { |
||
566 | if (File.GetLastAccessTime(file) < purgeLine) |
||
567 | { |
||
568 | File.Delete(file); |
||
569 | } |
||
570 | } |
||
571 | |||
572 | // Recurse into lower tiers |
||
573 | foreach (string subdir in Directory.GetDirectories(dir)) |
||
574 | { |
||
575 | CleanExpiredFiles(subdir, purgeLine); |
||
576 | } |
||
577 | |||
578 | // Check if a tier directory is empty, if so, delete it |
||
579 | int dirSize = Directory.GetFiles(dir).Length + Directory.GetDirectories(dir).Length; |
||
580 | if (dirSize == 0) |
||
581 | { |
||
582 | Directory.Delete(dir); |
||
583 | } |
||
584 | else if (dirSize >= m_CacheWarnAt) |
||
585 | { |
||
586 | m_log.WarnFormat( |
||
587 | "[FLOTSAM ASSET CACHE]: Cache folder exceeded CacheWarnAt limit {0} {1}. Suggest increasing tiers, tier length, or reducing cache expiration", |
||
588 | dir, dirSize); |
||
589 | } |
||
590 | } |
||
591 | catch (Exception e) |
||
592 | { |
||
593 | m_log.Warn( |
||
594 | string.Format("[FLOTSAM ASSET CACHE]: Could not complete clean of expired files in {0}, exception ", dir), e); |
||
595 | } |
||
596 | } |
||
597 | |||
598 | /// <summary> |
||
599 | /// Determines the filename for an AssetID stored in the file cache |
||
600 | /// </summary> |
||
601 | /// <param name="id"></param> |
||
602 | /// <returns></returns> |
||
603 | private string GetFileName(string id) |
||
604 | { |
||
605 | // Would it be faster to just hash the darn thing? |
||
606 | foreach (char c in m_InvalidChars) |
||
607 | { |
||
608 | id = id.Replace(c, '_'); |
||
609 | } |
||
610 | |||
611 | string path = m_CacheDirectory; |
||
612 | for (int p = 1; p <= m_CacheDirectoryTiers; p++) |
||
613 | { |
||
614 | string pathPart = id.Substring((p - 1) * m_CacheDirectoryTierLen, m_CacheDirectoryTierLen); |
||
615 | path = Path.Combine(path, pathPart); |
||
616 | } |
||
617 | |||
618 | return Path.Combine(path, id); |
||
619 | } |
||
620 | |||
621 | /// <summary> |
||
622 | /// Writes a file to the file cache, creating any nessesary |
||
623 | /// tier directories along the way |
||
624 | /// </summary> |
||
625 | /// <param name="filename"></param> |
||
626 | /// <param name="asset"></param> |
||
627 | private void WriteFileCache(string filename, AssetBase asset) |
||
628 | { |
||
629 | Stream stream = null; |
||
630 | |||
631 | // Make sure the target cache directory exists |
||
632 | string directory = Path.GetDirectoryName(filename); |
||
633 | |||
634 | // Write file first to a temp name, so that it doesn't look |
||
635 | // like it's already cached while it's still writing. |
||
636 | string tempname = Path.Combine(directory, Path.GetRandomFileName()); |
||
637 | |||
638 | try |
||
639 | { |
||
640 | try |
||
641 | { |
||
642 | if (!Directory.Exists(directory)) |
||
643 | { |
||
644 | Directory.CreateDirectory(directory); |
||
645 | } |
||
646 | |||
647 | stream = File.Open(tempname, FileMode.Create); |
||
648 | BinaryFormatter bformatter = new BinaryFormatter(); |
||
649 | bformatter.Serialize(stream, asset); |
||
650 | } |
||
651 | catch (IOException e) |
||
652 | { |
||
653 | m_log.WarnFormat( |
||
654 | "[FLOTSAM ASSET CACHE]: Failed to write asset {0} to temporary location {1} (final {2}) on cache in {3}. Exception {4} {5}.", |
||
655 | asset.ID, tempname, filename, directory, e.Message, e.StackTrace); |
||
656 | |||
657 | return; |
||
658 | } |
||
659 | finally |
||
660 | { |
||
661 | if (stream != null) |
||
662 | stream.Close(); |
||
663 | } |
||
664 | |||
665 | try |
||
666 | { |
||
667 | // Now that it's written, rename it so that it can be found. |
||
668 | // |
||
669 | // File.Copy(tempname, filename, true); |
||
670 | // File.Delete(tempname); |
||
671 | // |
||
672 | // For a brief period, this was done as a separate copy and then temporary file delete operation to |
||
673 | // avoid an IOException caused by move if some competing thread had already written the file. |
||
674 | // However, this causes exceptions on Windows when other threads attempt to read a file |
||
675 | // which is still being copied. So instead, go back to moving the file and swallow any IOException. |
||
676 | // |
||
677 | // This situation occurs fairly rarely anyway. We assume in this that moves are atomic on the |
||
678 | // filesystem. |
||
679 | File.Move(tempname, filename); |
||
680 | |||
681 | if (m_LogLevel >= 2) |
||
682 | m_log.DebugFormat("[FLOTSAM ASSET CACHE]: Cache Stored :: {0}", asset.ID); |
||
683 | } |
||
684 | catch (IOException) |
||
685 | { |
||
686 | // If we see an IOException here it's likely that some other competing thread has written the |
||
687 | // cache file first, so ignore. Other IOException errors (e.g. filesystem full) should be |
||
688 | // signally by the earlier temporary file writing code. |
||
689 | } |
||
690 | } |
||
691 | finally |
||
692 | { |
||
693 | // Even if the write fails with an exception, we need to make sure |
||
694 | // that we release the lock on that file, otherwise it'll never get |
||
695 | // cached |
||
696 | lock (m_CurrentlyWriting) |
||
697 | { |
||
698 | #if WAIT_ON_INPROGRESS_REQUESTS |
||
699 | ManualResetEvent waitEvent; |
||
700 | if (m_CurrentlyWriting.TryGetValue(filename, out waitEvent)) |
||
701 | { |
||
702 | m_CurrentlyWriting.Remove(filename); |
||
703 | waitEvent.Set(); |
||
704 | } |
||
705 | #else |
||
706 | m_CurrentlyWriting.Remove(filename); |
||
707 | #endif |
||
708 | } |
||
709 | } |
||
710 | } |
||
711 | |||
712 | /// <summary> |
||
713 | /// Scan through the file cache, and return number of assets currently cached. |
||
714 | /// </summary> |
||
715 | /// <param name="dir"></param> |
||
716 | /// <returns></returns> |
||
717 | private int GetFileCacheCount(string dir) |
||
718 | { |
||
719 | int count = Directory.GetFiles(dir).Length; |
||
720 | |||
721 | foreach (string subdir in Directory.GetDirectories(dir)) |
||
722 | { |
||
723 | count += GetFileCacheCount(subdir); |
||
724 | } |
||
725 | |||
726 | return count; |
||
727 | } |
||
728 | |||
729 | /// <summary> |
||
730 | /// This notes the last time the Region had a deep asset scan performed on it. |
||
731 | /// </summary> |
||
732 | /// <param name="regionID"></param> |
||
733 | private void StampRegionStatusFile(UUID regionID) |
||
734 | { |
||
735 | string RegionCacheStatusFile = Path.Combine(m_CacheDirectory, "RegionStatus_" + regionID.ToString() + ".fac"); |
||
736 | |||
737 | try |
||
738 | { |
||
739 | if (File.Exists(RegionCacheStatusFile)) |
||
740 | { |
||
741 | File.SetLastWriteTime(RegionCacheStatusFile, DateTime.Now); |
||
742 | } |
||
743 | else |
||
744 | { |
||
745 | File.WriteAllText( |
||
746 | RegionCacheStatusFile, |
||
747 | "Please do not delete this file unless you are manually clearing your Flotsam Asset Cache."); |
||
748 | } |
||
749 | } |
||
750 | catch (Exception e) |
||
751 | { |
||
752 | m_log.Warn( |
||
753 | string.Format( |
||
754 | "[FLOTSAM ASSET CACHE]: Could not stamp region status file for region {0}. Exception ", |
||
755 | regionID), |
||
756 | e); |
||
757 | } |
||
758 | } |
||
759 | |||
760 | /// <summary> |
||
761 | /// Iterates through all Scenes, doing a deep scan through assets |
||
762 | /// to update the access time of all assets present in the scene or referenced by assets |
||
763 | /// in the scene. |
||
764 | /// </summary> |
||
765 | /// <param name="storeUncached"> |
||
766 | /// If true, then assets scanned which are not found in cache are added to the cache. |
||
767 | /// </param> |
||
768 | /// <returns>Number of distinct asset references found in the scene.</returns> |
||
769 | private int TouchAllSceneAssets(bool storeUncached) |
||
770 | { |
||
771 | UuidGatherer gatherer = new UuidGatherer(m_AssetService); |
||
772 | |||
773 | HashSet<UUID> uniqueUuids = new HashSet<UUID>(); |
||
774 | Dictionary<UUID, sbyte> assets = new Dictionary<UUID, sbyte>(); |
||
775 | |||
776 | foreach (Scene s in m_Scenes) |
||
777 | { |
||
778 | StampRegionStatusFile(s.RegionInfo.RegionID); |
||
779 | |||
780 | s.ForEachSOG(delegate(SceneObjectGroup e) |
||
781 | { |
||
782 | gatherer.GatherAssetUuids(e, assets); |
||
783 | |||
784 | foreach (UUID assetID in assets.Keys) |
||
785 | { |
||
786 | uniqueUuids.Add(assetID); |
||
787 | |||
788 | string filename = GetFileName(assetID.ToString()); |
||
789 | |||
790 | if (File.Exists(filename)) |
||
791 | { |
||
792 | File.SetLastAccessTime(filename, DateTime.Now); |
||
793 | } |
||
794 | else if (storeUncached) |
||
795 | { |
||
796 | AssetBase cachedAsset = m_AssetService.Get(assetID.ToString()); |
||
797 | if (cachedAsset == null && assets[assetID] != (sbyte)AssetType.Unknown) |
||
798 | m_log.DebugFormat( |
||
799 | "[FLOTSAM ASSET CACHE]: Could not find asset {0}, type {1} referenced by object {2} at {3} in scene {4} when pre-caching all scene assets", |
||
800 | assetID, assets[assetID], e.Name, e.AbsolutePosition, s.Name); |
||
801 | } |
||
802 | } |
||
803 | |||
804 | assets.Clear(); |
||
805 | }); |
||
806 | } |
||
807 | |||
808 | |||
809 | return uniqueUuids.Count; |
||
810 | } |
||
811 | |||
812 | /// <summary> |
||
813 | /// Deletes all cache contents |
||
814 | /// </summary> |
||
815 | private void ClearFileCache() |
||
816 | { |
||
817 | foreach (string dir in Directory.GetDirectories(m_CacheDirectory)) |
||
818 | { |
||
819 | try |
||
820 | { |
||
821 | Directory.Delete(dir, true); |
||
822 | } |
||
823 | catch (Exception e) |
||
824 | { |
||
825 | m_log.WarnFormat( |
||
826 | "[FLOTSAM ASSET CACHE]: Couldn't clear asset cache directory {0} from {1}. Exception {2} {3}", |
||
827 | dir, m_CacheDirectory, e.Message, e.StackTrace); |
||
828 | } |
||
829 | } |
||
830 | |||
831 | foreach (string file in Directory.GetFiles(m_CacheDirectory)) |
||
832 | { |
||
833 | try |
||
834 | { |
||
835 | File.Delete(file); |
||
836 | } |
||
837 | catch (Exception e) |
||
838 | { |
||
839 | m_log.WarnFormat( |
||
840 | "[FLOTSAM ASSET CACHE]: Couldn't clear asset cache file {0} from {1}. Exception {1} {2}", |
||
841 | file, m_CacheDirectory, e.Message, e.StackTrace); |
||
842 | } |
||
843 | } |
||
844 | } |
||
845 | |||
846 | private List<string> GenerateCacheHitReport() |
||
847 | { |
||
848 | List<string> outputLines = new List<string>(); |
||
849 | |||
850 | double fileHitRate = (double)m_DiskHits / m_Requests * 100.0; |
||
851 | outputLines.Add( |
||
852 | string.Format("File Hit Rate: {0}% for {1} requests", fileHitRate.ToString("0.00"), m_Requests)); |
||
853 | |||
854 | if (m_MemoryCacheEnabled) |
||
855 | { |
||
856 | double memHitRate = (double)m_MemoryHits / m_Requests * 100.0; |
||
857 | |||
858 | outputLines.Add( |
||
859 | string.Format("Memory Hit Rate: {0}% for {1} requests", memHitRate.ToString("0.00"), m_Requests)); |
||
860 | } |
||
861 | |||
862 | outputLines.Add( |
||
863 | string.Format( |
||
864 | "Unnecessary requests due to requests for assets that are currently downloading: {0}", |
||
865 | m_RequestsForInprogress)); |
||
866 | |||
867 | return outputLines; |
||
868 | } |
||
869 | |||
870 | #region Console Commands |
||
871 | private void HandleConsoleCommand(string module, string[] cmdparams) |
||
872 | { |
||
873 | ICommandConsole con = MainConsole.Instance; |
||
874 | |||
875 | if (cmdparams.Length >= 2) |
||
876 | { |
||
877 | string cmd = cmdparams[1]; |
||
878 | |||
879 | switch (cmd) |
||
880 | { |
||
881 | case "status": |
||
882 | if (m_MemoryCacheEnabled) |
||
883 | con.OutputFormat("Memory Cache: {0} assets", m_MemoryCache.Count); |
||
884 | else |
||
885 | con.OutputFormat("Memory cache disabled"); |
||
886 | |||
887 | if (m_FileCacheEnabled) |
||
888 | { |
||
889 | int fileCount = GetFileCacheCount(m_CacheDirectory); |
||
890 | con.OutputFormat("File Cache: {0} assets", fileCount); |
||
891 | } |
||
892 | else |
||
893 | { |
||
894 | con.Output("File cache disabled"); |
||
895 | } |
||
896 | |||
897 | GenerateCacheHitReport().ForEach(l => con.Output(l)); |
||
898 | |||
899 | if (m_FileCacheEnabled) |
||
900 | { |
||
901 | con.Output("Deep scans have previously been performed on the following regions:"); |
||
902 | |||
903 | foreach (string s in Directory.GetFiles(m_CacheDirectory, "*.fac")) |
||
904 | { |
||
905 | string RegionID = s.Remove(0,s.IndexOf("_")).Replace(".fac",""); |
||
906 | DateTime RegionDeepScanTMStamp = File.GetLastWriteTime(s); |
||
907 | con.OutputFormat("Region: {0}, {1}", RegionID, RegionDeepScanTMStamp.ToString("MM/dd/yyyy hh:mm:ss")); |
||
908 | } |
||
909 | } |
||
910 | |||
911 | break; |
||
912 | |||
913 | case "clear": |
||
914 | if (cmdparams.Length < 2) |
||
915 | { |
||
916 | con.Output("Usage is fcache clear [file] [memory]"); |
||
917 | break; |
||
918 | } |
||
919 | |||
920 | bool clearMemory = false, clearFile = false; |
||
921 | |||
922 | if (cmdparams.Length == 2) |
||
923 | { |
||
924 | clearMemory = true; |
||
925 | clearFile = true; |
||
926 | } |
||
927 | foreach (string s in cmdparams) |
||
928 | { |
||
929 | if (s.ToLower() == "memory") |
||
930 | clearMemory = true; |
||
931 | else if (s.ToLower() == "file") |
||
932 | clearFile = true; |
||
933 | } |
||
934 | |||
935 | if (clearMemory) |
||
936 | { |
||
937 | if (m_MemoryCacheEnabled) |
||
938 | { |
||
939 | m_MemoryCache.Clear(); |
||
940 | con.Output("Memory cache cleared."); |
||
941 | } |
||
942 | else |
||
943 | { |
||
944 | con.Output("Memory cache not enabled."); |
||
945 | } |
||
946 | } |
||
947 | |||
948 | if (clearFile) |
||
949 | { |
||
950 | if (m_FileCacheEnabled) |
||
951 | { |
||
952 | ClearFileCache(); |
||
953 | con.Output("File cache cleared."); |
||
954 | } |
||
955 | else |
||
956 | { |
||
957 | con.Output("File cache not enabled."); |
||
958 | } |
||
959 | } |
||
960 | |||
961 | break; |
||
962 | |||
963 | case "assets": |
||
964 | con.Output("Ensuring assets are cached for all scenes."); |
||
965 | |||
966 | Util.FireAndForget(delegate { |
||
967 | int assetReferenceTotal = TouchAllSceneAssets(true); |
||
968 | con.OutputFormat("Completed check with {0} assets.", assetReferenceTotal); |
||
969 | }); |
||
970 | |||
971 | break; |
||
972 | |||
973 | case "expire": |
||
974 | if (cmdparams.Length < 3) |
||
975 | { |
||
976 | con.OutputFormat("Invalid parameters for Expire, please specify a valid date & time", cmd); |
||
977 | break; |
||
978 | } |
||
979 | |||
980 | string s_expirationDate = ""; |
||
981 | DateTime expirationDate; |
||
982 | |||
983 | if (cmdparams.Length > 3) |
||
984 | { |
||
985 | s_expirationDate = string.Join(" ", cmdparams, 2, cmdparams.Length - 2); |
||
986 | } |
||
987 | else |
||
988 | { |
||
989 | s_expirationDate = cmdparams[2]; |
||
990 | } |
||
991 | |||
992 | if (!DateTime.TryParse(s_expirationDate, out expirationDate)) |
||
993 | { |
||
994 | con.OutputFormat("{0} is not a valid date & time", cmd); |
||
995 | break; |
||
996 | } |
||
997 | |||
998 | if (m_FileCacheEnabled) |
||
999 | CleanExpiredFiles(m_CacheDirectory, expirationDate); |
||
1000 | else |
||
1001 | con.OutputFormat("File cache not active, not clearing."); |
||
1002 | |||
1003 | break; |
||
1004 | default: |
||
1005 | con.OutputFormat("Unknown command {0}", cmd); |
||
1006 | break; |
||
1007 | } |
||
1008 | } |
||
1009 | else if (cmdparams.Length == 1) |
||
1010 | { |
||
1011 | con.Output("fcache assets - Attempt a deep cache of all assets in all scenes"); |
||
1012 | con.Output("fcache expire <datetime> - Purge assets older then the specified date & time"); |
||
1013 | con.Output("fcache clear [file] [memory] - Remove cached assets"); |
||
1014 | con.Output("fcache status - Display cache status"); |
||
1015 | } |
||
1016 | } |
||
1017 | |||
1018 | #endregion |
||
1019 | |||
1020 | #region IAssetService Members |
||
1021 | |||
1022 | public AssetMetadata GetMetadata(string id) |
||
1023 | { |
||
1024 | AssetBase asset = Get(id); |
||
1025 | return asset.Metadata; |
||
1026 | } |
||
1027 | |||
1028 | public byte[] GetData(string id) |
||
1029 | { |
||
1030 | AssetBase asset = Get(id); |
||
1031 | return asset.Data; |
||
1032 | } |
||
1033 | |||
1034 | public bool Get(string id, object sender, AssetRetrieved handler) |
||
1035 | { |
||
1036 | AssetBase asset = Get(id); |
||
1037 | handler(id, sender, asset); |
||
1038 | return true; |
||
1039 | } |
||
1040 | |||
1041 | public string Store(AssetBase asset) |
||
1042 | { |
||
1043 | if (asset.FullID == UUID.Zero) |
||
1044 | { |
||
1045 | asset.FullID = UUID.Random(); |
||
1046 | } |
||
1047 | |||
1048 | Cache(asset); |
||
1049 | |||
1050 | return asset.ID; |
||
1051 | } |
||
1052 | |||
1053 | public bool UpdateContent(string id, byte[] data) |
||
1054 | { |
||
1055 | AssetBase asset = Get(id); |
||
1056 | asset.Data = data; |
||
1057 | Cache(asset); |
||
1058 | return true; |
||
1059 | } |
||
1060 | |||
1061 | public bool Delete(string id) |
||
1062 | { |
||
1063 | Expire(id); |
||
1064 | return true; |
||
1065 | } |
||
1066 | |||
1067 | #endregion |
||
1068 | } |
||
1069 | } |