clockwerk-opensim – Blame information for rev 1

Subversion Repositories:
Rev:
Rev Author Line No. Line
1 vero 1 /*
2 * Copyright (c) Contributors, http://opensimulator.org/
3 * See CONTRIBUTORS.TXT for a full list of copyright holders.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 * * Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * * Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 * * Neither the name of the OpenSimulator Project nor the
13 * names of its contributors may be used to endorse or promote products
14 * derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY
17 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY
20 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
23 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27  
28 using System;
29 using System.Collections.Generic;
30 using System.Data;
31 using System.IO;
32 using System.IO.Compression;
33 using System.Reflection;
34 using System.Security.Cryptography;
35 using System.Text;
36 using log4net;
37 using OpenMetaverse;
38 using OpenSim.Framework;
39 using OpenSim.Data;
40 using Npgsql;
41  
42 namespace OpenSim.Data.PGSQL
43 {
44 public class PGSQLXAssetData : IXAssetDataPlugin
45 {
46 private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
47  
48 protected virtual Assembly Assembly
49 {
50 get { return GetType().Assembly; }
51 }
52  
53 /// <summary>
54 /// Number of days that must pass before we update the access time on an asset when it has been fetched.
55 /// </summary>
56 private const int DaysBetweenAccessTimeUpdates = 30;
57  
58 private bool m_enableCompression = false;
59 private string m_connectionString;
60 private object m_dbLock = new object();
61  
62 /// <summary>
63 /// We can reuse this for all hashing since all methods are single-threaded through m_dbBLock
64 /// </summary>
65 private HashAlgorithm hasher = new SHA256CryptoServiceProvider();
66  
67 #region IPlugin Members
68  
69 public string Version { get { return "1.0.0.0"; } }
70  
71 /// <summary>
72 /// <para>Initialises Asset interface</para>
73 /// <para>
74 /// <list type="bullet">
75 /// <item>Loads and initialises the PGSQL storage plugin.</item>
76 /// <item>Warns and uses the obsolete pgsql_connection.ini if connect string is empty.</item>
77 /// <item>Check for migration</item>
78 /// </list>
79 /// </para>
80 /// </summary>
81 /// <param name="connect">connect string</param>
82 public void Initialise(string connect)
83 {
84 m_log.ErrorFormat("[PGSQL XASSETDATA]: ***********************************************************");
85 m_log.ErrorFormat("[PGSQL XASSETDATA]: ***********************************************************");
86 m_log.ErrorFormat("[PGSQL XASSETDATA]: ***********************************************************");
87 m_log.ErrorFormat("[PGSQL XASSETDATA]: THIS PLUGIN IS STRICTLY EXPERIMENTAL.");
88 m_log.ErrorFormat("[PGSQL XASSETDATA]: DO NOT USE FOR ANY DATA THAT YOU DO NOT MIND LOSING.");
89 m_log.ErrorFormat("[PGSQL XASSETDATA]: DATABASE TABLES CAN CHANGE AT ANY TIME, CAUSING EXISTING DATA TO BE LOST.");
90 m_log.ErrorFormat("[PGSQL XASSETDATA]: ***********************************************************");
91 m_log.ErrorFormat("[PGSQL XASSETDATA]: ***********************************************************");
92 m_log.ErrorFormat("[PGSQL XASSETDATA]: ***********************************************************");
93  
94 m_connectionString = connect;
95  
96 using (NpgsqlConnection dbcon = new NpgsqlConnection(m_connectionString))
97 {
98 dbcon.Open();
99 Migration m = new Migration(dbcon, Assembly, "XAssetStore");
100 m.Update();
101 }
102 }
103  
104 public void Initialise()
105 {
106 throw new NotImplementedException();
107 }
108  
109 public void Dispose() { }
110  
111 /// <summary>
112 /// The name of this DB provider
113 /// </summary>
114 public string Name
115 {
116 get { return "PGSQL XAsset storage engine"; }
117 }
118  
119 #endregion
120  
121 #region IAssetDataPlugin Members
122  
123 /// <summary>
124 /// Fetch Asset <paramref name="assetID"/> from database
125 /// </summary>
126 /// <param name="assetID">Asset UUID to fetch</param>
127 /// <returns>Return the asset</returns>
128 /// <remarks>On failure : throw an exception and attempt to reconnect to database</remarks>
129 public AssetBase GetAsset(UUID assetID)
130 {
131 // m_log.DebugFormat("[PGSQL XASSET DATA]: Looking for asset {0}", assetID);
132  
133 AssetBase asset = null;
134 lock (m_dbLock)
135 {
136 using (NpgsqlConnection dbcon = new NpgsqlConnection(m_connectionString))
137 {
138 dbcon.Open();
139  
140 using (NpgsqlCommand cmd = new NpgsqlCommand(
141 @"SELECT ""Name"", ""Description"", ""AccessTime"", ""AssetType"", ""Local"", ""Temporary"", ""AssetFlags"", ""CreatorID"", ""Data""
142 FROM XAssetsMeta
143 JOIN XAssetsData ON XAssetsMeta.Hash = XAssetsData.Hash WHERE ""ID""=:ID",
144 dbcon))
145 {
146 cmd.Parameters.AddWithValue("ID", assetID.ToString());
147  
148 try
149 {
150 using (NpgsqlDataReader dbReader = cmd.ExecuteReader(CommandBehavior.SingleRow))
151 {
152 if (dbReader.Read())
153 {
154 asset = new AssetBase(assetID, (string)dbReader["Name"], (sbyte)dbReader["AssetType"], dbReader["CreatorID"].ToString());
155 asset.Data = (byte[])dbReader["Data"];
156 asset.Description = (string)dbReader["Description"];
157  
158 string local = dbReader["Local"].ToString();
159 if (local.Equals("1") || local.Equals("true", StringComparison.InvariantCultureIgnoreCase))
160 asset.Local = true;
161 else
162 asset.Local = false;
163  
164 asset.Temporary = Convert.ToBoolean(dbReader["Temporary"]);
165 asset.Flags = (AssetFlags)Convert.ToInt32(dbReader["AssetFlags"]);
166  
167 if (m_enableCompression)
168 {
169 using (GZipStream decompressionStream = new GZipStream(new MemoryStream(asset.Data), CompressionMode.Decompress))
170 {
171 MemoryStream outputStream = new MemoryStream();
172 WebUtil.CopyStream(decompressionStream, outputStream, int.MaxValue);
173 // int compressedLength = asset.Data.Length;
174 asset.Data = outputStream.ToArray();
175  
176 // m_log.DebugFormat(
177 // "[XASSET DB]: Decompressed {0} {1} to {2} bytes from {3}",
178 // asset.ID, asset.Name, asset.Data.Length, compressedLength);
179 }
180 }
181  
182 UpdateAccessTime(asset.Metadata, (int)dbReader["AccessTime"]);
183 }
184 }
185 }
186 catch (Exception e)
187 {
188 m_log.Error(string.Format("[PGSQL XASSET DATA]: Failure fetching asset {0}", assetID), e);
189 }
190 }
191 }
192 }
193  
194 return asset;
195 }
196  
197 /// <summary>
198 /// Create an asset in database, or update it if existing.
199 /// </summary>
200 /// <param name="asset">Asset UUID to create</param>
201 /// <remarks>On failure : Throw an exception and attempt to reconnect to database</remarks>
202 public void StoreAsset(AssetBase asset)
203 {
204 // m_log.DebugFormat("[XASSETS DB]: Storing asset {0} {1}", asset.Name, asset.ID);
205  
206 lock (m_dbLock)
207 {
208 using (NpgsqlConnection dbcon = new NpgsqlConnection(m_connectionString))
209 {
210 dbcon.Open();
211  
212 using (NpgsqlTransaction transaction = dbcon.BeginTransaction())
213 {
214 string assetName = asset.Name;
215 if (asset.Name.Length > 64)
216 {
217 assetName = asset.Name.Substring(0, 64);
218 m_log.WarnFormat(
219 "[XASSET DB]: Name '{0}' for asset {1} truncated from {2} to {3} characters on add",
220 asset.Name, asset.ID, asset.Name.Length, assetName.Length);
221 }
222  
223 string assetDescription = asset.Description;
224 if (asset.Description.Length > 64)
225 {
226 assetDescription = asset.Description.Substring(0, 64);
227 m_log.WarnFormat(
228 "[XASSET DB]: Description '{0}' for asset {1} truncated from {2} to {3} characters on add",
229 asset.Description, asset.ID, asset.Description.Length, assetDescription.Length);
230 }
231  
232 if (m_enableCompression)
233 {
234 MemoryStream outputStream = new MemoryStream();
235  
236 using (GZipStream compressionStream = new GZipStream(outputStream, CompressionMode.Compress, false))
237 {
238 // Console.WriteLine(WebUtil.CopyTo(new MemoryStream(asset.Data), compressionStream, int.MaxValue));
239 // We have to close the compression stream in order to make sure it writes everything out to the underlying memory output stream.
240 compressionStream.Close();
241 byte[] compressedData = outputStream.ToArray();
242 asset.Data = compressedData;
243 }
244 }
245  
246 byte[] hash = hasher.ComputeHash(asset.Data);
247  
248 // m_log.DebugFormat(
249 // "[XASSET DB]: Compressed data size for {0} {1}, hash {2} is {3}",
250 // asset.ID, asset.Name, hash, compressedData.Length);
251  
252 try
253 {
254 using (NpgsqlCommand cmd =
255 new NpgsqlCommand(
256 @"insert INTO XAssetsMeta(""ID"", ""Hash"", ""Name"", ""Description"", ""AssetType"", ""Local"", ""Temporary"", ""CreateTime"", ""AccessTime"", ""AssetFlags"", ""CreatorID"")
257 Select :ID, :Hash, :Name, :Description, :AssetType, :Local, :Temporary, :CreateTime, :AccessTime, :AssetFlags, :CreatorID
258 where not exists( Select ""ID"" from XAssetsMeta where ""ID"" = :ID ;
259  
260 update XAssetsMeta
261 set ""ID"" = :ID, ""Hash"" = :Hash, ""Name"" = :Name, ""Description"" = :Description,
262 ""AssetType"" = :AssetType, ""Local"" = :Local, ""Temporary"" = :Temporary, ""CreateTime"" = :CreateTime,
263 ""AccessTime"" = :AccessTime, ""AssetFlags"" = :AssetFlags, ""CreatorID"" = :CreatorID
264 where ""ID"" = :ID;
265 ",
266 dbcon))
267 {
268 // create unix epoch time
269 int now = (int)Utils.DateTimeToUnixTime(DateTime.UtcNow);
270 cmd.Parameters.AddWithValue("ID", asset.ID);
271 cmd.Parameters.AddWithValue("Hash", hash);
272 cmd.Parameters.AddWithValue("Name", assetName);
273 cmd.Parameters.AddWithValue("Description", assetDescription);
274 cmd.Parameters.AddWithValue("AssetType", asset.Type);
275 cmd.Parameters.AddWithValue("Local", asset.Local);
276 cmd.Parameters.AddWithValue("Temporary", asset.Temporary);
277 cmd.Parameters.AddWithValue("CreateTime", now);
278 cmd.Parameters.AddWithValue("AccessTime", now);
279 cmd.Parameters.AddWithValue("CreatorID", asset.Metadata.CreatorID);
280 cmd.Parameters.AddWithValue("AssetFlags", (int)asset.Flags);
281 cmd.ExecuteNonQuery();
282 }
283 }
284 catch (Exception e)
285 {
286 m_log.ErrorFormat("[ASSET DB]: PGSQL failure creating asset metadata {0} with name \"{1}\". Error: {2}",
287 asset.FullID, asset.Name, e.Message);
288  
289 transaction.Rollback();
290  
291 return;
292 }
293  
294 if (!ExistsData(dbcon, transaction, hash))
295 {
296 try
297 {
298 using (NpgsqlCommand cmd =
299 new NpgsqlCommand(
300 @"INSERT INTO XAssetsData(""Hash"", ""Data"") VALUES(:Hash, :Data)",
301 dbcon))
302 {
303 cmd.Parameters.AddWithValue("Hash", hash);
304 cmd.Parameters.AddWithValue("Data", asset.Data);
305 cmd.ExecuteNonQuery();
306 }
307 }
308 catch (Exception e)
309 {
310 m_log.ErrorFormat("[XASSET DB]: PGSQL failure creating asset data {0} with name \"{1}\". Error: {2}",
311 asset.FullID, asset.Name, e.Message);
312  
313 transaction.Rollback();
314  
315 return;
316 }
317 }
318  
319 transaction.Commit();
320 }
321 }
322 }
323 }
324  
325 /// <summary>
326 /// Updates the access time of the asset if it was accessed above a given threshhold amount of time.
327 /// </summary>
328 /// <remarks>
329 /// This gives us some insight into assets which haven't ben accessed for a long period. This is only done
330 /// over the threshold time to avoid excessive database writes as assets are fetched.
331 /// </remarks>
332 /// <param name='asset'></param>
333 /// <param name='accessTime'></param>
334 private void UpdateAccessTime(AssetMetadata assetMetadata, int accessTime)
335 {
336 DateTime now = DateTime.UtcNow;
337  
338 if ((now - Utils.UnixTimeToDateTime(accessTime)).TotalDays < DaysBetweenAccessTimeUpdates)
339 return;
340  
341 lock (m_dbLock)
342 {
343 using (NpgsqlConnection dbcon = new NpgsqlConnection(m_connectionString))
344 {
345 dbcon.Open();
346 NpgsqlCommand cmd =
347 new NpgsqlCommand(@"update XAssetsMeta set ""AccessTime""=:AccessTime where ID=:ID", dbcon);
348  
349 try
350 {
351 using (cmd)
352 {
353 // create unix epoch time
354 cmd.Parameters.AddWithValue("ID", assetMetadata.ID);
355 cmd.Parameters.AddWithValue("AccessTime", (int)Utils.DateTimeToUnixTime(now));
356 cmd.ExecuteNonQuery();
357 }
358 }
359 catch (Exception e)
360 {
361 m_log.ErrorFormat(
362 "[XASSET PGSQL DB]: Failure updating access_time for asset {0} with name {1} : {2}",
363 assetMetadata.ID, assetMetadata.Name, e.Message);
364 }
365 }
366 }
367 }
368  
369 /// <summary>
370 /// We assume we already have the m_dbLock.
371 /// </summary>
372 /// TODO: need to actually use the transaction.
373 /// <param name="dbcon"></param>
374 /// <param name="transaction"></param>
375 /// <param name="hash"></param>
376 /// <returns></returns>
377 private bool ExistsData(NpgsqlConnection dbcon, NpgsqlTransaction transaction, byte[] hash)
378 {
379 // m_log.DebugFormat("[ASSETS DB]: Checking for asset {0}", uuid);
380  
381 bool exists = false;
382  
383 using (NpgsqlCommand cmd = new NpgsqlCommand(@"SELECT ""Hash"" FROM XAssetsData WHERE ""Hash""=:Hash", dbcon))
384 {
385 cmd.Parameters.AddWithValue("Hash", hash);
386  
387 try
388 {
389 using (NpgsqlDataReader dbReader = cmd.ExecuteReader(CommandBehavior.SingleRow))
390 {
391 if (dbReader.Read())
392 {
393 // m_log.DebugFormat("[ASSETS DB]: Found asset {0}", uuid);
394 exists = true;
395 }
396 }
397 }
398 catch (Exception e)
399 {
400 m_log.ErrorFormat(
401 "[XASSETS DB]: PGSql failure in ExistsData fetching hash {0}. Exception {1}{2}",
402 hash, e.Message, e.StackTrace);
403 }
404 }
405  
406 return exists;
407 }
408  
409 /// <summary>
410 /// Check if the assets exist in the database.
411 /// </summary>
412 /// <param name="uuids">The assets' IDs</param>
413 /// <returns>For each asset: true if it exists, false otherwise</returns>
414 public bool[] AssetsExist(UUID[] uuids)
415 {
416 if (uuids.Length == 0)
417 return new bool[0];
418  
419 HashSet<UUID> exist = new HashSet<UUID>();
420  
421 string ids = "'" + string.Join("','", uuids) + "'";
422 string sql = string.Format(@"SELECT ""ID"" FROM XAssetsMeta WHERE ""ID"" IN ({0})", ids);
423  
424 using (NpgsqlConnection conn = new NpgsqlConnection(m_connectionString))
425 {
426 conn.Open();
427 using (NpgsqlCommand cmd = new NpgsqlCommand(sql, conn))
428 {
429 using (NpgsqlDataReader reader = cmd.ExecuteReader())
430 {
431 while (reader.Read())
432 {
433 UUID id = DBGuid.FromDB(reader["id"]);
434 exist.Add(id);
435 }
436 }
437 }
438 }
439  
440 bool[] results = new bool[uuids.Length];
441 for (int i = 0; i < uuids.Length; i++)
442 results[i] = exist.Contains(uuids[i]);
443 return results;
444 }
445  
446 /// <summary>
447 /// Check if the asset exists in the database
448 /// </summary>
449 /// <param name="uuid">The asset UUID</param>
450 /// <returns>true if it exists, false otherwise.</returns>
451 public bool ExistsAsset(UUID uuid)
452 {
453 // m_log.DebugFormat("[ASSETS DB]: Checking for asset {0}", uuid);
454  
455 bool assetExists = false;
456  
457 lock (m_dbLock)
458 {
459 using (NpgsqlConnection dbcon = new NpgsqlConnection(m_connectionString))
460 {
461 dbcon.Open();
462 using (NpgsqlCommand cmd = new NpgsqlCommand(@"SELECT ""ID"" FROM XAssetsMeta WHERE ""ID""=:ID", dbcon))
463 {
464 cmd.Parameters.AddWithValue("ID", uuid.ToString());
465  
466 try
467 {
468 using (NpgsqlDataReader dbReader = cmd.ExecuteReader(CommandBehavior.SingleRow))
469 {
470 if (dbReader.Read())
471 {
472 // m_log.DebugFormat("[ASSETS DB]: Found asset {0}", uuid);
473 assetExists = true;
474 }
475 }
476 }
477 catch (Exception e)
478 {
479 m_log.Error(string.Format("[XASSETS DB]: PGSql failure fetching asset {0}", uuid), e);
480 }
481 }
482 }
483 }
484  
485 return assetExists;
486 }
487  
488  
489 /// <summary>
490 /// Returns a list of AssetMetadata objects. The list is a subset of
491 /// the entire data set offset by <paramref name="start" /> containing
492 /// <paramref name="count" /> elements.
493 /// </summary>
494 /// <param name="start">The number of results to discard from the total data set.</param>
495 /// <param name="count">The number of rows the returned list should contain.</param>
496 /// <returns>A list of AssetMetadata objects.</returns>
497 public List<AssetMetadata> FetchAssetMetadataSet(int start, int count)
498 {
499 List<AssetMetadata> retList = new List<AssetMetadata>(count);
500  
501 lock (m_dbLock)
502 {
503 using (NpgsqlConnection dbcon = new NpgsqlConnection(m_connectionString))
504 {
505 dbcon.Open();
506 NpgsqlCommand cmd = new NpgsqlCommand( @"SELECT ""Name"", ""Description"", ""AccessTime"", ""AssetType"", ""Temporary"", ""ID"", ""AssetFlags"", ""CreatorID""
507 FROM XAssetsMeta
508 LIMIT :start, :count", dbcon);
509 cmd.Parameters.AddWithValue("start", start);
510 cmd.Parameters.AddWithValue("count", count);
511  
512 try
513 {
514 using (NpgsqlDataReader dbReader = cmd.ExecuteReader())
515 {
516 while (dbReader.Read())
517 {
518 AssetMetadata metadata = new AssetMetadata();
519 metadata.Name = (string)dbReader["Name"];
520 metadata.Description = (string)dbReader["Description"];
521 metadata.Type = (sbyte)dbReader["AssetType"];
522 metadata.Temporary = Convert.ToBoolean(dbReader["Temporary"]); // Not sure if this is correct.
523 metadata.Flags = (AssetFlags)Convert.ToInt32(dbReader["AssetFlags"]);
524 metadata.FullID = DBGuid.FromDB(dbReader["ID"]);
525 metadata.CreatorID = dbReader["CreatorID"].ToString();
526  
527 // We'll ignore this for now - it appears unused!
528 // metadata.SHA1 = dbReader["hash"]);
529  
530 UpdateAccessTime(metadata, (int)dbReader["AccessTime"]);
531  
532 retList.Add(metadata);
533 }
534 }
535 }
536 catch (Exception e)
537 {
538 m_log.Error("[XASSETS DB]: PGSql failure fetching asset set" + Environment.NewLine + e.ToString());
539 }
540 }
541 }
542  
543 return retList;
544 }
545  
546 public bool Delete(string id)
547 {
548 // m_log.DebugFormat("[XASSETS DB]: Deleting asset {0}", id);
549  
550 lock (m_dbLock)
551 {
552 using (NpgsqlConnection dbcon = new NpgsqlConnection(m_connectionString))
553 {
554 dbcon.Open();
555  
556 using (NpgsqlCommand cmd = new NpgsqlCommand(@"delete from XAssetsMeta where ""ID""=:ID", dbcon))
557 {
558 cmd.Parameters.AddWithValue("ID", id);
559 cmd.ExecuteNonQuery();
560 }
561  
562 // TODO: How do we deal with data from deleted assets? Probably not easily reapable unless we
563 // keep a reference count (?)
564 }
565 }
566  
567 return true;
568 }
569  
570 #endregion
571 }
572 }