kapsikkum-unmanic – Blame information for rev 1

Subversion Repositories:
Rev:
Rev Author Line No. Line
1 office 1 #!/usr/bin/env python3
2 # -*- coding: utf-8 -*-
3  
4 """
5 unmanic.filetest.py
6  
7 Written by: Josh.5 <jsunnex@gmail.com>
8 Date: 28 Mar 2021, (7:28 PM)
9  
10 Copyright:
11 Copyright (C) Josh Sunnex - All Rights Reserved
12  
13 Permission is hereby granted, free of charge, to any person obtaining a copy
14 of this software and associated documentation files (the "Software"), to deal
15 in the Software without restriction, including without limitation the rights
16 to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
17 copies of the Software, and to permit persons to whom the Software is
18 furnished to do so, subject to the following conditions:
19  
20 The above copyright notice and this permission notice shall be included in all
21 copies or substantial portions of the Software.
22  
23 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
26 IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
27 DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
28 OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
29 OR OTHER DEALINGS IN THE SOFTWARE.
30  
31 """
32 import os
33 import queue
34 import threading
35 import time
36 from copy import deepcopy
37  
38 from unmanic import config
39 from unmanic.libs import history, common, unlogger
40 from unmanic.libs.plugins import PluginsHandler
41  
42  
43 class FileTest(object):
44 """
45 FileTest
46  
47 Object to manage tests carried out on files discovered
48 during a library scan or inode event
49  
50 """
51  
52 def __init__(self, library_id: int):
53 self.settings = config.Config()
54 unmanic_logging = unlogger.UnmanicLogger.__call__()
55 self.logger = unmanic_logging.get_logger(__class__.__name__)
56  
57 # Init plugins
58 self.library_id = library_id
59 self.plugin_handler = PluginsHandler()
60 self.plugin_modules = self.plugin_handler.get_enabled_plugin_modules_by_type('library_management.file_test',
61 library_id=library_id)
62  
63 # List of filed tasks
64 self.failed_paths = []
65  
66 def _log(self, message, message2='', level="info"):
67 message = common.format_message(message, message2)
68 getattr(self.logger, level)(message)
69  
70 def set_file(self):
71 pass
72  
73 def file_failed_in_history(self, path):
74 """
75 Check if file has already failed in history
76  
77 :return:
78 """
79 # Fetch historical tasks
80 history_logging = history.History()
81 if not self.failed_paths:
82 failed_tasks = history_logging.get_historic_tasks_list_with_source_probe(task_success=False)
83 for task in failed_tasks:
84 self.failed_paths.append(task.get('abspath'))
85 if path in self.failed_paths:
86 # That pathname was found in the results of failed historic tasks
87 return True
88 # No results were found matching that pathname
89 return False
90  
91 def file_in_unmanic_ignore_lockfile(self, path):
92 """
93 Check if folder contains a '.unmanicignore' lockfile
94  
95 :return:
96 """
97 # Get file parent directory
98 dirname = os.path.dirname(path)
99 # Check if lockfile (.unmanicignore) exists
100 unmanic_ignore_file = os.path.join(dirname, '.unmanicignore')
101 if os.path.exists(unmanic_ignore_file):
102 # Get file basename
103 basename = os.path.basename(path)
104 # Read the file and check for any entry with this file name
105 with open(unmanic_ignore_file) as f:
106 for line in f:
107 if basename in line:
108 return True
109 return False
110  
111 def should_file_be_added_to_task_list(self, path):
112 """
113 Test if this file needs to be added to the task list
114  
115 :return:
116 """
117 return_value = None
118 file_issues = []
119  
120 # TODO: Remove this
121 if self.file_in_unmanic_ignore_lockfile(path):
122 file_issues.append({
123 'id': 'unmanicignore',
124 'message': "File found in unmanic ignore file - '{}'".format(path),
125 })
126 return_value = False
127  
128 # Check if file has failed in history.
129 if self.file_failed_in_history(path):
130 file_issues.append({
131 'id': 'blacklisted',
132 'message': "File found already failed in history - '{}'".format(path),
133 })
134 return_value = False
135  
136 # Only run checks with plugins if other tests were not conclusive
137 priority_score_modification = 0
138 if return_value is None:
139 # Set the initial data with just the priority score.
140 data = {
141 'priority_score': 0,
142 'shared_info': {},
143 }
144 # Run tests against plugins
145 for plugin_module in self.plugin_modules:
146 data['library_id'] = self.library_id
147 data['path'] = path
148 data['issues'] = deepcopy(file_issues)
149 data['add_file_to_pending_tasks'] = None
150  
151 # Run plugin to update data
152 if not self.plugin_handler.exec_plugin_runner(data, plugin_module.get('plugin_id'),
153 'library_management.file_test'):
154 continue
155  
156 # Append any file issues found during previous tests
157 file_issues = data.get('issues')
158  
159 # Set the return_value based on the plugin results
160 # If the add_file_to_pending_tasks returned an answer (True/False) then break the loop.
161 # No need to continue.
162 if data.get('add_file_to_pending_tasks') is not None:
163 return_value = data.get('add_file_to_pending_tasks')
164 break
165 # Set the priority score modification
166 priority_score_modification = data.get('priority_score', 0)
167  
168 return return_value, file_issues, priority_score_modification
169  
170  
171 class FileTesterThread(threading.Thread):
172 def __init__(self, name, files_to_test, files_to_process, status_updates, library_id, event):
173 super(FileTesterThread, self).__init__(name=name)
174 self.settings = config.Config()
175 self.logger = None
176 self.event = event
177 self.files_to_test = files_to_test
178 self.files_to_process = files_to_process
179 self.library_id = library_id
180 self.status_updates = status_updates
181 self.abort_flag = threading.Event()
182 self.abort_flag.clear()
183  
184 def _log(self, message, message2='', level="info"):
185 if not self.logger:
186 unmanic_logging = unlogger.UnmanicLogger.__call__()
187 self.logger = unmanic_logging.get_logger(self.name)
188 message = common.format_message(message, message2)
189 getattr(self.logger, level)(message)
190  
191 def stop(self):
192 self.abort_flag.set()
193  
194 def run(self):
195 self._log("Starting {}".format(self.name))
196 file_test = FileTest(self.library_id)
197 while not self.abort_flag.is_set():
198 try:
199 # Pending task queue has an item available. Fetch it.
200 next_file = self.files_to_test.get_nowait()
201 self.status_updates.put(next_file)
202 except queue.Empty:
203 self.event.wait(2)
204 continue
205 except Exception as e:
206 self._log("Exception in fetching library scan result for path {}:".format(self.name), message2=str(e),
207 level="exception")
208  
209 # Test file to be added to task list. Add it if required
210 try:
211 result, issues, priority_score = file_test.should_file_be_added_to_task_list(next_file)
212 # Log any error messages
213 for issue in issues:
214 if type(issue) is dict:
215 self._log(issue.get('message'))
216 else:
217 self._log(issue)
218 # If file needs to be added, then add it
219 if result:
220 self.add_path_to_queue({
221 'path': next_file,
222 'priority_score': priority_score,
223 })
224 except UnicodeEncodeError:
225 self._log("File contains Unicode characters that cannot be processed. Ignoring.", level="warning")
226 except Exception as e:
227 self._log("Exception testing file path in {}. Ignoring.".format(self.name), message2=str(e),
228 level="exception")
229  
230 self._log("Exiting {}".format(self.name))
231  
232 def add_path_to_queue(self, item):
233 self.files_to_process.put(item)