corrade-nucleus-nucleons – Blame information for rev 20

Subversion Repositories:
Rev:
Rev Author Line No. Line
20 office 1 #!/usr/bin/env python2.7
2 from __future__ import absolute_import, unicode_literals, print_function, division
3  
4 from sys import argv
5 from os import environ, stat, remove as _delete_file
6 from os.path import isfile, dirname, basename, abspath
7 from hashlib import sha256
8 from subprocess import check_call as run
9  
10 from boto.s3.connection import S3Connection
11 from boto.s3.key import Key
12 from boto.exception import S3ResponseError
13  
14  
15 NEED_TO_UPLOAD_MARKER = '.need-to-upload'
16 BYTES_PER_MB = 1024 * 1024
17 try:
18 BUCKET_NAME = environ['TWBS_S3_BUCKET']
19 except KeyError:
20 raise SystemExit("TWBS_S3_BUCKET environment variable not set!")
21  
22  
23 def _sha256_of_file(filename):
24 hasher = sha256()
25 with open(filename, 'rb') as input_file:
26 hasher.update(input_file.read())
27 file_hash = hasher.hexdigest()
28 print('sha256({}) = {}'.format(filename, file_hash))
29 return file_hash
30  
31  
32 def _delete_file_quietly(filename):
33 try:
34 _delete_file(filename)
35 except (OSError, IOError):
36 pass
37  
38  
39 def _tarball_size(directory):
40 kib = stat(_tarball_filename_for(directory)).st_size // BYTES_PER_MB
41 return "{} MiB".format(kib)
42  
43  
44 def _tarball_filename_for(directory):
45 return abspath('./{}.tar.gz'.format(basename(directory)))
46  
47  
48 def _create_tarball(directory):
49 print("Creating tarball of {}...".format(directory))
50 run(['tar', '-czf', _tarball_filename_for(directory), '-C', dirname(directory), basename(directory)])
51  
52  
53 def _extract_tarball(directory):
54 print("Extracting tarball of {}...".format(directory))
55 run(['tar', '-xzf', _tarball_filename_for(directory), '-C', dirname(directory)])
56  
57  
58 def download(directory):
59 _delete_file_quietly(NEED_TO_UPLOAD_MARKER)
60 try:
61 print("Downloading {} tarball from S3...".format(friendly_name))
62 key.get_contents_to_filename(_tarball_filename_for(directory))
63 except S3ResponseError as err:
64 open(NEED_TO_UPLOAD_MARKER, 'a').close()
65 print(err)
66 raise SystemExit("Cached {} download failed!".format(friendly_name))
67 print("Downloaded {}.".format(_tarball_size(directory)))
68 _extract_tarball(directory)
69 print("{} successfully installed from cache.".format(friendly_name))
70  
71  
72 def upload(directory):
73 _create_tarball(directory)
74 print("Uploading {} tarball to S3... ({})".format(friendly_name, _tarball_size(directory)))
75 key.set_contents_from_filename(_tarball_filename_for(directory))
76 print("{} cache successfully updated.".format(friendly_name))
77 _delete_file_quietly(NEED_TO_UPLOAD_MARKER)
78  
79  
80 if __name__ == '__main__':
81 # Uses environment variables:
82 # AWS_ACCESS_KEY_ID -- AWS Access Key ID
83 # AWS_SECRET_ACCESS_KEY -- AWS Secret Access Key
84 argv.pop(0)
85 if len(argv) != 4:
86 raise SystemExit("USAGE: s3_cache.py <download | upload> <friendly name> <dependencies file> <directory>")
87 mode, friendly_name, dependencies_file, directory = argv
88  
89 conn = S3Connection()
90 bucket = conn.lookup(BUCKET_NAME, validate=False)
91 if bucket is None:
92 raise SystemExit("Could not access bucket!")
93  
94 dependencies_file_hash = _sha256_of_file(dependencies_file)
95  
96 key = Key(bucket, dependencies_file_hash)
97 key.storage_class = 'REDUCED_REDUNDANCY'
98  
99 if mode == 'download':
100 download(directory)
101 elif mode == 'upload':
102 if isfile(NEED_TO_UPLOAD_MARKER): # FIXME
103 upload(directory)
104 else:
105 print("No need to upload anything.")
106 else:
107 raise SystemExit("Unrecognized mode {!r}".format(mode))