corrade-nucleus-nucleons – Blame information for rev 20
?pathlinks?
Rev | Author | Line No. | Line |
---|---|---|---|
20 | office | 1 | #!/usr/bin/env python2.7 |
2 | from __future__ import absolute_import, unicode_literals, print_function, division |
||
3 | |||
4 | from sys import argv |
||
5 | from os import environ, stat, remove as _delete_file |
||
6 | from os.path import isfile, dirname, basename, abspath |
||
7 | from hashlib import sha256 |
||
8 | from subprocess import check_call as run |
||
9 | |||
10 | from boto.s3.connection import S3Connection |
||
11 | from boto.s3.key import Key |
||
12 | from boto.exception import S3ResponseError |
||
13 | |||
14 | |||
15 | NEED_TO_UPLOAD_MARKER = '.need-to-upload' |
||
16 | BYTES_PER_MB = 1024 * 1024 |
||
17 | try: |
||
18 | BUCKET_NAME = environ['TWBS_S3_BUCKET'] |
||
19 | except KeyError: |
||
20 | raise SystemExit("TWBS_S3_BUCKET environment variable not set!") |
||
21 | |||
22 | |||
23 | def _sha256_of_file(filename): |
||
24 | hasher = sha256() |
||
25 | with open(filename, 'rb') as input_file: |
||
26 | hasher.update(input_file.read()) |
||
27 | file_hash = hasher.hexdigest() |
||
28 | print('sha256({}) = {}'.format(filename, file_hash)) |
||
29 | return file_hash |
||
30 | |||
31 | |||
32 | def _delete_file_quietly(filename): |
||
33 | try: |
||
34 | _delete_file(filename) |
||
35 | except (OSError, IOError): |
||
36 | pass |
||
37 | |||
38 | |||
39 | def _tarball_size(directory): |
||
40 | kib = stat(_tarball_filename_for(directory)).st_size // BYTES_PER_MB |
||
41 | return "{} MiB".format(kib) |
||
42 | |||
43 | |||
44 | def _tarball_filename_for(directory): |
||
45 | return abspath('./{}.tar.gz'.format(basename(directory))) |
||
46 | |||
47 | |||
48 | def _create_tarball(directory): |
||
49 | print("Creating tarball of {}...".format(directory)) |
||
50 | run(['tar', '-czf', _tarball_filename_for(directory), '-C', dirname(directory), basename(directory)]) |
||
51 | |||
52 | |||
53 | def _extract_tarball(directory): |
||
54 | print("Extracting tarball of {}...".format(directory)) |
||
55 | run(['tar', '-xzf', _tarball_filename_for(directory), '-C', dirname(directory)]) |
||
56 | |||
57 | |||
58 | def download(directory): |
||
59 | _delete_file_quietly(NEED_TO_UPLOAD_MARKER) |
||
60 | try: |
||
61 | print("Downloading {} tarball from S3...".format(friendly_name)) |
||
62 | key.get_contents_to_filename(_tarball_filename_for(directory)) |
||
63 | except S3ResponseError as err: |
||
64 | open(NEED_TO_UPLOAD_MARKER, 'a').close() |
||
65 | print(err) |
||
66 | raise SystemExit("Cached {} download failed!".format(friendly_name)) |
||
67 | print("Downloaded {}.".format(_tarball_size(directory))) |
||
68 | _extract_tarball(directory) |
||
69 | print("{} successfully installed from cache.".format(friendly_name)) |
||
70 | |||
71 | |||
72 | def upload(directory): |
||
73 | _create_tarball(directory) |
||
74 | print("Uploading {} tarball to S3... ({})".format(friendly_name, _tarball_size(directory))) |
||
75 | key.set_contents_from_filename(_tarball_filename_for(directory)) |
||
76 | print("{} cache successfully updated.".format(friendly_name)) |
||
77 | _delete_file_quietly(NEED_TO_UPLOAD_MARKER) |
||
78 | |||
79 | |||
80 | if __name__ == '__main__': |
||
81 | # Uses environment variables: |
||
82 | # AWS_ACCESS_KEY_ID -- AWS Access Key ID |
||
83 | # AWS_SECRET_ACCESS_KEY -- AWS Secret Access Key |
||
84 | argv.pop(0) |
||
85 | if len(argv) != 4: |
||
86 | raise SystemExit("USAGE: s3_cache.py <download | upload> <friendly name> <dependencies file> <directory>") |
||
87 | mode, friendly_name, dependencies_file, directory = argv |
||
88 | |||
89 | conn = S3Connection() |
||
90 | bucket = conn.lookup(BUCKET_NAME, validate=False) |
||
91 | if bucket is None: |
||
92 | raise SystemExit("Could not access bucket!") |
||
93 | |||
94 | dependencies_file_hash = _sha256_of_file(dependencies_file) |
||
95 | |||
96 | key = Key(bucket, dependencies_file_hash) |
||
97 | key.storage_class = 'REDUCED_REDUNDANCY' |
||
98 | |||
99 | if mode == 'download': |
||
100 | download(directory) |
||
101 | elif mode == 'upload': |
||
102 | if isfile(NEED_TO_UPLOAD_MARKER): # FIXME |
||
103 | upload(directory) |
||
104 | else: |
||
105 | print("No need to upload anything.") |
||
106 | else: |
||
107 | raise SystemExit("Unrecognized mode {!r}".format(mode)) |