1 | """ |
---|
2 | <Author> |
---|
3 | Justin Cappos |
---|
4 | (inspired from a previous version by Geremy Condra) |
---|
5 | |
---|
6 | <Start Date> |
---|
7 | May 17th, 2011 |
---|
8 | |
---|
9 | <Description> |
---|
10 | Creates a manifest from the files in a directory. This takes a set of |
---|
11 | files that one wants to serve (rooted in 'vendorroot') and prepares the |
---|
12 | necessary metadata (manifest file) to serve them. The client, mirror, |
---|
13 | and vendor all need this file in order for upPIR to function. Note that |
---|
14 | this file includes the host name of the vendor and so must be regenerated |
---|
15 | if the vendor's host name changes. |
---|
16 | |
---|
17 | The block size is the minimum number of bytes that must be downloaded from |
---|
18 | a mirror. A setting of 1MB will work well for most applications. However, |
---|
19 | for more information about tuning this, please see the upPIR website. |
---|
20 | |
---|
21 | For more technical explanation, please see the upPIR papers on my website. |
---|
22 | |
---|
23 | |
---|
24 | <Usage> |
---|
25 | $ python uppir_create_manifest.py vendorroot blocksize vendorhostname |
---|
26 | |
---|
27 | |
---|
28 | <Options> |
---|
29 | |
---|
30 | See below |
---|
31 | |
---|
32 | """ |
---|
33 | |
---|
34 | |
---|
35 | # This file is laid out in two main parts. First, we parse the command line |
---|
36 | # options using parse_options(). Next, we generate the mainfest in the |
---|
37 | # main part (not in a function). |
---|
38 | # |
---|
39 | # EXTENSION POINTS: |
---|
40 | # |
---|
41 | # To change the way that files are mapped to blocks, one should create a |
---|
42 | # function and add it to: _offsetoptionname_to_functionmap. This can be |
---|
43 | # used to cause files to intentionally span blocks (or not span blocks) to |
---|
44 | # better mask what is downloaded. |
---|
45 | # |
---|
46 | # The manifest file could also be extended to support huge files (those that |
---|
47 | # span multiple releases). This would primarily require client changes, but |
---|
48 | # it may be useful to add metadata to the manifest to further indicate |
---|
49 | # information about how to stitch these files back together. |
---|
50 | |
---|
51 | |
---|
52 | import sys |
---|
53 | |
---|
54 | import uppirlib |
---|
55 | |
---|
56 | import optparse |
---|
57 | |
---|
58 | # Check the python version |
---|
59 | if sys.version_info[0] != 2 or sys.version_info[1] < 5: |
---|
60 | print "Requires Python >= 2.5 and < 3.0" |
---|
61 | sys.exit(1) |
---|
62 | |
---|
63 | # get JSON |
---|
64 | if sys.version_info[1] == 5: |
---|
65 | try: |
---|
66 | import simplejson as json |
---|
67 | except ImportError: |
---|
68 | # This may have plausibly been forgotten |
---|
69 | print "Requires simplejson on Python 2.5.X" |
---|
70 | sys.exit(1) |
---|
71 | else: |
---|
72 | # This really should be there. Let's ignore the try-except block... |
---|
73 | import json |
---|
74 | |
---|
75 | |
---|
76 | # This says which function corresponds to an option |
---|
77 | _offsetoptionname_to_functionmap = {'nogaps':uppirlib.nogaps_offset_assignment_function} |
---|
78 | |
---|
79 | |
---|
80 | |
---|
81 | |
---|
82 | def parse_options(): |
---|
83 | """ |
---|
84 | <Purpose> |
---|
85 | Parses command line arguments. |
---|
86 | |
---|
87 | <Arguments> |
---|
88 | None |
---|
89 | |
---|
90 | <Side Effects> |
---|
91 | None |
---|
92 | |
---|
93 | <Exceptions> |
---|
94 | These are handled by optparse internally. I believe it will print / exit |
---|
95 | itself without raising exceptions further. I do print an error and |
---|
96 | exit if there are extra args... |
---|
97 | |
---|
98 | <Returns> |
---|
99 | The command line options (includes the rootdir and blocksize) |
---|
100 | """ |
---|
101 | |
---|
102 | |
---|
103 | parser = optparse.OptionParser() |
---|
104 | |
---|
105 | parser.add_option("","--manifestfile", dest="manifestfile", type="string", |
---|
106 | metavar="manifestfile", default="manifest.dat", |
---|
107 | help="Use this name for the manifest file (default manifest.dat)") |
---|
108 | |
---|
109 | parser.add_option("","--vendorport", dest="vendorport", type="int", |
---|
110 | metavar="port", default=62293, |
---|
111 | help="The vendor will listen on this port (default 62293)") |
---|
112 | |
---|
113 | |
---|
114 | |
---|
115 | parser.add_option("","--hashalgorithm", dest="hashalgorithm", type="string", |
---|
116 | metavar="algorithm", default="sha256-hex", |
---|
117 | help="Chooses which algorithm to use for the secure hash (default sha1-base64)") |
---|
118 | |
---|
119 | parser.add_option("","--offsetalgorithm", dest="offsetalgorithm", |
---|
120 | type="string", metavar="algorithm", default="nogaps", |
---|
121 | help="Chooses how to put the files into blocks (default is nogaps). The supported values are nogaps, (more to come)") |
---|
122 | |
---|
123 | |
---|
124 | |
---|
125 | # let's parse the args |
---|
126 | (commandlineoptions, remainingargs) = parser.parse_args() |
---|
127 | |
---|
128 | # check the arguments |
---|
129 | if commandlineoptions.offsetalgorithm not in _offsetoptionname_to_functionmap: |
---|
130 | print "Unknown offsetalgorithm, try one of:",_offsetoptionname_to_functionmap.keys() |
---|
131 | sys.exit(1) |
---|
132 | |
---|
133 | # replace the string with a function reference. |
---|
134 | # JAC: Stylistically, I don't like this, but I don't know an easy way |
---|
135 | # to improve it. |
---|
136 | commandlineoptions.offsetalgorithm = _offsetoptionname_to_functionmap[commandlineoptions.offsetalgorithm] |
---|
137 | |
---|
138 | if len(remainingargs) != 3: |
---|
139 | print "Requires exactly three additional arguments: rootdir blocksize vendorhostname" |
---|
140 | sys.exit(1) |
---|
141 | |
---|
142 | # add these to the object to parse later... |
---|
143 | commandlineoptions.rootdir = remainingargs[0] |
---|
144 | |
---|
145 | commandlineoptions.blocksize = int(remainingargs[1]) |
---|
146 | |
---|
147 | commandlineoptions.vendorhostname = remainingargs[2] |
---|
148 | |
---|
149 | if commandlineoptions.blocksize <=0: |
---|
150 | print "Specified blocksize number is not positive" |
---|
151 | sys.exit(1) |
---|
152 | |
---|
153 | if commandlineoptions.blocksize %64: |
---|
154 | print "Blocksize must be divisible by 64" |
---|
155 | sys.exit(1) |
---|
156 | |
---|
157 | if commandlineoptions.vendorport <=0 or commandlineoptions.vendorport > 65535: |
---|
158 | print "Invalid vendorport" |
---|
159 | sys.exit(1) |
---|
160 | |
---|
161 | return commandlineoptions |
---|
162 | |
---|
163 | |
---|
164 | |
---|
165 | |
---|
166 | if __name__ == '__main__': |
---|
167 | # parse user provided data |
---|
168 | commandlineoptions = parse_options() |
---|
169 | |
---|
170 | # create the dict |
---|
171 | manifestdict = uppirlib.create_manifest(rootdir=commandlineoptions.rootdir, |
---|
172 | hashalgorithm=commandlineoptions.hashalgorithm, |
---|
173 | block_size=commandlineoptions.blocksize, |
---|
174 | offset_assignment_function=commandlineoptions.offsetalgorithm, |
---|
175 | vendorhostname=commandlineoptions.vendorhostname, |
---|
176 | vendorport=commandlineoptions.vendorport) |
---|
177 | |
---|
178 | # open the destination file |
---|
179 | manifestfo = open(commandlineoptions.manifestfile,'w') |
---|
180 | |
---|
181 | # and write it in a safely serialized format (JSON). |
---|
182 | rawmanifest = json.dumps(manifestdict) |
---|
183 | manifestfo.write(rawmanifest) |
---|
184 | |
---|
185 | manifestfo.close() |
---|
186 | |
---|
187 | print "Generated",commandlineoptions.manifestfile,"describing xordatastore with",manifestdict['blockcount'],manifestdict['blocksize'],'byte blocks' |
---|