source: code/uppir_create_manifest.py

Last change on this file was 23, checked in by trishank, 7 years ago

Add pre-release upPIR from June 2011.

File size: 5.7 KB
Line 
1"""
2<Author>
3  Justin Cappos
4  (inspired from a previous version by Geremy Condra)
5
6<Start Date>
7  May 17th, 2011
8
9<Description>
10  Creates a manifest from the files in a directory.   This takes a set of
11  files that one wants to serve (rooted in 'vendorroot') and prepares the
12  necessary metadata (manifest file) to serve them.   The client, mirror,
13  and vendor all need this file in order for upPIR to function.   Note that
14  this file includes the host name of the vendor and so must be regenerated
15  if the vendor's host name changes.
16
17  The block size is the minimum number of bytes that must be downloaded from
18  a mirror.   A setting of 1MB will work well for most applications.   However,
19  for more information about tuning this, please see the upPIR website.
20
21  For more technical explanation, please see the upPIR papers on my website.
22 
23
24<Usage>
25  $ python uppir_create_manifest.py vendorroot blocksize vendorhostname
26
27
28<Options>
29
30See below
31
32"""
33
34
35# This file is laid out in two main parts.   First, we parse the command line
36# options using parse_options().   Next, we generate the mainfest in the
37# main part (not in a function).
38#
39# EXTENSION POINTS:
40#
41# To change the way that files are mapped to blocks, one should create a
42# function and add it to: _offsetoptionname_to_functionmap.   This can be
43# used to cause files to intentionally span blocks (or not span blocks) to
44# better mask what is downloaded.
45#
46# The manifest file could also be extended to support huge files (those that
47# span multiple releases).   This would primarily require client changes, but
48# it may be useful to add metadata to the manifest to further indicate
49# information about how to stitch these files back together.
50
51
52import sys
53
54import uppirlib
55
56import optparse
57
58# Check the python version
59if sys.version_info[0] != 2 or sys.version_info[1] < 5:
60  print "Requires Python >= 2.5 and < 3.0"
61  sys.exit(1)
62
63# get JSON
64if sys.version_info[1] == 5:
65  try:
66    import simplejson as json
67  except ImportError:
68    # This may have plausibly been forgotten
69    print "Requires simplejson on Python 2.5.X"
70    sys.exit(1)
71else:
72  # This really should be there.   Let's ignore the try-except block...
73  import json
74
75
76# This says which function corresponds to an option
77_offsetoptionname_to_functionmap = {'nogaps':uppirlib.nogaps_offset_assignment_function}
78
79
80
81
82def parse_options():
83  """
84  <Purpose>
85    Parses command line arguments.
86
87  <Arguments>
88    None
89 
90  <Side Effects>
91    None
92
93  <Exceptions>
94    These are handled by optparse internally.   I believe it will print / exit
95    itself without raising exceptions further.   I do print an error and
96    exit if there are extra args...
97
98  <Returns>
99    The command line options (includes the rootdir and blocksize)
100  """
101
102
103  parser = optparse.OptionParser()
104
105  parser.add_option("","--manifestfile", dest="manifestfile", type="string",
106        metavar="manifestfile", default="manifest.dat",
107        help="Use this name for the manifest file (default manifest.dat)")
108
109  parser.add_option("","--vendorport", dest="vendorport", type="int",
110        metavar="port", default=62293,
111        help="The vendor will listen on this port (default 62293)")
112
113
114
115  parser.add_option("","--hashalgorithm", dest="hashalgorithm", type="string",
116        metavar="algorithm", default="sha256-hex",
117        help="Chooses which algorithm to use for the secure hash (default sha1-base64)")
118
119  parser.add_option("","--offsetalgorithm", dest="offsetalgorithm",
120        type="string", metavar="algorithm", default="nogaps",
121        help="Chooses how to put the files into blocks (default is nogaps).   The supported values are nogaps, (more to come)")
122
123
124
125  # let's parse the args
126  (commandlineoptions, remainingargs) = parser.parse_args()
127
128  # check the arguments
129  if commandlineoptions.offsetalgorithm not in _offsetoptionname_to_functionmap:
130    print "Unknown offsetalgorithm, try one of:",_offsetoptionname_to_functionmap.keys()
131    sys.exit(1)
132
133  # replace the string with a function reference.   
134  # JAC: Stylistically, I don't like this, but I don't know an easy way
135  # to improve it.
136  commandlineoptions.offsetalgorithm = _offsetoptionname_to_functionmap[commandlineoptions.offsetalgorithm]
137
138  if len(remainingargs) != 3:
139    print "Requires exactly three additional arguments: rootdir blocksize vendorhostname"
140    sys.exit(1)
141
142  # add these to the object to parse later...
143  commandlineoptions.rootdir = remainingargs[0]
144
145  commandlineoptions.blocksize = int(remainingargs[1])
146
147  commandlineoptions.vendorhostname = remainingargs[2]
148
149  if commandlineoptions.blocksize <=0:
150    print "Specified blocksize number is not positive"
151    sys.exit(1)
152
153  if commandlineoptions.blocksize %64:
154    print "Blocksize must be divisible by 64"
155    sys.exit(1)
156
157  if commandlineoptions.vendorport <=0 or commandlineoptions.vendorport > 65535:
158    print "Invalid vendorport"
159    sys.exit(1)
160
161  return commandlineoptions
162
163
164
165
166if __name__ == '__main__':
167  # parse user provided data
168  commandlineoptions = parse_options()
169 
170  # create the dict
171  manifestdict = uppirlib.create_manifest(rootdir=commandlineoptions.rootdir,
172        hashalgorithm=commandlineoptions.hashalgorithm,
173        block_size=commandlineoptions.blocksize,
174        offset_assignment_function=commandlineoptions.offsetalgorithm,
175        vendorhostname=commandlineoptions.vendorhostname,
176        vendorport=commandlineoptions.vendorport)
177
178  # open the destination file
179  manifestfo = open(commandlineoptions.manifestfile,'w')
180
181  # and write it in a safely serialized format (JSON).
182  rawmanifest = json.dumps(manifestdict)
183  manifestfo.write(rawmanifest)
184 
185  manifestfo.close()
186
187  print "Generated",commandlineoptions.manifestfile,"describing xordatastore with",manifestdict['blockcount'],manifestdict['blocksize'],'byte blocks'
Note: See TracBrowser for help on using the repository browser.