fwfetcher.py
Go to the documentation of this file.
1 #!/usr/bin/env python2
2 
3 from urllib2 import Request, urlopen, URLError
4 import hashlib
5 import os
6 import StringIO
7 import struct
8 import sys
9 import time
10 import zipfile
11 
12 # fwfetcher.py - a program to extract the Kinect audio firmware from an Xbox360
13 # system update. This program includes substantial portions of extract360.py,
14 # which is copyright Rene Ladan and others as noted below and provided under
15 # the BSD 2-clause license.
16 
17 """Program to extract typical XBox 360 files.
18  It can handle LIVE/PIRS, CON (partially), FMIM, and XUIZ files.
19 
20  What about CRA (aka .arc) files? (Dead Rising demo)
21 
22  Copyright (c) 2007, 2008, Rene Ladan <r.c.ladan@gmail.com>, 2-claused BSD
23  license. Portions from various contributors as mentioned in-source.
24 
25  Note that it dumps UTF-16 characters in text strings as-is.
26 """
27 
28 ################################################################################
29 
30 def check_size(fsize, minsize):
31  """Ensure that the filesize is at least minsize bytes.
32 
33  @param fsize the filesize
34  @param minsize the minimal file size
35  @return fsize >= minsize
36  """
37 
38  if fsize < minsize:
39  print "Input file too small: %i instead of at least %i bytes." % \
40  (fsize, minsize)
41  return fsize >= minsize
42 
43 ################################################################################
44 
45 def nice_open_file(filename):
46  """Checks if the output file with the given name already exists,
47  and if so, asks for overwrite permission.
48 
49  @param filename name of the output file to open
50  @return overwrite permission
51  """
52 
53  if os.path.isfile(filename):
54  print filename, "already exists, overwrite? (y/n)",
55  answer = raw_input("")
56  return len(answer) > 0 and answer[0] in ["Y", "y"]
57  else:
58  return True
59 
60 ################################################################################
61 
62 def nice_open_dir(dirname):
63  """Checks if the output directory with the given name already exists,
64  and if so, asks for overwrite permission. This means that any file
65  in that directory might be overwritten.
66 
67  @param dirname name of the output directory to open
68  @return overwrite permission
69  """
70 
71  if os.path.isdir(dirname):
72  print dirname, "already exists, ok to overwrite files in it? (y/n)",
73  answer = raw_input("")
74  return len(answer) > 0 and answer[0] in ["Y", "y"]
75  else:
76  return True
77 
78 ################################################################################
79 
80 def do_mkdir(dirname):
81  """Version of os.mkdir() which does not throw an exception if the directory
82  already exists.
83 
84  @param dirname name of the directory to create
85  """
86 
87  try:
88  os.mkdir(dirname)
89  except OSError, (errno):
90  if errno == 17:
91  pass # directory already exists
92 
93 ################################################################################
94 
95 def strip_blanks(instring):
96  """Strip the leading and trailing blanks from the input string.
97  Blanks are: 0x00 (only trailing) space \t \n \r \v \f 0xFF
98 
99  @param instring the input string
100  @return stripped version of instring
101  """
102 
103  rstr = instring.rstrip("\0 \t\n\r\v\f\377")
104  return rstr.lstrip(" \t\n\r\v\f\377")
105 
106 ################################################################################
107 
108 def open_info_file(infile):
109  """Open the informational text file.
110  The name is based on that of the input file.
111 
112  @param infile pointer to the input file
113  @return pointer to the informational text file or None if there was no
114  overwrite permission
115  """
116 
117  txtname = os.path.basename(infile.name) + ".txt"
118  if nice_open_file(txtname):
119  print "Writing information file", txtname
120  txtfile = open(txtname, "w")
121  return txtfile
122  else:
123  return None
124 
125 ################################################################################
126 
127 def dump_png(infile, pnglen, maxlen, pngid):
128  """Dump the embedded PNG file from the archive file to an output file.
129 
130  @param infile pointer to the archive file
131  @param pnglen size of the PNG file in bytes
132  @param maxlen maximum size of the PNG file in bytes
133  @param pngid indicates if this is the first or second PNG file.
134  """
135 
136  # dump PNG icon
137  if pnglen <= maxlen:
138  outname = os.path.basename(infile.name) + "_" + pngid + ".png"
139  if nice_open_file(outname):
140  buf = infile.read(pnglen)
141  print "Writing PNG file", outname
142  outfile = open(outname, "wb")
143  print >> outfile, buf,
144  outfile.close()
145  else:
146  print "PNG image %s too large (%i instead of maximal %i bytes), " \
147  "file not written." % (pngid, pnglen, maxlen)
148 
149 ################################################################################
150 
151 def dump_info(infile, txtfile, what):
152  """Dumps the 9 information strings from the input file.
153 
154  @param infile pointer to the input file
155  @param txtfile pointer to the resulting text file
156  @param what indicates if the information consists of titles or
157  descriptions
158  """
159 
160  print >> txtfile, "\n", what, ":"
161  for i in xrange(9):
162  info = strip_blanks(infile.read(0x100))
163  if len(info) > 0:
164  print >> txtfile, lang[i], ":", info
165 
166 ################################################################################
167 
168 def mstime(intime):
169  """Convert the time given in Microsoft format to a normal time tuple.
170 
171  @param intime the time in Microsoft format
172  @return the time tuple
173  """
174 
175  num_d = (intime & 0xFFFF0000L) >> 16
176  num_t = intime & 0x0000FFFFL
177  # format below is : year, month, day, hour, minute, second,
178  # weekday (Monday), yearday (unused), DST flag (guess)
179  return ((num_d >> 9) + 1980, (num_d >> 5) & 0x0F, num_d & 0x1F,
180  (num_t & 0xFFFF) >> 11, (num_t >> 5) & 0x3F, (num_t & 0x1F) * 2,
181  0, 0, -1)
182 
183 ################################################################################
184 
185 def do_utime(targetname, atime, mtime):
186  """Set the access and update date/time of the target.
187  Taken from tarfile.py (builtin lib)
188 
189  @param targetname name of the target
190  @param atime the desired access date/time
191  @param mtime the desired update date/time
192  """
193 
194  if not hasattr(os, "utime"):
195  return
196  if not (sys.platform == "win32" and os.path.isdir(targetname)):
197  # Using utime() on directories is not allowed on Win32 according to
198  # msdn.microsoft.com
199  os.utime(targetname,
200  (time.mktime(mstime(atime)), time.mktime(mstime(mtime))))
201 
202 ################################################################################
203 
204 def check_sha1(sha1, entry, infile, start, end):
205  """Check the SHA1 value of the specified range of the input file.
206 
207  @param sha1 the reported SHA1 value
208  @param entry the id of the hash
209  @param infile the input file to check
210  @param start the start position
211  @param end the end position
212  @return string reporting if the hash is correct
213  """
214 
215  infile.seek(start)
216  found_sha1 = hashlib.sha1(infile.read(end - start))
217  found_digest = found_sha1.digest()
218  # SHA1 hashes are 20 bytes (160 bits) long
219  ret = "SHA1 " + hex(entry) + " "
220  if found_digest == sha1:
221  return ret + "ok (" + found_sha1.hexdigest() + ")"
222  else:
223  hexdig = ""
224  for i in sha1:
225  if ord(i) < 10:
226  val = "0"
227  else:
228  val = ""
229  val += hex(ord(i))[2:]
230  hexdig += val
231  return ret + "wrong (should be " + hexdig + " actual " + \
232  found_sha1.hexdigest() + ")"
233 
234 ################################################################################
235 
236 def get_cluster(startclust, offset):
237  """get the real starting cluster"""
238  rst = 0
239  # BEGIN wxPirs
240  while startclust >= 170:
241  startclust //= 170
242  rst += (startclust + 1) * offset
243  # END wxPirs
244  return rst
245 
246 ################################################################################
247 
248 def fill_directory(infile, txtfile, contents, firstclust, makedir, start,
249  offset):
250  """Fill the directory structure with the files contained in the archive.
251 
252  @param infile pointer to the archive
253  @param txtfile pointer to the resulting information text file
254  @param contents contains the directory information
255  @param firstclust address of the starting cluster of the first file in
256  infile (in 4kB blocks, minus start bytes)
257  @param makedir flag if directory should be filled, useful if only return
258  is wanted
259  @param start start of directory data
260  @param offset increment for calculating real starting cluster
261  """
262 
263  # dictionary which holds the directory structure,
264  # patch 0xFFFF is the 'root' directory.
265  paths = {0xFFFF:""}
266 
267  oldpathind = 0xFFFF # initial path, speed up file/dir creation
268 
269  for i in xrange(0x1000 * firstclust // 64):
270  cur = contents[i * 64:(i + 1) * 64]
271  if ord(cur[40]) == 0:
272  # if filename length is zero, we're done
273  break
274  (outname, namelen, clustsize1, val1, clustsize2, val2, startclust,
275  val3) = struct.unpack("<40sBHBHBHB", cur[0:50])
276  # sizes and starting cluster are 24 bits long
277  clustsize1 += val1 << 16
278  clustsize2 += val2 << 16
279  startclust += val3 << 16
280  (pathind, filelen, dati1, dati2) = struct.unpack(">HLLL", cur[50:64])
281 
282  if not makedir:
283  continue
284 
285  nlen = namelen & ~0xC0
286  if nlen < 1 or nlen > 40:
287  print "Filename length (%i) out of range, skipping file." % nlen
288  continue
289  outname = outname[0:nlen] # strip trailing 0x00 from filename
290 
291  if txtfile != None:
292  if namelen & 0x80 == 0x80:
293  print >> txtfile, "Directory",
294  else:
295  print >> txtfile, "File",
296  print >> txtfile, "name:", outname
297  if namelen & 0x40 == 0x40:
298  print >> txtfile, "Bit 6 of namelen is set."
299 
300  if clustsize1 != clustsize2:
301  print "Cluster sizes don't match (%i != %i), skipping file." % \
302  (clustsize1, clustsize2)
303  continue
304  if startclust < 1 and namelen & 0x80 == 0:
305  print "Starting cluster must be 1 or greater, skipping file."
306  continue
307  if filelen > 0x1000 * clustsize1:
308  print "File length (%i) is greater than the size in clusters " \
309  "(%i), skipping file." % (filelen, clustsize1)
310  continue
311 
312  if pathind != oldpathind:
313  # working directory changed
314  for _ in xrange(paths[oldpathind].count("/")):
315  os.chdir("..") # go back to root directory
316  os.chdir(paths[pathind])
317  oldpathind = pathind
318  if namelen & 0x80 == 0x80:
319  # this is a directory
320  paths[i] = paths[pathind] + outname + "/"
321  do_mkdir(outname)
322  else:
323  # this is a file
324  # space between files is set to 0x00
325  adstart = startclust * 0x1000 + start
326  if txtfile != None:
327  print >> txtfile, "Starting: advertized", hex(adstart)
328 
329  # block reading algorithm originally from wxPirs
330  buf = ""
331  while filelen > 0:
332  realstart = adstart + get_cluster(startclust, offset)
333  infile.seek(realstart)
334  buf += infile.read(min(0x1000, filelen))
335  startclust += 1
336  adstart += 0x1000
337  filelen -= 0x1000
338  outfile = open(outname, "wb")
339  print >> outfile, buf,
340  outfile.close()
341 
342  do_utime(outname, dati2, dati1)
343 
344  # pop directory
345  for _ in xrange(paths[oldpathind].count("/")):
346  os.chdir("..")
347 
348 ################################################################################
349 
350 def write_common_part(infile, txtfile, png2stop, start):
351  """Writes out the common part of PIRS/LIVE and CON files.
352 
353  @param infile pointer to the PIRS/LIVE or CON file
354  @param txtfile pointer to the resulting text file
355  @param png2stop location where the second PNG image stops
356  (PIRS/LIVE : 0xB000, CON : 0xA000)
357  @param start start of directory data, from wxPirs
358  """
359 
360  infile.seek(0x32C)
361  mhash = infile.read(20) # xbox180 : SHA1 hash of 0x0344-0xB000,
362  # CON : 0x0344 - 0xA000 (i.e. png2stop)
363  (mentry_id, content_type) = struct.unpack(">LL", infile.read(8))
364 
365  if txtfile != None:
366  print >> txtfile, "\nMaster SHA1 hash :", \
367  check_sha1(mhash, mentry_id, infile, 0x0344, png2stop)
368  print >> txtfile, "\nContent type", hex(content_type), ":",
369  # content type list partially from V1kt0R
370  # su20076000_00000000 has type 0x000b0000,
371  # i.e. "Full game demo" & "Theme" ...
372  if content_type == 0:
373  print >> txtfile, "(no type)"
374  elif content_type & 0x00000001:
375  print >> txtfile, "Game save"
376  elif content_type & 0x00000002:
377  print >> txtfile, "Game add-on"
378  elif content_type & 0x00030000:
379  print >> txtfile, "Theme"
380  elif content_type & 0x00090000:
381  print >> txtfile, "Video clip"
382  elif content_type & 0x000C0000:
383  print >> txtfile, "Game trailer"
384  elif content_type & 0x000D0000:
385  print >> txtfile, "XBox Live Arcade"
386  elif content_type & 0x00010000:
387  print >> txtfile, "Gamer profile"
388  elif content_type & 0x00020000:
389  print >> txtfile, "Gamer picture"
390  elif content_type & 0x00040000:
391  print >> txtfile, "System update"
392  elif content_type & 0x00080000:
393  print >> txtfile, "Full game demo"
394  else:
395  print >> txtfile, "(unknown)"
396 
397  print >> txtfile, "\nDirectory data at (hex)", hex(start)
398  infile.seek(0x410)
399  dump_info(infile, txtfile, "Titles")
400  dump_info(infile, txtfile, "Descriptions")
401  print >> txtfile, "\nPublisher:", strip_blanks(infile.read(0x80)), "\n"
402  print >> txtfile, "\nFilename:", strip_blanks(infile.read(0x80)), "\n"
403  infile.seek(0x1710)
404  (val1, png1len, png2len) = struct.unpack(">HLL", infile.read(10))
405  if txtfile != None:
406  print >> txtfile, "Value:", val1
407 
408  if png1len > 0:
409  dump_png(infile, png1len, 0x571A - 0x171A, "1")
410 
411  if png2len > 0:
412  infile.seek(0x571A)
413  dump_png(infile, png2len, png2stop - 0x571A, "2")
414 
415  # entries are 64 bytes long
416  # unused entries are set to 0x00
417  infile.seek(start + 0x2F)
418  (firstclust, ) = struct.unpack("<H", infile.read(2))
419 
420  infile.seek(start)
421  buf = infile.read(0x1000 * firstclust)
422 
423  outname = os.path.basename(infile.name) + ".dir"
424  makedir = nice_open_dir(outname)
425  if makedir:
426  print "Creating and filling content directory", outname
427  do_mkdir(outname)
428  os.chdir(outname)
429  if png2stop == 0xB000 and start == 0xC000:
430  offset = 0x1000
431  else:
432  offset = 0x2000
433  fill_directory(infile, txtfile, buf, firstclust, makedir, start, offset)
434 
435  # table of SHA1 hashes of payload
436  if txtfile != None:
437  print >> txtfile
438  infile.seek(png2stop)
439  buf = infile.read(start - png2stop)
440  numempty = 0
441  for i in xrange(len(buf) // 24):
442  entry = buf[i * 24: i * 24 + 24]
443  if entry.count("\0") < 24:
444  if numempty > 0:
445  print >> txtfile, "\nEmpty entries:", numempty
446  numempty = 0
447  print >> txtfile, "Hash (hex):",
448  for j in xrange(20):
449  print >> txtfile, hex(ord(entry[j])),
450  (j, ) = struct.unpack(">L", entry[20:24])
451  print >> txtfile, "\nEntry id:", hex(j)
452  else:
453  numempty += 1
454 
455  print >> txtfile, "\nTrailing data (hex):",
456  for i in buf[len(buf) - (len(buf) % 24):]:
457  print >> txtfile, hex(ord(i)),
458  print >> txtfile
459 
460  txtfile.close()
461 
462 ################################################################################
463 
464 def handle_live_pirs(infile, fsize):
465  """LIVE and PIRS files are archive files.
466  They contain a certificate, payload, SHA1 checksums,
467  2 icons, textual information, and the files themselves.
468 
469  @param infile pointer to the archive file
470  @param fsize size of infile
471  """
472 
473  print "Handling LIVE/PIRS file."
474 
475  if not check_size(fsize, 0xD000):
476  return
477 
478  txtfile = open_info_file(infile)
479  if txtfile != None:
480  print >> txtfile, "Certificate (hex):",
481  cert = infile.read(0x100)
482  for i in cert:
483  print >> txtfile, hex(ord(i)),
484 
485  print >> txtfile, "\n\nData (hex):",
486  data = infile.read(0x228)
487  for i in data:
488  print >> txtfile, hex(ord(i)),
489  print >> txtfile
490 
491  ### BEGIN wxPirs ###
492  infile.seek(0xC032) # originally 4 bytes at 0xC030
493  (pathind, ) = struct.unpack(">H", infile.read(2))
494  if pathind == 0xFFFF:
495  start = 0xC000
496  else:
497  start = 0xD000
498  ### END wxPirs ###
499  write_common_part(infile, txtfile, 0xB000, start)
500 
501 ################################################################################
502 
503 # End of code taken from extract360.py.
504 
505 def getFileOrURL(filename, url):
506  # Check if a file named filename exists on disk.
507  # If so, return its contents. If not, download it, save it, and return its contents.
508  try:
509  f = open(filename)
510  print "Found", filename, "cached on disk, using local copy"
511  retval = f.read()
512  return retval
513  except IOError, e:
514  pass
515  print "Downloading", filename, "from", url
516  req = Request(url)
517  try:
518  response = urlopen(req)
519  except URLError, e:
520  if hasattr(e, 'reason'):
521  print "Failed to reach download server. Reason:", e.reason
522  elif hasattr(e, 'code'):
523  print "The server couldn't fulfill the request. Error code:", e.code
524  print "Reading response..."
525  retval = response.read()
526  # Save downloaded file to disk
527  f = open(filename, "wb")
528  f.write(retval)
529  f.close()
530  print "done, saved to", filename
531  return retval
532 
533 def extractPirsFromZip(systemupdate):
534  print "Extracting $systemupdate/FFFE07DF00000001 from system update file..."
535  updatefile = StringIO.StringIO(systemupdate)
536  z = zipfile.ZipFile(updatefile)
537  #print z.namelist()
538  pirs = z.open("$systemupdate/FFFE07DF00000001").read()
539  print "done."
540  return pirs
541 
542 if __name__ == "__main__":
543  target = "audios.bin"
544  if len(sys.argv) == 2:
545  target = sys.argv[1]
546  if not os.path.isfile(target):
547  fw = getFileOrURL("SystemUpdate.zip", "http://www.xbox.com/system-update-usb")
549 
550  lang = ["English", "Japanese", "German", "French", "Spanish", "Italian",
551  "Korean", "Chinese", "Portuguese"]
552  sio = StringIO.StringIO(pirs)
553  basename = "FFFE07DF00000001"
554  sio.name = basename
555  pwd = os.getcwd()
556  handle_live_pirs(sio, len(pirs)-4)
557 
558  os.chdir(pwd)
559  print "Moving audios.bin to current folder"
560  os.rename(os.path.join(basename + ".dir", "audios.bin"), target)
561 
562  print "Cleaning up"
563  os.unlink(basename + ".txt")
564  for root, dirs, files in os.walk(basename + ".dir"):
565  for name in files:
566  os.remove(os.path.join(root, name))
567  for name in dirs:
568  os.rmdir(os.path.join(root, name))
569  os.rmdir(root)
570  os.unlink("SystemUpdate.zip")
571  print "Done!"
572  else:
573  print "Already have audios.bin"
def handle_live_pirs(infile, fsize)
Definition: fwfetcher.py:464
def check_sha1(sha1, entry, infile, start, end)
Definition: fwfetcher.py:204
def open_info_file(infile)
Definition: fwfetcher.py:108
def nice_open_dir(dirname)
Definition: fwfetcher.py:62
def write_common_part(infile, txtfile, png2stop, start)
Definition: fwfetcher.py:350
def fill_directory(infile, txtfile, contents, firstclust, makedir, start, offset)
Definition: fwfetcher.py:249
def dump_png(infile, pnglen, maxlen, pngid)
Definition: fwfetcher.py:127
def do_mkdir(dirname)
Definition: fwfetcher.py:80
def strip_blanks(instring)
Definition: fwfetcher.py:95
def nice_open_file(filename)
Definition: fwfetcher.py:45
def get_cluster(startclust, offset)
Definition: fwfetcher.py:236
def mstime(intime)
Definition: fwfetcher.py:168
def getFileOrURL(filename, url)
Definition: fwfetcher.py:505
def dump_info(infile, txtfile, what)
Definition: fwfetcher.py:151
def extractPirsFromZip(systemupdate)
Definition: fwfetcher.py:533
def do_utime(targetname, atime, mtime)
Definition: fwfetcher.py:185
def check_size(fsize, minsize)
Definition: fwfetcher.py:30


libfreenect
Author(s): Hector Martin, Josh Blake, Kyle Machulis, OpenKinect community
autogenerated on Thu Jun 6 2019 19:25:38