diff --git a/contrib/linearize/README.md b/contrib/linearize/README.md index b5c6e7824..157586e4d 100644 --- a/contrib/linearize/README.md +++ b/contrib/linearize/README.md @@ -27,6 +27,7 @@ output. Optional config file setting for linearize-data: * "netmagic": network magic number * "max_out_sz": maximum output file size (default 1000*1000*1000) -* "split_year": Split files when a new year is first seen, in addition to +* "split_timestamp": Split files when a new month is first seen, in addition to reaching a maximum file size. - +* "file_timestamp": Set each file's last-modified time to that of the +most recent block in that file. diff --git a/contrib/linearize/linearize-data.py b/contrib/linearize/linearize-data.py index ea94f25fa..383bb3819 100755 --- a/contrib/linearize/linearize-data.py +++ b/contrib/linearize/linearize-data.py @@ -10,11 +10,13 @@ import json import struct import re +import os import base64 import httplib import sys import hashlib import datetime +import time settings = {} @@ -58,10 +60,12 @@ def calc_hash_str(blk_hdr): hash_str = hash.encode('hex') return hash_str -def get_blk_year(blk_hdr): +def get_blk_dt(blk_hdr): members = struct.unpack(" maxOutSz): outF.close() + if setFileTime: + os.utime(outFname, (int(time.time()), highTS)) outF = None + outFname = None outFn = outFn + 1 outsz = 0 - if splitYear: - blkYear = get_blk_year(blk_hdr) - if blkYear > lastYear: - print("New year " + str(blkYear) + " @ " + hash_str) - lastYear = blkYear - if outF: - outF.close() - outF = None - outFn = outFn + 1 - outsz = 0 + (blkDate, blkTS) = get_blk_dt(blk_hdr) + if timestampSplit and (blkDate > lastDate): + print("New month " + blkDate.strftime("%Y-%m") + " @ " + hash_str) + lastDate = blkDate + if outF: + outF.close() + if setFileTime: + os.utime(outFname, (int(time.time()), highTS)) + outF = None + outFname = None + outFn = outFn + 1 + outsz = 0 if not outF: if fileOutput: - fname = settings['output_file'] + outFname = settings['output_file'] else: - fname = "%s/blk%05d.dat" % (settings['output'], outFn) - print("Output file" + fname) - outF = open(fname, "wb") + outFname = "%s/blk%05d.dat" % (settings['output'], outFn) + print("Output file" + outFname) + outF = open(outFname, "wb") outF.write(inhdr) outF.write(rawblock) outsz = outsz + inLen + 8 blkCount = blkCount + 1 + if blkTS > highTS: + highTS = blkTS if (blkCount % 1000) == 0: print("Wrote " + str(blkCount) + " blocks") @@ -184,13 +206,16 @@ if __name__ == '__main__': settings['input'] = 'input' if 'hashlist' not in settings: settings['hashlist'] = 'hashlist.txt' - if 'split_year' not in settings: - settings['split_year'] = 0 + if 'file_timestamp' not in settings: + settings['file_timestamp'] = 0 + if 'split_timestamp' not in settings: + settings['split_timestamp'] = 0 if 'max_out_sz' not in settings: settings['max_out_sz'] = 1000L * 1000 * 1000 settings['max_out_sz'] = long(settings['max_out_sz']) - settings['split_year'] = int(settings['split_year']) + settings['split_timestamp'] = int(settings['split_timestamp']) + settings['file_timestamp'] = int(settings['file_timestamp']) settings['netmagic'] = settings['netmagic'].decode('hex') if 'output_file' not in settings and 'output' not in settings: