You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
449 lines
14 KiB
449 lines
14 KiB
/* gzjoin -- command to join gzip files into one gzip file |
|
|
|
Copyright (C) 2004, 2005, 2012 Mark Adler, all rights reserved |
|
version 1.2, 14 Aug 2012 |
|
|
|
This software is provided 'as-is', without any express or implied |
|
warranty. In no event will the author be held liable for any damages |
|
arising from the use of this software. |
|
|
|
Permission is granted to anyone to use this software for any purpose, |
|
including commercial applications, and to alter it and redistribute it |
|
freely, subject to the following restrictions: |
|
|
|
1. The origin of this software must not be misrepresented; you must not |
|
claim that you wrote the original software. If you use this software |
|
in a product, an acknowledgment in the product documentation would be |
|
appreciated but is not required. |
|
2. Altered source versions must be plainly marked as such, and must not be |
|
misrepresented as being the original software. |
|
3. This notice may not be removed or altered from any source distribution. |
|
|
|
Mark Adler madler@alumni.caltech.edu |
|
*/ |
|
|
|
/* |
|
* Change history: |
|
* |
|
* 1.0 11 Dec 2004 - First version |
|
* 1.1 12 Jun 2005 - Changed ssize_t to long for portability |
|
* 1.2 14 Aug 2012 - Clean up for z_const usage |
|
*/ |
|
|
|
/* |
|
gzjoin takes one or more gzip files on the command line and writes out a |
|
single gzip file that will uncompress to the concatenation of the |
|
uncompressed data from the individual gzip files. gzjoin does this without |
|
having to recompress any of the data and without having to calculate a new |
|
crc32 for the concatenated uncompressed data. gzjoin does however have to |
|
decompress all of the input data in order to find the bits in the compressed |
|
data that need to be modified to concatenate the streams. |
|
|
|
gzjoin does not do an integrity check on the input gzip files other than |
|
checking the gzip header and decompressing the compressed data. They are |
|
otherwise assumed to be complete and correct. |
|
|
|
Each joint between gzip files removes at least 18 bytes of previous trailer |
|
and subsequent header, and inserts an average of about three bytes to the |
|
compressed data in order to connect the streams. The output gzip file |
|
has a minimal ten-byte gzip header with no file name or modification time. |
|
|
|
This program was written to illustrate the use of the Z_BLOCK option of |
|
inflate() and the crc32_combine() function. gzjoin will not compile with |
|
versions of zlib earlier than 1.2.3. |
|
*/ |
|
|
|
#include <stdio.h> /* fputs(), fprintf(), fwrite(), putc() */ |
|
#include <stdlib.h> /* exit(), malloc(), free() */ |
|
#include <fcntl.h> /* open() */ |
|
#include <unistd.h> /* close(), read(), lseek() */ |
|
#include "zlib.h" |
|
/* crc32(), crc32_combine(), inflateInit2(), inflate(), inflateEnd() */ |
|
|
|
#define local static |
|
|
|
/* exit with an error (return a value to allow use in an expression) */ |
|
local int bail(char *why1, char *why2) |
|
{ |
|
fprintf(stderr, "gzjoin error: %s%s, output incomplete\n", why1, why2); |
|
exit(1); |
|
return 0; |
|
} |
|
|
|
/* -- simple buffered file input with access to the buffer -- */ |
|
|
|
#define CHUNK 32768 /* must be a power of two and fit in unsigned */ |
|
|
|
/* bin buffered input file type */ |
|
typedef struct { |
|
char *name; /* name of file for error messages */ |
|
int fd; /* file descriptor */ |
|
unsigned left; /* bytes remaining at next */ |
|
unsigned char *next; /* next byte to read */ |
|
unsigned char *buf; /* allocated buffer of length CHUNK */ |
|
} bin; |
|
|
|
/* close a buffered file and free allocated memory */ |
|
local void bclose(bin *in) |
|
{ |
|
if (in != NULL) { |
|
if (in->fd != -1) |
|
close(in->fd); |
|
if (in->buf != NULL) |
|
free(in->buf); |
|
free(in); |
|
} |
|
} |
|
|
|
/* open a buffered file for input, return a pointer to type bin, or NULL on |
|
failure */ |
|
local bin *bopen(char *name) |
|
{ |
|
bin *in; |
|
|
|
in = malloc(sizeof(bin)); |
|
if (in == NULL) |
|
return NULL; |
|
in->buf = malloc(CHUNK); |
|
in->fd = open(name, O_RDONLY, 0); |
|
if (in->buf == NULL || in->fd == -1) { |
|
bclose(in); |
|
return NULL; |
|
} |
|
in->left = 0; |
|
in->next = in->buf; |
|
in->name = name; |
|
return in; |
|
} |
|
|
|
/* load buffer from file, return -1 on read error, 0 or 1 on success, with |
|
1 indicating that end-of-file was reached */ |
|
local int bload(bin *in) |
|
{ |
|
long len; |
|
|
|
if (in == NULL) |
|
return -1; |
|
if (in->left != 0) |
|
return 0; |
|
in->next = in->buf; |
|
do { |
|
len = (long)read(in->fd, in->buf + in->left, CHUNK - in->left); |
|
if (len < 0) |
|
return -1; |
|
in->left += (unsigned)len; |
|
} while (len != 0 && in->left < CHUNK); |
|
return len == 0 ? 1 : 0; |
|
} |
|
|
|
/* get a byte from the file, bail if end of file */ |
|
#define bget(in) (in->left ? 0 : bload(in), \ |
|
in->left ? (in->left--, *(in->next)++) : \ |
|
bail("unexpected end of file on ", in->name)) |
|
|
|
/* get a four-byte little-endian unsigned integer from file */ |
|
local unsigned long bget4(bin *in) |
|
{ |
|
unsigned long val; |
|
|
|
val = bget(in); |
|
val += (unsigned long)(bget(in)) << 8; |
|
val += (unsigned long)(bget(in)) << 16; |
|
val += (unsigned long)(bget(in)) << 24; |
|
return val; |
|
} |
|
|
|
/* skip bytes in file */ |
|
local void bskip(bin *in, unsigned skip) |
|
{ |
|
/* check pointer */ |
|
if (in == NULL) |
|
return; |
|
|
|
/* easy case -- skip bytes in buffer */ |
|
if (skip <= in->left) { |
|
in->left -= skip; |
|
in->next += skip; |
|
return; |
|
} |
|
|
|
/* skip what's in buffer, discard buffer contents */ |
|
skip -= in->left; |
|
in->left = 0; |
|
|
|
/* seek past multiples of CHUNK bytes */ |
|
if (skip > CHUNK) { |
|
unsigned left; |
|
|
|
left = skip & (CHUNK - 1); |
|
if (left == 0) { |
|
/* exact number of chunks: seek all the way minus one byte to check |
|
for end-of-file with a read */ |
|
lseek(in->fd, skip - 1, SEEK_CUR); |
|
if (read(in->fd, in->buf, 1) != 1) |
|
bail("unexpected end of file on ", in->name); |
|
return; |
|
} |
|
|
|
/* skip the integral chunks, update skip with remainder */ |
|
lseek(in->fd, skip - left, SEEK_CUR); |
|
skip = left; |
|
} |
|
|
|
/* read more input and skip remainder */ |
|
bload(in); |
|
if (skip > in->left) |
|
bail("unexpected end of file on ", in->name); |
|
in->left -= skip; |
|
in->next += skip; |
|
} |
|
|
|
/* -- end of buffered input functions -- */ |
|
|
|
/* skip the gzip header from file in */ |
|
local void gzhead(bin *in) |
|
{ |
|
int flags; |
|
|
|
/* verify gzip magic header and compression method */ |
|
if (bget(in) != 0x1f || bget(in) != 0x8b || bget(in) != 8) |
|
bail(in->name, " is not a valid gzip file"); |
|
|
|
/* get and verify flags */ |
|
flags = bget(in); |
|
if ((flags & 0xe0) != 0) |
|
bail("unknown reserved bits set in ", in->name); |
|
|
|
/* skip modification time, extra flags, and os */ |
|
bskip(in, 6); |
|
|
|
/* skip extra field if present */ |
|
if (flags & 4) { |
|
unsigned len; |
|
|
|
len = bget(in); |
|
len += (unsigned)(bget(in)) << 8; |
|
bskip(in, len); |
|
} |
|
|
|
/* skip file name if present */ |
|
if (flags & 8) |
|
while (bget(in) != 0) |
|
; |
|
|
|
/* skip comment if present */ |
|
if (flags & 16) |
|
while (bget(in) != 0) |
|
; |
|
|
|
/* skip header crc if present */ |
|
if (flags & 2) |
|
bskip(in, 2); |
|
} |
|
|
|
/* write a four-byte little-endian unsigned integer to out */ |
|
local void put4(unsigned long val, FILE *out) |
|
{ |
|
putc(val & 0xff, out); |
|
putc((val >> 8) & 0xff, out); |
|
putc((val >> 16) & 0xff, out); |
|
putc((val >> 24) & 0xff, out); |
|
} |
|
|
|
/* Load up zlib stream from buffered input, bail if end of file */ |
|
local void zpull(z_streamp strm, bin *in) |
|
{ |
|
if (in->left == 0) |
|
bload(in); |
|
if (in->left == 0) |
|
bail("unexpected end of file on ", in->name); |
|
strm->avail_in = in->left; |
|
strm->next_in = in->next; |
|
} |
|
|
|
/* Write header for gzip file to out and initialize trailer. */ |
|
local void gzinit(unsigned long *crc, unsigned long *tot, FILE *out) |
|
{ |
|
fwrite("\x1f\x8b\x08\0\0\0\0\0\0\xff", 1, 10, out); |
|
*crc = crc32(0L, Z_NULL, 0); |
|
*tot = 0; |
|
} |
|
|
|
/* Copy the compressed data from name, zeroing the last block bit of the last |
|
block if clr is true, and adding empty blocks as needed to get to a byte |
|
boundary. If clr is false, then the last block becomes the last block of |
|
the output, and the gzip trailer is written. crc and tot maintains the |
|
crc and length (modulo 2^32) of the output for the trailer. The resulting |
|
gzip file is written to out. gzinit() must be called before the first call |
|
of gzcopy() to write the gzip header and to initialize crc and tot. */ |
|
local void gzcopy(char *name, int clr, unsigned long *crc, unsigned long *tot, |
|
FILE *out) |
|
{ |
|
int ret; /* return value from zlib functions */ |
|
int pos; /* where the "last block" bit is in byte */ |
|
int last; /* true if processing the last block */ |
|
bin *in; /* buffered input file */ |
|
unsigned char *start; /* start of compressed data in buffer */ |
|
unsigned char *junk; /* buffer for uncompressed data -- discarded */ |
|
z_off_t len; /* length of uncompressed data (support > 4 GB) */ |
|
z_stream strm; /* zlib inflate stream */ |
|
|
|
/* open gzip file and skip header */ |
|
in = bopen(name); |
|
if (in == NULL) |
|
bail("could not open ", name); |
|
gzhead(in); |
|
|
|
/* allocate buffer for uncompressed data and initialize raw inflate |
|
stream */ |
|
junk = malloc(CHUNK); |
|
strm.zalloc = Z_NULL; |
|
strm.zfree = Z_NULL; |
|
strm.opaque = Z_NULL; |
|
strm.avail_in = 0; |
|
strm.next_in = Z_NULL; |
|
ret = inflateInit2(&strm, -15); |
|
if (junk == NULL || ret != Z_OK) |
|
bail("out of memory", ""); |
|
|
|
/* inflate and copy compressed data, clear last-block bit if requested */ |
|
len = 0; |
|
zpull(&strm, in); |
|
start = in->next; |
|
last = start[0] & 1; |
|
if (last && clr) |
|
start[0] &= ~1; |
|
strm.avail_out = 0; |
|
for (;;) { |
|
/* if input used and output done, write used input and get more */ |
|
if (strm.avail_in == 0 && strm.avail_out != 0) { |
|
fwrite(start, 1, strm.next_in - start, out); |
|
start = in->buf; |
|
in->left = 0; |
|
zpull(&strm, in); |
|
} |
|
|
|
/* decompress -- return early when end-of-block reached */ |
|
strm.avail_out = CHUNK; |
|
strm.next_out = junk; |
|
ret = inflate(&strm, Z_BLOCK); |
|
switch (ret) { |
|
case Z_MEM_ERROR: |
|
bail("out of memory", ""); |
|
case Z_DATA_ERROR: |
|
bail("invalid compressed data in ", in->name); |
|
} |
|
|
|
/* update length of uncompressed data */ |
|
len += CHUNK - strm.avail_out; |
|
|
|
/* check for block boundary (only get this when block copied out) */ |
|
if (strm.data_type & 128) { |
|
/* if that was the last block, then done */ |
|
if (last) |
|
break; |
|
|
|
/* number of unused bits in last byte */ |
|
pos = strm.data_type & 7; |
|
|
|
/* find the next last-block bit */ |
|
if (pos != 0) { |
|
/* next last-block bit is in last used byte */ |
|
pos = 0x100 >> pos; |
|
last = strm.next_in[-1] & pos; |
|
if (last && clr) |
|
in->buf[strm.next_in - in->buf - 1] &= ~pos; |
|
} |
|
else { |
|
/* next last-block bit is in next unused byte */ |
|
if (strm.avail_in == 0) { |
|
/* don't have that byte yet -- get it */ |
|
fwrite(start, 1, strm.next_in - start, out); |
|
start = in->buf; |
|
in->left = 0; |
|
zpull(&strm, in); |
|
} |
|
last = strm.next_in[0] & 1; |
|
if (last && clr) |
|
in->buf[strm.next_in - in->buf] &= ~1; |
|
} |
|
} |
|
} |
|
|
|
/* update buffer with unused input */ |
|
in->left = strm.avail_in; |
|
in->next = in->buf + (strm.next_in - in->buf); |
|
|
|
/* copy used input, write empty blocks to get to byte boundary */ |
|
pos = strm.data_type & 7; |
|
fwrite(start, 1, in->next - start - 1, out); |
|
last = in->next[-1]; |
|
if (pos == 0 || !clr) |
|
/* already at byte boundary, or last file: write last byte */ |
|
putc(last, out); |
|
else { |
|
/* append empty blocks to last byte */ |
|
last &= ((0x100 >> pos) - 1); /* assure unused bits are zero */ |
|
if (pos & 1) { |
|
/* odd -- append an empty stored block */ |
|
putc(last, out); |
|
if (pos == 1) |
|
putc(0, out); /* two more bits in block header */ |
|
fwrite("\0\0\xff\xff", 1, 4, out); |
|
} |
|
else { |
|
/* even -- append 1, 2, or 3 empty fixed blocks */ |
|
switch (pos) { |
|
case 6: |
|
putc(last | 8, out); |
|
last = 0; |
|
case 4: |
|
putc(last | 0x20, out); |
|
last = 0; |
|
case 2: |
|
putc(last | 0x80, out); |
|
putc(0, out); |
|
} |
|
} |
|
} |
|
|
|
/* update crc and tot */ |
|
*crc = crc32_combine(*crc, bget4(in), len); |
|
*tot += (unsigned long)len; |
|
|
|
/* clean up */ |
|
inflateEnd(&strm); |
|
free(junk); |
|
bclose(in); |
|
|
|
/* write trailer if this is the last gzip file */ |
|
if (!clr) { |
|
put4(*crc, out); |
|
put4(*tot, out); |
|
} |
|
} |
|
|
|
/* join the gzip files on the command line, write result to stdout */ |
|
int main(int argc, char **argv) |
|
{ |
|
unsigned long crc, tot; /* running crc and total uncompressed length */ |
|
|
|
/* skip command name */ |
|
argc--; |
|
argv++; |
|
|
|
/* show usage if no arguments */ |
|
if (argc == 0) { |
|
fputs("gzjoin usage: gzjoin f1.gz [f2.gz [f3.gz ...]] > fjoin.gz\n", |
|
stderr); |
|
return 0; |
|
} |
|
|
|
/* join gzip files on command line and write to stdout */ |
|
gzinit(&crc, &tot, stdout); |
|
while (argc--) |
|
gzcopy(*argv++, argc, &crc, &tot, stdout); |
|
|
|
/* done */ |
|
return 0; |
|
}
|
|
|