diff options
author | Mark Adler <madler@alumni.caltech.edu> | 2011-09-09 23:24:33 -0700 |
---|---|---|
committer | Mark Adler <madler@alumni.caltech.edu> | 2011-09-09 23:24:33 -0700 |
commit | 0484693e1723bbab791c56f95597bd7dbe867d03 (patch) | |
tree | 8f31dbed98b4390da74a90b484f2accf8f8a3a8e /examples/gzappend.c | |
parent | 9811b53dd9e8f67015c7199fff12b5bfc6965330 (diff) | |
download | zlib-1.2.2.2.tar.gz zlib-1.2.2.2.tar.bz2 zlib-1.2.2.2.zip |
zlib 1.2.2.2v1.2.2.2
Diffstat (limited to 'examples/gzappend.c')
-rw-r--r-- | examples/gzappend.c | 500 |
1 files changed, 500 insertions, 0 deletions
diff --git a/examples/gzappend.c b/examples/gzappend.c new file mode 100644 index 0000000..e9e878e --- /dev/null +++ b/examples/gzappend.c | |||
@@ -0,0 +1,500 @@ | |||
1 | /* gzappend -- command to append to a gzip file | ||
2 | |||
3 | Copyright (C) 2003 Mark Adler, all rights reserved | ||
4 | version 1.1, 4 Nov 2003 | ||
5 | |||
6 | This software is provided 'as-is', without any express or implied | ||
7 | warranty. In no event will the author be held liable for any damages | ||
8 | arising from the use of this software. | ||
9 | |||
10 | Permission is granted to anyone to use this software for any purpose, | ||
11 | including commercial applications, and to alter it and redistribute it | ||
12 | freely, subject to the following restrictions: | ||
13 | |||
14 | 1. The origin of this software must not be misrepresented; you must not | ||
15 | claim that you wrote the original software. If you use this software | ||
16 | in a product, an acknowledgment in the product documentation would be | ||
17 | appreciated but is not required. | ||
18 | 2. Altered source versions must be plainly marked as such, and must not be | ||
19 | misrepresented as being the original software. | ||
20 | 3. This notice may not be removed or altered from any source distribution. | ||
21 | |||
22 | Mark Adler madler@alumni.caltech.edu | ||
23 | */ | ||
24 | |||
25 | /* | ||
26 | * Change history: | ||
27 | * | ||
28 | * 1.0 19 Oct 2003 - First version | ||
29 | * 1.1 4 Nov 2003 - Expand and clarify some comments and notes | ||
30 | * - Add version and copyright to help | ||
31 | * - Send help to stdout instead of stderr | ||
32 | * - Add some preemptive typecasts | ||
33 | * - Add L to constants in lseek() calls | ||
34 | * - Remove some debugging information in error messages | ||
35 | * - Use new data_type definition for zlib 1.2.1 | ||
36 | * - Simplfy and unify file operations | ||
37 | * - Finish off gzip file in gztack() | ||
38 | * - Use deflatePrime() instead of adding empty blocks | ||
39 | * - Keep gzip file clean on appended file read errors | ||
40 | * - Use in-place rotate instead of auxiliary buffer | ||
41 | * (Why you ask? Because it was fun to write!) | ||
42 | */ | ||
43 | |||
44 | /* | ||
45 | gzappend takes a gzip file and appends to it, compressing files from the | ||
46 | command line or data from stdin. The gzip file is written to directly, to | ||
47 | avoid copying that file, in case it's large. Note that this results in the | ||
48 | unfriendly behavior that if gzappend fails, the gzip file is corrupted. | ||
49 | |||
50 | This program was written to illustrate the use of the new Z_BLOCK option of | ||
51 | zlib 1.2.x's inflate() function. This option returns from inflate() at each | ||
52 | block boundary to facilitate locating and modifying the last block bit at | ||
53 | the start of the final deflate block. Also whether using Z_BLOCK or not, | ||
54 | another required feature of zlib 1.2.x is that inflate() now provides the | ||
55 | number of unusued bits in the last input byte used. gzappend will not work | ||
56 | with versions of zlib earlier than 1.2.1. | ||
57 | |||
58 | gzappend first decompresses the gzip file internally, discarding all but | ||
59 | the last 32K of uncompressed data, and noting the location of the last block | ||
60 | bit and the number of unused bits in the last byte of the compressed data. | ||
61 | The gzip trailer containing the CRC-32 and length of the uncompressed data | ||
62 | is verified. This trailer will be later overwritten. | ||
63 | |||
64 | Then the last block bit is cleared by seeking back in the file and rewriting | ||
65 | the byte that contains it. Seeking forward, the last byte of the compressed | ||
66 | data is saved along with the number of unused bits to initialize deflate. | ||
67 | |||
68 | A deflate process is initialized, using the last 32K of the uncompressed | ||
69 | data from the gzip file to initialize the dictionary. If the total | ||
70 | uncompressed data was less than 32K, then all of it is used to initialize | ||
71 | the dictionary. The deflate output bit buffer is also initialized with the | ||
72 | last bits from the original deflate stream. From here on, the data to | ||
73 | append is simply compressed using deflate, and written to the gzip file. | ||
74 | When that is complete, the new CRC-32 and uncompressed length are written | ||
75 | as the trailer of the gzip file. | ||
76 | */ | ||
77 | |||
78 | #include <stdio.h> | ||
79 | #include <stdlib.h> | ||
80 | #include <string.h> | ||
81 | #include <fcntl.h> | ||
82 | #include <unistd.h> | ||
83 | #include "zlib.h" | ||
84 | |||
85 | #define local static | ||
86 | #define LGCHUNK 14 | ||
87 | #define CHUNK (1U << LGCHUNK) | ||
88 | #define DSIZE 32768U | ||
89 | |||
90 | /* print an error message and terminate with extreme prejudice */ | ||
91 | local void bye(char *msg1, char *msg2) | ||
92 | { | ||
93 | fprintf(stderr, "gzappend error: %s%s\n", msg1, msg2); | ||
94 | exit(1); | ||
95 | } | ||
96 | |||
97 | /* return the greatest common divisor of a and b using Euclid's algorithm, | ||
98 | modified to be fast when one argument much greater than the other, and | ||
99 | coded to avoid unnecessary swapping */ | ||
100 | local unsigned gcd(unsigned a, unsigned b) | ||
101 | { | ||
102 | unsigned c; | ||
103 | |||
104 | while (a && b) | ||
105 | if (a > b) { | ||
106 | c = b; | ||
107 | while (a - c >= c) | ||
108 | c <<= 1; | ||
109 | a -= c; | ||
110 | } | ||
111 | else { | ||
112 | c = a; | ||
113 | while (b - c >= c) | ||
114 | c <<= 1; | ||
115 | b -= c; | ||
116 | } | ||
117 | return a + b; | ||
118 | } | ||
119 | |||
120 | /* rotate list[0..len-1] left by rot positions, in place */ | ||
121 | local void rotate(unsigned char *list, unsigned len, unsigned rot) | ||
122 | { | ||
123 | unsigned char tmp; | ||
124 | unsigned cycles; | ||
125 | unsigned char *start, *last, *to, *from; | ||
126 | |||
127 | /* normalize rot and handle degenerate cases */ | ||
128 | if (len < 2) return; | ||
129 | if (rot >= len) rot %= len; | ||
130 | if (rot == 0) return; | ||
131 | |||
132 | /* pointer to last entry in list */ | ||
133 | last = list + (len - 1); | ||
134 | |||
135 | /* do simple left shift by one */ | ||
136 | if (rot == 1) { | ||
137 | tmp = *list; | ||
138 | memcpy(list, list + 1, len - 1); | ||
139 | *last = tmp; | ||
140 | return; | ||
141 | } | ||
142 | |||
143 | /* do simple right shift by one */ | ||
144 | if (rot == len - 1) { | ||
145 | tmp = *last; | ||
146 | memmove(list + 1, list, len - 1); | ||
147 | *list = tmp; | ||
148 | return; | ||
149 | } | ||
150 | |||
151 | /* otherwise do rotate as a set of cycles in place */ | ||
152 | cycles = gcd(len, rot); /* number of cycles */ | ||
153 | do { | ||
154 | start = from = list + cycles; /* start index is arbitrary */ | ||
155 | tmp = *from; /* save entry to be overwritten */ | ||
156 | for (;;) { | ||
157 | to = from; /* next step in cycle */ | ||
158 | from += rot; /* go right rot positions */ | ||
159 | if (from > last) from -= len; /* (pointer better not wrap) */ | ||
160 | if (from == start) break; /* all but one shifted */ | ||
161 | *to = *from; /* shift left */ | ||
162 | } | ||
163 | *to = tmp; /* complete the circle */ | ||
164 | } while (--cycles); | ||
165 | } | ||
166 | |||
167 | /* structure for gzip file read operations */ | ||
168 | typedef struct { | ||
169 | int fd; /* file descriptor */ | ||
170 | int size; /* 1 << size is bytes in buf */ | ||
171 | unsigned left; /* bytes available at next */ | ||
172 | unsigned char *buf; /* buffer */ | ||
173 | unsigned char *next; /* next byte in buffer */ | ||
174 | char *name; /* file name for error messages */ | ||
175 | } file; | ||
176 | |||
177 | /* reload buffer */ | ||
178 | local int readin(file *in) | ||
179 | { | ||
180 | int len; | ||
181 | |||
182 | len = read(in->fd, in->buf, 1 << in->size); | ||
183 | if (len == -1) bye("error reading ", in->name); | ||
184 | in->left = (unsigned)len; | ||
185 | in->next = in->buf; | ||
186 | return len; | ||
187 | } | ||
188 | |||
189 | /* read from file in, exit if end-of-file */ | ||
190 | local int readmore(file *in) | ||
191 | { | ||
192 | if (readin(in) == 0) bye("unexpected end of ", in->name); | ||
193 | return 0; | ||
194 | } | ||
195 | |||
196 | #define read1(in) (in->left == 0 ? readmore(in) : 0, \ | ||
197 | in->left--, *(in->next)++) | ||
198 | |||
199 | /* skip over n bytes of in */ | ||
200 | local void skip(file *in, unsigned n) | ||
201 | { | ||
202 | unsigned bypass; | ||
203 | |||
204 | if (n > in->left) { | ||
205 | n -= in->left; | ||
206 | bypass = n & ~((1U << in->size) - 1); | ||
207 | if (bypass) { | ||
208 | if (lseek(in->fd, (off_t)bypass, SEEK_CUR) == -1) | ||
209 | bye("seeking ", in->name); | ||
210 | n -= bypass; | ||
211 | } | ||
212 | readmore(in); | ||
213 | if (n > in->left) | ||
214 | bye("unexpected end of ", in->name); | ||
215 | } | ||
216 | in->left -= n; | ||
217 | in->next += n; | ||
218 | } | ||
219 | |||
220 | /* read a four-byte unsigned integer, little-endian, from in */ | ||
221 | unsigned long read4(file *in) | ||
222 | { | ||
223 | unsigned long val; | ||
224 | |||
225 | val = read1(in); | ||
226 | val += (unsigned)read1(in) << 8; | ||
227 | val += (unsigned long)read1(in) << 16; | ||
228 | val += (unsigned long)read1(in) << 24; | ||
229 | return val; | ||
230 | } | ||
231 | |||
232 | /* skip over gzip header */ | ||
233 | local void gzheader(file *in) | ||
234 | { | ||
235 | int flags; | ||
236 | unsigned n; | ||
237 | |||
238 | if (read1(in) != 31 || read1(in) != 139) bye(in->name, " not a gzip file"); | ||
239 | if (read1(in) != 8) bye("unknown compression method in", in->name); | ||
240 | flags = read1(in); | ||
241 | if (flags & 0xe0) bye("unknown header flags set in", in->name); | ||
242 | skip(in, 6); | ||
243 | if (flags & 4) { | ||
244 | n = read1(in); | ||
245 | n += (unsigned)(read1(in)) << 8; | ||
246 | skip(in, n); | ||
247 | } | ||
248 | if (flags & 8) while (read1(in) != 0) ; | ||
249 | if (flags & 16) while (read1(in) != 0) ; | ||
250 | if (flags & 2) skip(in, 2); | ||
251 | } | ||
252 | |||
253 | /* decompress gzip file "name", return strm with a deflate stream ready to | ||
254 | continue compression of the data in the gzip file, and return a file | ||
255 | descriptor pointing to where to write the compressed data -- the deflate | ||
256 | stream is initialized to compress using level "level" */ | ||
257 | local int gzscan(char *name, z_stream *strm, int level) | ||
258 | { | ||
259 | int ret, lastbit, left, full; | ||
260 | unsigned have; | ||
261 | unsigned long crc, tot; | ||
262 | unsigned char *window; | ||
263 | off_t lastoff, end; | ||
264 | file gz; | ||
265 | |||
266 | /* open gzip file */ | ||
267 | gz.name = name; | ||
268 | gz.fd = open(name, O_RDWR, 0); | ||
269 | if (gz.fd == -1) bye("cannot open ", name); | ||
270 | gz.buf = malloc(CHUNK); | ||
271 | if (gz.buf == NULL) bye("out of memory", ""); | ||
272 | gz.size = LGCHUNK; | ||
273 | gz.left = 0; | ||
274 | |||
275 | /* skip gzip header */ | ||
276 | gzheader(&gz); | ||
277 | |||
278 | /* prepare to decompress */ | ||
279 | window = malloc(DSIZE); | ||
280 | if (window == NULL) bye("out of memory", ""); | ||
281 | strm->zalloc = Z_NULL; | ||
282 | strm->zfree = Z_NULL; | ||
283 | strm->opaque = Z_NULL; | ||
284 | ret = inflateInit2(strm, -15); | ||
285 | if (ret != Z_OK) bye("out of memory", " or library mismatch"); | ||
286 | |||
287 | /* decompress the deflate stream, saving append information */ | ||
288 | lastbit = 0; | ||
289 | lastoff = lseek(gz.fd, 0L, SEEK_CUR) - gz.left; | ||
290 | left = 0; | ||
291 | strm->avail_in = gz.left; | ||
292 | strm->next_in = gz.next; | ||
293 | crc = crc32(0L, Z_NULL, 0); | ||
294 | have = full = 0; | ||
295 | do { | ||
296 | /* if needed, get more input */ | ||
297 | if (strm->avail_in == 0) { | ||
298 | readmore(&gz); | ||
299 | strm->avail_in = gz.left; | ||
300 | strm->next_in = gz.next; | ||
301 | } | ||
302 | |||
303 | /* set up output to next available section of sliding window */ | ||
304 | strm->avail_out = DSIZE - have; | ||
305 | strm->next_out = window + have; | ||
306 | |||
307 | /* inflate and check for errors */ | ||
308 | ret = inflate(strm, Z_BLOCK); | ||
309 | if (ret == Z_STREAM_ERROR) bye("internal stream error!", ""); | ||
310 | if (ret == Z_MEM_ERROR) bye("out of memory", ""); | ||
311 | if (ret == Z_DATA_ERROR) | ||
312 | bye("invalid compressed data--format violated in", name); | ||
313 | |||
314 | /* update crc and sliding window pointer */ | ||
315 | crc = crc32(crc, window + have, DSIZE - have - strm->avail_out); | ||
316 | if (strm->avail_out) | ||
317 | have = DSIZE - strm->avail_out; | ||
318 | else { | ||
319 | have = 0; | ||
320 | full = 1; | ||
321 | } | ||
322 | |||
323 | /* process end of block */ | ||
324 | if (strm->data_type & 128) { | ||
325 | if (strm->data_type & 64) | ||
326 | left = strm->data_type & 0x1f; | ||
327 | else { | ||
328 | lastbit = strm->data_type & 0x1f; | ||
329 | lastoff = lseek(gz.fd, 0L, SEEK_CUR) - strm->avail_in; | ||
330 | } | ||
331 | } | ||
332 | } while (ret != Z_STREAM_END); | ||
333 | inflateEnd(strm); | ||
334 | gz.left = strm->avail_in; | ||
335 | gz.next = strm->next_in; | ||
336 | |||
337 | /* save the location of the end of the compressed data */ | ||
338 | end = lseek(gz.fd, 0L, SEEK_CUR) - gz.left; | ||
339 | |||
340 | /* check gzip trailer and save total for deflate */ | ||
341 | if (crc != read4(&gz)) | ||
342 | bye("invalid compressed data--crc mismatch in ", name); | ||
343 | tot = strm->total_out; | ||
344 | if ((tot & 0xffffffffUL) != read4(&gz)) | ||
345 | bye("invalid compressed data--length mismatch in", name); | ||
346 | |||
347 | /* if not at end of file, warn */ | ||
348 | if (gz.left || readin(&gz)) | ||
349 | fprintf(stderr, | ||
350 | "gzappend warning: junk at end of gzip file overwritten\n"); | ||
351 | |||
352 | /* clear last block bit */ | ||
353 | lseek(gz.fd, lastoff - (lastbit != 0), SEEK_SET); | ||
354 | if (read(gz.fd, gz.buf, 1) != 1) bye("reading after seek on ", name); | ||
355 | *gz.buf = (unsigned char)(*gz.buf ^ (1 << ((8 - lastbit) & 7))); | ||
356 | lseek(gz.fd, -1L, SEEK_CUR); | ||
357 | if (write(gz.fd, gz.buf, 1) != 1) bye("writing after seek to ", name); | ||
358 | |||
359 | /* if window wrapped, build dictionary from window by rotating */ | ||
360 | if (full) { | ||
361 | rotate(window, DSIZE, have); | ||
362 | have = DSIZE; | ||
363 | } | ||
364 | |||
365 | /* set up deflate stream with window, crc, total_in, and leftover bits */ | ||
366 | ret = deflateInit2(strm, level, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY); | ||
367 | if (ret != Z_OK) bye("out of memory", ""); | ||
368 | deflateSetDictionary(strm, window, have); | ||
369 | strm->adler = crc; | ||
370 | strm->total_in = tot; | ||
371 | if (left) { | ||
372 | lseek(gz.fd, --end, SEEK_SET); | ||
373 | if (read(gz.fd, gz.buf, 1) != 1) bye("reading after seek on ", name); | ||
374 | deflatePrime(strm, 8 - left, *gz.buf); | ||
375 | } | ||
376 | lseek(gz.fd, end, SEEK_SET); | ||
377 | |||
378 | /* clean up and return */ | ||
379 | free(window); | ||
380 | free(gz.buf); | ||
381 | return gz.fd; | ||
382 | } | ||
383 | |||
384 | /* append file "name" to gzip file gd using deflate stream strm -- if last | ||
385 | is true, then finish off the deflate stream at the end */ | ||
386 | local void gztack(char *name, int gd, z_stream *strm, int last) | ||
387 | { | ||
388 | int fd, len, ret; | ||
389 | unsigned left; | ||
390 | unsigned char *in, *out; | ||
391 | |||
392 | /* open file to compress and append */ | ||
393 | fd = 0; | ||
394 | if (name != NULL) { | ||
395 | fd = open(name, O_RDONLY, 0); | ||
396 | if (fd == -1) | ||
397 | fprintf(stderr, "gzappend warning: %s not found, skipping ...\n", | ||
398 | name); | ||
399 | } | ||
400 | |||
401 | /* allocate buffers */ | ||
402 | in = fd == -1 ? NULL : malloc(CHUNK); | ||
403 | out = malloc(CHUNK); | ||
404 | if (out == NULL) bye("out of memory", ""); | ||
405 | |||
406 | /* compress input file and append to gzip file */ | ||
407 | do { | ||
408 | /* get more input */ | ||
409 | len = fd == -1 ? 0 : read(fd, in, CHUNK); | ||
410 | if (len == -1) { | ||
411 | fprintf(stderr, | ||
412 | "gzappend warning: error reading %s, skipping rest ...\n", | ||
413 | name); | ||
414 | len = 0; | ||
415 | } | ||
416 | strm->avail_in = (unsigned)len; | ||
417 | strm->next_in = in; | ||
418 | if (len) strm->adler = crc32(strm->adler, in, (unsigned)len); | ||
419 | |||
420 | /* compress and write all available output */ | ||
421 | do { | ||
422 | strm->avail_out = CHUNK; | ||
423 | strm->next_out = out; | ||
424 | ret = deflate(strm, last && len == 0 ? Z_FINISH : Z_NO_FLUSH); | ||
425 | left = CHUNK - strm->avail_out; | ||
426 | while (left) { | ||
427 | len = write(gd, out + CHUNK - strm->avail_out - left, left); | ||
428 | if (len == -1) bye("writing gzip file", ""); | ||
429 | left -= (unsigned)len; | ||
430 | } | ||
431 | } while (strm->avail_out == 0 && ret != Z_STREAM_END); | ||
432 | } while (len != 0); | ||
433 | |||
434 | /* write trailer after last entry */ | ||
435 | if (last) { | ||
436 | deflateEnd(strm); | ||
437 | out[0] = (unsigned char)(strm->adler); | ||
438 | out[1] = (unsigned char)(strm->adler >> 8); | ||
439 | out[2] = (unsigned char)(strm->adler >> 16); | ||
440 | out[3] = (unsigned char)(strm->adler >> 24); | ||
441 | out[4] = (unsigned char)(strm->total_in); | ||
442 | out[5] = (unsigned char)(strm->total_in >> 8); | ||
443 | out[6] = (unsigned char)(strm->total_in >> 16); | ||
444 | out[7] = (unsigned char)(strm->total_in >> 24); | ||
445 | len = 8; | ||
446 | do { | ||
447 | ret = write(gd, out + 8 - len, len); | ||
448 | if (ret == -1) bye("writing gzip file", ""); | ||
449 | len -= ret; | ||
450 | } while (len); | ||
451 | close(gd); | ||
452 | } | ||
453 | |||
454 | /* clean up and return */ | ||
455 | free(out); | ||
456 | if (in != NULL) free(in); | ||
457 | if (fd > 0) close(fd); | ||
458 | } | ||
459 | |||
460 | /* process the compression level option if present, scan the gzip file, and | ||
461 | append the specified files, or append the data from stdin if no other file | ||
462 | names are provided on the command line -- the gzip file must be writable | ||
463 | and seekable */ | ||
464 | int main(int argc, char **argv) | ||
465 | { | ||
466 | int gd, level; | ||
467 | z_stream strm; | ||
468 | |||
469 | /* ignore command name */ | ||
470 | argv++; | ||
471 | |||
472 | /* provide usage if no arguments */ | ||
473 | if (*argv == NULL) { | ||
474 | printf("gzappend 1.1 (4 Nov 2003) Copyright (C) 2003 Mark Adler\n"); | ||
475 | printf( | ||
476 | "usage: gzappend [-level] file.gz [ addthis [ andthis ... ]]\n"); | ||
477 | return 0; | ||
478 | } | ||
479 | |||
480 | /* set compression level */ | ||
481 | level = Z_DEFAULT_COMPRESSION; | ||
482 | if (argv[0][0] == '-') { | ||
483 | if (argv[0][1] < '0' || argv[0][1] > '9' || argv[0][2] != 0) | ||
484 | bye("invalid compression level", ""); | ||
485 | level = argv[0][1] - '0'; | ||
486 | if (*++argv == NULL) bye("no gzip file name after options", ""); | ||
487 | } | ||
488 | |||
489 | /* prepare to append to gzip file */ | ||
490 | gd = gzscan(*argv++, &strm, level); | ||
491 | |||
492 | /* append files on command line, or from stdin if none */ | ||
493 | if (*argv == NULL) | ||
494 | gztack(NULL, gd, &strm, 1); | ||
495 | else | ||
496 | do { | ||
497 | gztack(*argv, gd, &strm, argv[1] == NULL); | ||
498 | } while (*++argv != NULL); | ||
499 | return 0; | ||
500 | } | ||