diff options
author | Mark Adler <madler@alumni.caltech.edu> | 2011-09-09 23:24:33 -0700 |
---|---|---|
committer | Mark Adler <madler@alumni.caltech.edu> | 2011-09-09 23:24:33 -0700 |
commit | 0484693e1723bbab791c56f95597bd7dbe867d03 (patch) | |
tree | 8f31dbed98b4390da74a90b484f2accf8f8a3a8e /examples/gzlog.c | |
parent | 9811b53dd9e8f67015c7199fff12b5bfc6965330 (diff) | |
download | zlib-1.2.2.2.tar.gz zlib-1.2.2.2.tar.bz2 zlib-1.2.2.2.zip |
zlib 1.2.2.2v1.2.2.2
Diffstat (limited to 'examples/gzlog.c')
-rw-r--r-- | examples/gzlog.c | 413 |
1 files changed, 413 insertions, 0 deletions
diff --git a/examples/gzlog.c b/examples/gzlog.c new file mode 100644 index 0000000..f71f817 --- /dev/null +++ b/examples/gzlog.c | |||
@@ -0,0 +1,413 @@ | |||
1 | /* | ||
2 | * gzlog.c | ||
3 | * Copyright (C) 2004 Mark Adler | ||
4 | * For conditions of distribution and use, see copyright notice in gzlog.h | ||
5 | * version 1.0, 26 Nov 2004 | ||
6 | * | ||
7 | */ | ||
8 | |||
9 | #include <string.h> /* memcmp() */ | ||
10 | #include <stdlib.h> /* malloc(), free(), NULL */ | ||
11 | #include <sys/types.h> /* size_t, off_t */ | ||
12 | #include <unistd.h> /* read(), close(), sleep(), ftruncate(), */ | ||
13 | /* lseek() */ | ||
14 | #include <fcntl.h> /* open() */ | ||
15 | #include <sys/file.h> /* flock() */ | ||
16 | #include "zlib.h" /* deflateInit2(), deflate(), deflateEnd() */ | ||
17 | |||
18 | #include "gzlog.h" /* interface */ | ||
19 | #define local static | ||
20 | |||
21 | /* log object structure */ | ||
22 | typedef struct { | ||
23 | int id; /* object identifier */ | ||
24 | int fd; /* log file descriptor */ | ||
25 | off_t extra; /* offset of extra "ap" subfield */ | ||
26 | off_t mark_off; /* offset of marked data */ | ||
27 | off_t last_off; /* offset of last block */ | ||
28 | unsigned long crc; /* uncompressed crc */ | ||
29 | unsigned long len; /* uncompressed length (modulo 2^32) */ | ||
30 | unsigned stored; /* length of current stored block */ | ||
31 | } gz_log; | ||
32 | |||
33 | #define GZLOGID 19334 /* gz_log object identifier */ | ||
34 | |||
35 | #define LOCK_RETRY 1 /* retry lock once a second */ | ||
36 | #define LOCK_PATIENCE 1200 /* try about twenty minutes before forcing */ | ||
37 | |||
38 | /* acquire a lock on a file */ | ||
39 | local int lock(int fd) | ||
40 | { | ||
41 | int patience; | ||
42 | |||
43 | /* try to lock every LOCK_RETRY seconds for LOCK_PATIENCE seconds */ | ||
44 | patience = LOCK_PATIENCE; | ||
45 | do { | ||
46 | if (flock(fd, LOCK_EX + LOCK_NB) == 0) | ||
47 | return 0; | ||
48 | (void)sleep(LOCK_RETRY); | ||
49 | patience -= LOCK_RETRY; | ||
50 | } while (patience > 0); | ||
51 | |||
52 | /* we've run out of patience -- give up */ | ||
53 | return -1; | ||
54 | } | ||
55 | |||
56 | /* release lock */ | ||
57 | local void unlock(int fd) | ||
58 | { | ||
59 | (void)flock(fd, LOCK_UN); | ||
60 | } | ||
61 | |||
62 | /* release a log object */ | ||
63 | local void log_clean(gz_log *log) | ||
64 | { | ||
65 | unlock(log->fd); | ||
66 | (void)close(log->fd); | ||
67 | free(log); | ||
68 | } | ||
69 | |||
70 | /* read an unsigned long from a byte buffer little-endian */ | ||
71 | local unsigned long make_ulg(unsigned char *buf) | ||
72 | { | ||
73 | int n; | ||
74 | unsigned long val; | ||
75 | |||
76 | val = (unsigned long)(*buf++); | ||
77 | for (n = 8; n < 32; n += 8) | ||
78 | val += (unsigned long)(*buf++) << n; | ||
79 | return val; | ||
80 | } | ||
81 | |||
82 | /* read an off_t from a byte buffer little-endian */ | ||
83 | local off_t make_off(unsigned char *buf) | ||
84 | { | ||
85 | int n; | ||
86 | off_t val; | ||
87 | |||
88 | val = (off_t)(*buf++); | ||
89 | for (n = 8; n < 64; n += 8) | ||
90 | val += (off_t)(*buf++) << n; | ||
91 | return val; | ||
92 | } | ||
93 | |||
94 | /* write an unsigned long little-endian to byte buffer */ | ||
95 | local void dice_ulg(unsigned long val, unsigned char *buf) | ||
96 | { | ||
97 | int n; | ||
98 | |||
99 | for (n = 0; n < 4; n++) { | ||
100 | *buf++ = val & 0xff; | ||
101 | val >>= 8; | ||
102 | } | ||
103 | } | ||
104 | |||
105 | /* write an off_t little-endian to byte buffer */ | ||
106 | local void dice_off(off_t val, unsigned char *buf) | ||
107 | { | ||
108 | int n; | ||
109 | |||
110 | for (n = 0; n < 8; n++) { | ||
111 | *buf++ = val & 0xff; | ||
112 | val >>= 8; | ||
113 | } | ||
114 | } | ||
115 | |||
116 | /* initial, empty gzip file for appending */ | ||
117 | local char empty_gz[] = { | ||
118 | 0x1f, 0x8b, /* magic gzip id */ | ||
119 | 8, /* compression method is deflate */ | ||
120 | 4, /* there is an extra field */ | ||
121 | 0, 0, 0, 0, /* no modification time provided */ | ||
122 | 0, 0xff, /* no extra flags, no OS */ | ||
123 | 20, 0, 'a', 'p', 16, 0, /* extra field with "ap" subfield */ | ||
124 | 32, 0, 0, 0, 0, 0, 0, 0, /* offset of uncompressed data */ | ||
125 | 32, 0, 0, 0, 0, 0, 0, 0, /* offset of last block */ | ||
126 | 1, 0, 0, 0xff, 0xff, /* empty stored block (last) */ | ||
127 | 0, 0, 0, 0, /* crc */ | ||
128 | 0, 0, 0, 0 /* uncompressed length */ | ||
129 | }; | ||
130 | |||
131 | /* initialize a log object with locking */ | ||
132 | void *gzlog_open(char *path) | ||
133 | { | ||
134 | unsigned xlen; | ||
135 | unsigned char temp[20]; | ||
136 | unsigned sub_len; | ||
137 | int good; | ||
138 | gz_log *log; | ||
139 | |||
140 | /* allocate log structure */ | ||
141 | log = malloc(sizeof(gz_log)); | ||
142 | if (log == NULL) | ||
143 | return NULL; | ||
144 | log->id = GZLOGID; | ||
145 | |||
146 | /* open file, creating it if necessary, and locking it */ | ||
147 | log->fd = open(path, O_RDWR | O_CREAT, 0600); | ||
148 | if (log->fd < 0) { | ||
149 | free(log); | ||
150 | return NULL; | ||
151 | } | ||
152 | if (lock(log->fd)) { | ||
153 | close(log->fd); | ||
154 | free(log); | ||
155 | return NULL; | ||
156 | } | ||
157 | |||
158 | /* if file is empty, write new gzip stream */ | ||
159 | if (lseek(log->fd, 0, SEEK_END) == 0) { | ||
160 | if (write(log->fd, empty_gz, sizeof(empty_gz)) != sizeof(empty_gz)) { | ||
161 | log_clean(log); | ||
162 | return NULL; | ||
163 | } | ||
164 | } | ||
165 | |||
166 | /* check gzip header */ | ||
167 | (void)lseek(log->fd, 0, SEEK_SET); | ||
168 | if (read(log->fd, temp, 12) != 12 || temp[0] != 0x1f || | ||
169 | temp[1] != 0x8b || temp[2] != 8 || (temp[3] & 4) == 0) { | ||
170 | log_clean(log); | ||
171 | return NULL; | ||
172 | } | ||
173 | |||
174 | /* process extra field to find "ap" sub-field */ | ||
175 | xlen = temp[10] + (temp[11] << 8); | ||
176 | good = 0; | ||
177 | while (xlen) { | ||
178 | if (xlen < 4 || read(log->fd, temp, 4) != 4) | ||
179 | break; | ||
180 | sub_len = temp[2]; | ||
181 | sub_len += temp[3] << 8; | ||
182 | xlen -= 4; | ||
183 | if (memcmp(temp, "ap", 2) == 0 && sub_len == 16) { | ||
184 | good = 1; | ||
185 | break; | ||
186 | } | ||
187 | if (xlen < sub_len) | ||
188 | break; | ||
189 | (void)lseek(log->fd, sub_len, SEEK_CUR); | ||
190 | xlen -= sub_len; | ||
191 | } | ||
192 | if (!good) { | ||
193 | log_clean(log); | ||
194 | return NULL; | ||
195 | } | ||
196 | |||
197 | /* read in "ap" sub-field */ | ||
198 | log->extra = lseek(log->fd, 0, SEEK_CUR); | ||
199 | if (read(log->fd, temp, 16) != 16) { | ||
200 | log_clean(log); | ||
201 | return NULL; | ||
202 | } | ||
203 | log->mark_off = make_off(temp); | ||
204 | log->last_off = make_off(temp + 8); | ||
205 | |||
206 | /* get crc, length of gzip file */ | ||
207 | (void)lseek(log->fd, log->last_off, SEEK_SET); | ||
208 | if (read(log->fd, temp, 13) != 13 || | ||
209 | memcmp(temp, "\001\000\000\377\377", 5) != 0) { | ||
210 | log_clean(log); | ||
211 | return NULL; | ||
212 | } | ||
213 | log->crc = make_ulg(temp + 5); | ||
214 | log->len = make_ulg(temp + 9); | ||
215 | |||
216 | /* set up to write over empty last block */ | ||
217 | (void)lseek(log->fd, log->last_off + 5, SEEK_SET); | ||
218 | log->stored = 0; | ||
219 | return (void *)log; | ||
220 | } | ||
221 | |||
222 | /* maximum amount to put in a stored block before starting a new one */ | ||
223 | #define MAX_BLOCK 16384 | ||
224 | |||
225 | /* write a block to a log object */ | ||
226 | int gzlog_write(void *obj, char *data, size_t len) | ||
227 | { | ||
228 | size_t some; | ||
229 | unsigned char temp[5]; | ||
230 | gz_log *log; | ||
231 | |||
232 | /* check object */ | ||
233 | log = (gz_log *)obj; | ||
234 | if (log == NULL || log->id != GZLOGID) | ||
235 | return 1; | ||
236 | |||
237 | /* write stored blocks until all of the input is written */ | ||
238 | do { | ||
239 | some = MAX_BLOCK - log->stored; | ||
240 | if (some > len) | ||
241 | some = len; | ||
242 | if (write(log->fd, data, some) != some) | ||
243 | return 1; | ||
244 | log->crc = crc32(log->crc, data, some); | ||
245 | log->len += some; | ||
246 | len -= some; | ||
247 | data += some; | ||
248 | log->stored += some; | ||
249 | |||
250 | /* if the stored block is full, end it and start another */ | ||
251 | if (log->stored == MAX_BLOCK) { | ||
252 | (void)lseek(log->fd, log->last_off, SEEK_SET); | ||
253 | temp[0] = 0; | ||
254 | dice_ulg(log->stored + ((unsigned long)(~log->stored) << 16), | ||
255 | temp + 1); | ||
256 | if (write(log->fd, temp, 5) != 5) | ||
257 | return 1; | ||
258 | log->last_off = lseek(log->fd, log->stored, SEEK_CUR); | ||
259 | (void)lseek(log->fd, 5, SEEK_CUR); | ||
260 | log->stored = 0; | ||
261 | } | ||
262 | } while (len); | ||
263 | return 0; | ||
264 | } | ||
265 | |||
266 | /* recompress the remaining stored deflate data in place */ | ||
267 | local int recomp(gz_log *log) | ||
268 | { | ||
269 | z_stream strm; | ||
270 | size_t len, max; | ||
271 | unsigned char *in; | ||
272 | unsigned char *out; | ||
273 | unsigned char temp[16]; | ||
274 | |||
275 | /* allocate space and read it all in (it's around 1 MB) */ | ||
276 | len = log->last_off - log->mark_off; | ||
277 | max = len + (len >> 12) + (len >> 14) + 11; | ||
278 | out = malloc(max); | ||
279 | if (out == NULL) | ||
280 | return 1; | ||
281 | in = malloc(len); | ||
282 | if (in == NULL) { | ||
283 | free(out); | ||
284 | return 1; | ||
285 | } | ||
286 | (void)lseek(log->fd, log->mark_off, SEEK_SET); | ||
287 | if (read(log->fd, in, len) != len) { | ||
288 | free(in); | ||
289 | free(out); | ||
290 | return 1; | ||
291 | } | ||
292 | |||
293 | /* recompress in memory, decoding stored data as we go */ | ||
294 | /* note: this assumes that unsigned is four bytes or more */ | ||
295 | /* consider not making that assumption */ | ||
296 | strm.zalloc = Z_NULL; | ||
297 | strm.zfree = Z_NULL; | ||
298 | strm.opaque = Z_NULL; | ||
299 | if (deflateInit2(&strm, Z_BEST_COMPRESSION, Z_DEFLATED, -15, 8, | ||
300 | Z_DEFAULT_STRATEGY) != Z_OK) { | ||
301 | free(in); | ||
302 | free(out); | ||
303 | return 1; | ||
304 | } | ||
305 | strm.next_in = in; | ||
306 | strm.avail_out = max; | ||
307 | strm.next_out = out; | ||
308 | while (len >= 5) { | ||
309 | if (strm.next_in[0] != 0) | ||
310 | break; | ||
311 | strm.avail_in = strm.next_in[1] + (strm.next_in[2] << 8); | ||
312 | strm.next_in += 5; | ||
313 | len -= 5; | ||
314 | if (strm.avail_in != 0) { | ||
315 | if (len < strm.avail_in) | ||
316 | break; | ||
317 | len -= strm.avail_in; | ||
318 | (void)deflate(&strm, Z_NO_FLUSH); | ||
319 | if (strm.avail_in != 0 || strm.avail_out == 0) | ||
320 | break; | ||
321 | } | ||
322 | } | ||
323 | (void)deflate(&strm, Z_SYNC_FLUSH); | ||
324 | (void)deflateEnd(&strm); | ||
325 | free(in); | ||
326 | if (len != 0 || strm.avail_out == 0) { | ||
327 | free(out); | ||
328 | return 1; | ||
329 | } | ||
330 | |||
331 | /* overwrite stored data with compressed data */ | ||
332 | (void)lseek(log->fd, log->mark_off, SEEK_SET); | ||
333 | len = max - strm.avail_out; | ||
334 | if (write(log->fd, out, len) != len) { | ||
335 | free(out); | ||
336 | return 1; | ||
337 | } | ||
338 | free(out); | ||
339 | |||
340 | /* write last empty block, crc, and length */ | ||
341 | log->mark_off = log->last_off = lseek(log->fd, 0, SEEK_CUR); | ||
342 | temp[0] = 1; | ||
343 | dice_ulg(0xffffL << 16, temp + 1); | ||
344 | dice_ulg(log->crc, temp + 5); | ||
345 | dice_ulg(log->len, temp + 9); | ||
346 | if (write(log->fd, temp, 13) != 13) | ||
347 | return 1; | ||
348 | |||
349 | /* truncate file to discard remaining stored data and old trailer */ | ||
350 | ftruncate(log->fd, lseek(log->fd, 0, SEEK_CUR)); | ||
351 | |||
352 | /* update extra field to point to new last empty block */ | ||
353 | (void)lseek(log->fd, log->extra, SEEK_SET); | ||
354 | dice_off(log->mark_off, temp); | ||
355 | dice_off(log->last_off, temp + 8); | ||
356 | if (write(log->fd, temp, 16) != 16) | ||
357 | return 1; | ||
358 | return 0; | ||
359 | } | ||
360 | |||
361 | /* maximum accumulation of stored blocks before compressing */ | ||
362 | #define MAX_STORED 1048576 | ||
363 | |||
364 | /* close log object */ | ||
365 | int gzlog_close(void *obj) | ||
366 | { | ||
367 | unsigned char temp[8]; | ||
368 | gz_log *log; | ||
369 | |||
370 | /* check object */ | ||
371 | log = (gz_log *)obj; | ||
372 | if (log == NULL || log->id != GZLOGID) | ||
373 | return 1; | ||
374 | |||
375 | /* go to start of most recent block being written */ | ||
376 | (void)lseek(log->fd, log->last_off, SEEK_SET); | ||
377 | |||
378 | /* if some stuff was put there, update block */ | ||
379 | if (log->stored) { | ||
380 | temp[0] = 0; | ||
381 | dice_ulg(log->stored + ((unsigned long)(~log->stored) << 16), | ||
382 | temp + 1); | ||
383 | if (write(log->fd, temp, 5) != 5) | ||
384 | return 1; | ||
385 | log->last_off = lseek(log->fd, log->stored, SEEK_CUR); | ||
386 | } | ||
387 | |||
388 | /* write last block (empty) */ | ||
389 | if (write(log->fd, "\001\000\000\377\377", 5) != 5) | ||
390 | return 1; | ||
391 | |||
392 | /* write updated crc and uncompressed length */ | ||
393 | dice_ulg(log->crc, temp); | ||
394 | dice_ulg(log->len, temp + 4); | ||
395 | if (write(log->fd, temp, 8) != 8) | ||
396 | return 1; | ||
397 | |||
398 | /* put offset of that last block in gzip extra block */ | ||
399 | (void)lseek(log->fd, log->extra + 8, SEEK_SET); | ||
400 | dice_off(log->last_off, temp); | ||
401 | if (write(log->fd, temp, 8) != 8) | ||
402 | return 1; | ||
403 | |||
404 | /* if more than 1 MB stored, then time to compress it */ | ||
405 | if (log->last_off - log->mark_off > MAX_STORED) { | ||
406 | if (recomp(log)) | ||
407 | return 1; | ||
408 | } | ||
409 | |||
410 | /* unlock and close file */ | ||
411 | log_clean(log); | ||
412 | return 0; | ||
413 | } | ||