diff options
author | Igor Pavlov <87184205+ip7z@users.noreply.github.com> | 2021-12-27 00:00:00 +0000 |
---|---|---|
committer | Igor Pavlov <87184205+ip7z@users.noreply.github.com> | 2022-03-18 15:35:13 +0500 |
commit | f19f813537c7aea1c20749c914e756b54a9c3cf5 (patch) | |
tree | 816ba62ca7c0fa19f2eb46d9e9d6f7dd7c3a744d /C/Ppmd8.c | |
parent | 98e06a519b63b81986abe76d28887f6984a7732b (diff) | |
download | 7zip-f19f813537c7aea1c20749c914e756b54a9c3cf5.tar.gz 7zip-f19f813537c7aea1c20749c914e756b54a9c3cf5.tar.bz2 7zip-f19f813537c7aea1c20749c914e756b54a9c3cf5.zip |
'21.07'21.07
Diffstat (limited to 'C/Ppmd8.c')
-rw-r--r-- | C/Ppmd8.c | 1537 |
1 files changed, 1537 insertions, 0 deletions
diff --git a/C/Ppmd8.c b/C/Ppmd8.c new file mode 100644 index 0000000..fda8b88 --- /dev/null +++ b/C/Ppmd8.c | |||
@@ -0,0 +1,1537 @@ | |||
1 | /* Ppmd8.c -- PPMdI codec | ||
2 | 2021-04-13 : Igor Pavlov : Public domain | ||
3 | This code is based on PPMd var.I (2002): Dmitry Shkarin : Public domain */ | ||
4 | |||
5 | #include "Precomp.h" | ||
6 | |||
7 | #include <string.h> | ||
8 | |||
9 | #include "Ppmd8.h" | ||
10 | |||
11 | |||
12 | |||
13 | |||
14 | MY_ALIGN(16) | ||
15 | static const Byte PPMD8_kExpEscape[16] = { 25, 14, 9, 7, 5, 5, 4, 4, 4, 3, 3, 3, 2, 2, 2, 2 }; | ||
16 | MY_ALIGN(16) | ||
17 | static const UInt16 kInitBinEsc[] = { 0x3CDD, 0x1F3F, 0x59BF, 0x48F3, 0x64A1, 0x5ABC, 0x6632, 0x6051}; | ||
18 | |||
19 | #define MAX_FREQ 124 | ||
20 | #define UNIT_SIZE 12 | ||
21 | |||
22 | #define U2B(nu) ((UInt32)(nu) * UNIT_SIZE) | ||
23 | #define U2I(nu) (p->Units2Indx[(size_t)(nu) - 1]) | ||
24 | #define I2U(indx) ((unsigned)p->Indx2Units[indx]) | ||
25 | |||
26 | |||
27 | #define REF(ptr) Ppmd_GetRef(p, ptr) | ||
28 | |||
29 | #define STATS_REF(ptr) ((CPpmd_State_Ref)REF(ptr)) | ||
30 | |||
31 | #define CTX(ref) ((CPpmd8_Context *)Ppmd8_GetContext(p, ref)) | ||
32 | #define STATS(ctx) Ppmd8_GetStats(p, ctx) | ||
33 | #define ONE_STATE(ctx) Ppmd8Context_OneState(ctx) | ||
34 | #define SUFFIX(ctx) CTX((ctx)->Suffix) | ||
35 | |||
36 | typedef CPpmd8_Context * CTX_PTR; | ||
37 | |||
38 | struct CPpmd8_Node_; | ||
39 | |||
40 | typedef Ppmd_Ref_Type(struct CPpmd8_Node_) CPpmd8_Node_Ref; | ||
41 | |||
42 | typedef struct CPpmd8_Node_ | ||
43 | { | ||
44 | UInt32 Stamp; | ||
45 | |||
46 | CPpmd8_Node_Ref Next; | ||
47 | UInt32 NU; | ||
48 | } CPpmd8_Node; | ||
49 | |||
50 | #define NODE(r) Ppmd_GetPtr_Type(p, r, CPpmd8_Node) | ||
51 | |||
52 | void Ppmd8_Construct(CPpmd8 *p) | ||
53 | { | ||
54 | unsigned i, k, m; | ||
55 | |||
56 | p->Base = NULL; | ||
57 | |||
58 | for (i = 0, k = 0; i < PPMD_NUM_INDEXES; i++) | ||
59 | { | ||
60 | unsigned step = (i >= 12 ? 4 : (i >> 2) + 1); | ||
61 | do { p->Units2Indx[k++] = (Byte)i; } while (--step); | ||
62 | p->Indx2Units[i] = (Byte)k; | ||
63 | } | ||
64 | |||
65 | p->NS2BSIndx[0] = (0 << 1); | ||
66 | p->NS2BSIndx[1] = (1 << 1); | ||
67 | memset(p->NS2BSIndx + 2, (2 << 1), 9); | ||
68 | memset(p->NS2BSIndx + 11, (3 << 1), 256 - 11); | ||
69 | |||
70 | for (i = 0; i < 5; i++) | ||
71 | p->NS2Indx[i] = (Byte)i; | ||
72 | |||
73 | for (m = i, k = 1; i < 260; i++) | ||
74 | { | ||
75 | p->NS2Indx[i] = (Byte)m; | ||
76 | if (--k == 0) | ||
77 | k = (++m) - 4; | ||
78 | } | ||
79 | |||
80 | memcpy(p->ExpEscape, PPMD8_kExpEscape, 16); | ||
81 | } | ||
82 | |||
83 | |||
84 | void Ppmd8_Free(CPpmd8 *p, ISzAllocPtr alloc) | ||
85 | { | ||
86 | ISzAlloc_Free(alloc, p->Base); | ||
87 | p->Size = 0; | ||
88 | p->Base = NULL; | ||
89 | } | ||
90 | |||
91 | |||
92 | BoolInt Ppmd8_Alloc(CPpmd8 *p, UInt32 size, ISzAllocPtr alloc) | ||
93 | { | ||
94 | if (!p->Base || p->Size != size) | ||
95 | { | ||
96 | Ppmd8_Free(p, alloc); | ||
97 | p->AlignOffset = (4 - size) & 3; | ||
98 | if ((p->Base = (Byte *)ISzAlloc_Alloc(alloc, p->AlignOffset + size)) == NULL) | ||
99 | return False; | ||
100 | p->Size = size; | ||
101 | } | ||
102 | return True; | ||
103 | } | ||
104 | |||
105 | |||
106 | |||
107 | // ---------- Internal Memory Allocator ---------- | ||
108 | |||
109 | |||
110 | |||
111 | |||
112 | |||
113 | |||
114 | #define EMPTY_NODE 0xFFFFFFFF | ||
115 | |||
116 | |||
117 | static void InsertNode(CPpmd8 *p, void *node, unsigned indx) | ||
118 | { | ||
119 | ((CPpmd8_Node *)node)->Stamp = EMPTY_NODE; | ||
120 | ((CPpmd8_Node *)node)->Next = (CPpmd8_Node_Ref)p->FreeList[indx]; | ||
121 | ((CPpmd8_Node *)node)->NU = I2U(indx); | ||
122 | p->FreeList[indx] = REF(node); | ||
123 | p->Stamps[indx]++; | ||
124 | } | ||
125 | |||
126 | |||
127 | static void *RemoveNode(CPpmd8 *p, unsigned indx) | ||
128 | { | ||
129 | CPpmd8_Node *node = NODE((CPpmd8_Node_Ref)p->FreeList[indx]); | ||
130 | p->FreeList[indx] = node->Next; | ||
131 | p->Stamps[indx]--; | ||
132 | |||
133 | return node; | ||
134 | } | ||
135 | |||
136 | |||
137 | static void SplitBlock(CPpmd8 *p, void *ptr, unsigned oldIndx, unsigned newIndx) | ||
138 | { | ||
139 | unsigned i, nu = I2U(oldIndx) - I2U(newIndx); | ||
140 | ptr = (Byte *)ptr + U2B(I2U(newIndx)); | ||
141 | if (I2U(i = U2I(nu)) != nu) | ||
142 | { | ||
143 | unsigned k = I2U(--i); | ||
144 | InsertNode(p, ((Byte *)ptr) + U2B(k), nu - k - 1); | ||
145 | } | ||
146 | InsertNode(p, ptr, i); | ||
147 | } | ||
148 | |||
149 | |||
150 | |||
151 | |||
152 | |||
153 | |||
154 | |||
155 | |||
156 | |||
157 | |||
158 | |||
159 | |||
160 | |||
161 | |||
162 | static void GlueFreeBlocks(CPpmd8 *p) | ||
163 | { | ||
164 | /* | ||
165 | we use first UInt32 field of 12-bytes UNITs as record type stamp | ||
166 | CPpmd_State { Byte Symbol; Byte Freq; : Freq != 0xFF | ||
167 | CPpmd8_Context { Byte NumStats; Byte Flags; UInt16 SummFreq; : Flags != 0xFF ??? | ||
168 | CPpmd8_Node { UInt32 Stamp : Stamp == 0xFFFFFFFF for free record | ||
169 | : Stamp == 0 for guard | ||
170 | Last 12-bytes UNIT in array is always contains 12-bytes order-0 CPpmd8_Context record | ||
171 | */ | ||
172 | CPpmd8_Node_Ref n; | ||
173 | |||
174 | p->GlueCount = 1 << 13; | ||
175 | memset(p->Stamps, 0, sizeof(p->Stamps)); | ||
176 | |||
177 | /* we set guard NODE at LoUnit */ | ||
178 | if (p->LoUnit != p->HiUnit) | ||
179 | ((CPpmd8_Node *)(void *)p->LoUnit)->Stamp = 0; | ||
180 | |||
181 | { | ||
182 | /* Glue free blocks */ | ||
183 | CPpmd8_Node_Ref *prev = &n; | ||
184 | unsigned i; | ||
185 | for (i = 0; i < PPMD_NUM_INDEXES; i++) | ||
186 | { | ||
187 | |||
188 | CPpmd8_Node_Ref next = (CPpmd8_Node_Ref)p->FreeList[i]; | ||
189 | p->FreeList[i] = 0; | ||
190 | while (next != 0) | ||
191 | { | ||
192 | CPpmd8_Node *node = NODE(next); | ||
193 | UInt32 nu = node->NU; | ||
194 | *prev = next; | ||
195 | next = node->Next; | ||
196 | if (nu != 0) | ||
197 | { | ||
198 | CPpmd8_Node *node2; | ||
199 | prev = &(node->Next); | ||
200 | while ((node2 = node + nu)->Stamp == EMPTY_NODE) | ||
201 | { | ||
202 | nu += node2->NU; | ||
203 | node2->NU = 0; | ||
204 | node->NU = nu; | ||
205 | } | ||
206 | } | ||
207 | } | ||
208 | } | ||
209 | |||
210 | *prev = 0; | ||
211 | } | ||
212 | |||
213 | |||
214 | |||
215 | |||
216 | |||
217 | |||
218 | |||
219 | |||
220 | |||
221 | |||
222 | |||
223 | |||
224 | |||
225 | |||
226 | |||
227 | |||
228 | |||
229 | |||
230 | |||
231 | |||
232 | /* Fill lists of free blocks */ | ||
233 | while (n != 0) | ||
234 | { | ||
235 | CPpmd8_Node *node = NODE(n); | ||
236 | UInt32 nu = node->NU; | ||
237 | unsigned i; | ||
238 | n = node->Next; | ||
239 | if (nu == 0) | ||
240 | continue; | ||
241 | for (; nu > 128; nu -= 128, node += 128) | ||
242 | InsertNode(p, node, PPMD_NUM_INDEXES - 1); | ||
243 | if (I2U(i = U2I(nu)) != nu) | ||
244 | { | ||
245 | unsigned k = I2U(--i); | ||
246 | InsertNode(p, node + k, (unsigned)nu - k - 1); | ||
247 | } | ||
248 | InsertNode(p, node, i); | ||
249 | } | ||
250 | } | ||
251 | |||
252 | |||
253 | MY_NO_INLINE | ||
254 | static void *AllocUnitsRare(CPpmd8 *p, unsigned indx) | ||
255 | { | ||
256 | unsigned i; | ||
257 | |||
258 | if (p->GlueCount == 0) | ||
259 | { | ||
260 | GlueFreeBlocks(p); | ||
261 | if (p->FreeList[indx] != 0) | ||
262 | return RemoveNode(p, indx); | ||
263 | } | ||
264 | |||
265 | i = indx; | ||
266 | |||
267 | do | ||
268 | { | ||
269 | if (++i == PPMD_NUM_INDEXES) | ||
270 | { | ||
271 | UInt32 numBytes = U2B(I2U(indx)); | ||
272 | Byte *us = p->UnitsStart; | ||
273 | p->GlueCount--; | ||
274 | return ((UInt32)(us - p->Text) > numBytes) ? (p->UnitsStart = us - numBytes) : (NULL); | ||
275 | } | ||
276 | } | ||
277 | while (p->FreeList[i] == 0); | ||
278 | |||
279 | { | ||
280 | void *block = RemoveNode(p, i); | ||
281 | SplitBlock(p, block, i, indx); | ||
282 | return block; | ||
283 | } | ||
284 | } | ||
285 | |||
286 | |||
287 | static void *AllocUnits(CPpmd8 *p, unsigned indx) | ||
288 | { | ||
289 | if (p->FreeList[indx] != 0) | ||
290 | return RemoveNode(p, indx); | ||
291 | { | ||
292 | UInt32 numBytes = U2B(I2U(indx)); | ||
293 | Byte *lo = p->LoUnit; | ||
294 | if ((UInt32)(p->HiUnit - lo) >= numBytes) | ||
295 | { | ||
296 | p->LoUnit = lo + numBytes; | ||
297 | return lo; | ||
298 | } | ||
299 | } | ||
300 | return AllocUnitsRare(p, indx); | ||
301 | } | ||
302 | |||
303 | |||
304 | #define MyMem12Cpy(dest, src, num) \ | ||
305 | { UInt32 *d = (UInt32 *)dest; const UInt32 *z = (const UInt32 *)src; UInt32 n = num; \ | ||
306 | do { d[0] = z[0]; d[1] = z[1]; d[2] = z[2]; z += 3; d += 3; } while (--n); } | ||
307 | |||
308 | |||
309 | |||
310 | static void *ShrinkUnits(CPpmd8 *p, void *oldPtr, unsigned oldNU, unsigned newNU) | ||
311 | { | ||
312 | unsigned i0 = U2I(oldNU); | ||
313 | unsigned i1 = U2I(newNU); | ||
314 | if (i0 == i1) | ||
315 | return oldPtr; | ||
316 | if (p->FreeList[i1] != 0) | ||
317 | { | ||
318 | void *ptr = RemoveNode(p, i1); | ||
319 | MyMem12Cpy(ptr, oldPtr, newNU); | ||
320 | InsertNode(p, oldPtr, i0); | ||
321 | return ptr; | ||
322 | } | ||
323 | SplitBlock(p, oldPtr, i0, i1); | ||
324 | return oldPtr; | ||
325 | } | ||
326 | |||
327 | |||
328 | static void FreeUnits(CPpmd8 *p, void *ptr, unsigned nu) | ||
329 | { | ||
330 | InsertNode(p, ptr, U2I(nu)); | ||
331 | } | ||
332 | |||
333 | |||
334 | static void SpecialFreeUnit(CPpmd8 *p, void *ptr) | ||
335 | { | ||
336 | if ((Byte *)ptr != p->UnitsStart) | ||
337 | InsertNode(p, ptr, 0); | ||
338 | else | ||
339 | { | ||
340 | #ifdef PPMD8_FREEZE_SUPPORT | ||
341 | *(UInt32 *)ptr = EMPTY_NODE; /* it's used for (Flags == 0xFF) check in RemoveBinContexts() */ | ||
342 | #endif | ||
343 | p->UnitsStart += UNIT_SIZE; | ||
344 | } | ||
345 | } | ||
346 | |||
347 | |||
348 | /* | ||
349 | static void *MoveUnitsUp(CPpmd8 *p, void *oldPtr, unsigned nu) | ||
350 | { | ||
351 | unsigned indx = U2I(nu); | ||
352 | void *ptr; | ||
353 | if ((Byte *)oldPtr > p->UnitsStart + (1 << 14) || REF(oldPtr) > p->FreeList[indx]) | ||
354 | return oldPtr; | ||
355 | ptr = RemoveNode(p, indx); | ||
356 | MyMem12Cpy(ptr, oldPtr, nu); | ||
357 | if ((Byte *)oldPtr != p->UnitsStart) | ||
358 | InsertNode(p, oldPtr, indx); | ||
359 | else | ||
360 | p->UnitsStart += U2B(I2U(indx)); | ||
361 | return ptr; | ||
362 | } | ||
363 | */ | ||
364 | |||
365 | static void ExpandTextArea(CPpmd8 *p) | ||
366 | { | ||
367 | UInt32 count[PPMD_NUM_INDEXES]; | ||
368 | unsigned i; | ||
369 | |||
370 | memset(count, 0, sizeof(count)); | ||
371 | if (p->LoUnit != p->HiUnit) | ||
372 | ((CPpmd8_Node *)(void *)p->LoUnit)->Stamp = 0; | ||
373 | |||
374 | { | ||
375 | CPpmd8_Node *node = (CPpmd8_Node *)(void *)p->UnitsStart; | ||
376 | while (node->Stamp == EMPTY_NODE) | ||
377 | { | ||
378 | UInt32 nu = node->NU; | ||
379 | node->Stamp = 0; | ||
380 | count[U2I(nu)]++; | ||
381 | node += nu; | ||
382 | } | ||
383 | p->UnitsStart = (Byte *)node; | ||
384 | } | ||
385 | |||
386 | for (i = 0; i < PPMD_NUM_INDEXES; i++) | ||
387 | { | ||
388 | UInt32 cnt = count[i]; | ||
389 | if (cnt == 0) | ||
390 | continue; | ||
391 | { | ||
392 | CPpmd8_Node_Ref *prev = (CPpmd8_Node_Ref *)&p->FreeList[i]; | ||
393 | CPpmd8_Node_Ref n = *prev; | ||
394 | p->Stamps[i] -= cnt; | ||
395 | for (;;) | ||
396 | { | ||
397 | CPpmd8_Node *node = NODE(n); | ||
398 | n = node->Next; | ||
399 | if (node->Stamp != 0) | ||
400 | { | ||
401 | prev = &node->Next; | ||
402 | continue; | ||
403 | } | ||
404 | *prev = n; | ||
405 | if (--cnt == 0) | ||
406 | break; | ||
407 | } | ||
408 | } | ||
409 | } | ||
410 | } | ||
411 | |||
412 | |||
413 | #define SUCCESSOR(p) Ppmd_GET_SUCCESSOR(p) | ||
414 | static void SetSuccessor(CPpmd_State *p, CPpmd_Void_Ref v) | ||
415 | { | ||
416 | Ppmd_SET_SUCCESSOR(p, v); | ||
417 | } | ||
418 | |||
419 | #define RESET_TEXT(offs) { p->Text = p->Base + p->AlignOffset + (offs); } | ||
420 | |||
421 | MY_NO_INLINE | ||
422 | static | ||
423 | void RestartModel(CPpmd8 *p) | ||
424 | { | ||
425 | unsigned i, k, m; | ||
426 | |||
427 | memset(p->FreeList, 0, sizeof(p->FreeList)); | ||
428 | memset(p->Stamps, 0, sizeof(p->Stamps)); | ||
429 | RESET_TEXT(0); | ||
430 | p->HiUnit = p->Text + p->Size; | ||
431 | p->LoUnit = p->UnitsStart = p->HiUnit - p->Size / 8 / UNIT_SIZE * 7 * UNIT_SIZE; | ||
432 | p->GlueCount = 0; | ||
433 | |||
434 | p->OrderFall = p->MaxOrder; | ||
435 | p->RunLength = p->InitRL = -(Int32)((p->MaxOrder < 12) ? p->MaxOrder : 12) - 1; | ||
436 | p->PrevSuccess = 0; | ||
437 | |||
438 | { | ||
439 | CPpmd8_Context *mc = (CTX_PTR)(void *)(p->HiUnit -= UNIT_SIZE); /* AllocContext(p); */ | ||
440 | CPpmd_State *s = (CPpmd_State *)p->LoUnit; /* AllocUnits(p, PPMD_NUM_INDEXES - 1); */ | ||
441 | |||
442 | p->LoUnit += U2B(256 / 2); | ||
443 | p->MaxContext = p->MinContext = mc; | ||
444 | p->FoundState = s; | ||
445 | mc->Flags = 0; | ||
446 | mc->NumStats = 256 - 1; | ||
447 | mc->Union2.SummFreq = 256 + 1; | ||
448 | mc->Union4.Stats = REF(s); | ||
449 | mc->Suffix = 0; | ||
450 | |||
451 | for (i = 0; i < 256; i++, s++) | ||
452 | { | ||
453 | s->Symbol = (Byte)i; | ||
454 | s->Freq = 1; | ||
455 | SetSuccessor(s, 0); | ||
456 | } | ||
457 | } | ||
458 | |||
459 | |||
460 | |||
461 | |||
462 | |||
463 | |||
464 | |||
465 | |||
466 | |||
467 | |||
468 | |||
469 | |||
470 | for (i = m = 0; m < 25; m++) | ||
471 | { | ||
472 | while (p->NS2Indx[i] == m) | ||
473 | i++; | ||
474 | for (k = 0; k < 8; k++) | ||
475 | { | ||
476 | unsigned r; | ||
477 | UInt16 *dest = p->BinSumm[m] + k; | ||
478 | UInt16 val = (UInt16)(PPMD_BIN_SCALE - kInitBinEsc[k] / (i + 1)); | ||
479 | for (r = 0; r < 64; r += 8) | ||
480 | dest[r] = val; | ||
481 | } | ||
482 | } | ||
483 | |||
484 | for (i = m = 0; m < 24; m++) | ||
485 | { | ||
486 | unsigned summ; | ||
487 | CPpmd_See *s; | ||
488 | while (p->NS2Indx[(size_t)i + 3] == m + 3) | ||
489 | i++; | ||
490 | s = p->See[m]; | ||
491 | summ = ((2 * i + 5) << (PPMD_PERIOD_BITS - 4)); | ||
492 | for (k = 0; k < 32; k++, s++) | ||
493 | { | ||
494 | s->Summ = (UInt16)summ; | ||
495 | s->Shift = (PPMD_PERIOD_BITS - 4); | ||
496 | s->Count = 7; | ||
497 | } | ||
498 | } | ||
499 | |||
500 | p->DummySee.Summ = 0; /* unused */ | ||
501 | p->DummySee.Shift = PPMD_PERIOD_BITS; | ||
502 | p->DummySee.Count = 64; /* unused */ | ||
503 | } | ||
504 | |||
505 | |||
506 | void Ppmd8_Init(CPpmd8 *p, unsigned maxOrder, unsigned restoreMethod) | ||
507 | { | ||
508 | p->MaxOrder = maxOrder; | ||
509 | p->RestoreMethod = restoreMethod; | ||
510 | RestartModel(p); | ||
511 | } | ||
512 | |||
513 | |||
514 | #define FLAG_RESCALED (1 << 2) | ||
515 | // #define FLAG_SYM_HIGH (1 << 3) | ||
516 | #define FLAG_PREV_HIGH (1 << 4) | ||
517 | |||
518 | #define HiBits_Prepare(sym) ((unsigned)(sym) + 0xC0) | ||
519 | |||
520 | #define HiBits_Convert_3(flags) (((flags) >> (8 - 3)) & (1 << 3)) | ||
521 | #define HiBits_Convert_4(flags) (((flags) >> (8 - 4)) & (1 << 4)) | ||
522 | |||
523 | #define PPMD8_HiBitsFlag_3(sym) HiBits_Convert_3(HiBits_Prepare(sym)) | ||
524 | #define PPMD8_HiBitsFlag_4(sym) HiBits_Convert_4(HiBits_Prepare(sym)) | ||
525 | |||
526 | // #define PPMD8_HiBitsFlag_3(sym) (0x08 * ((sym) >= 0x40)) | ||
527 | // #define PPMD8_HiBitsFlag_4(sym) (0x10 * ((sym) >= 0x40)) | ||
528 | |||
529 | /* | ||
530 | Refresh() is called when we remove some symbols (successors) in context. | ||
531 | It increases Escape_Freq for sum of all removed symbols. | ||
532 | */ | ||
533 | |||
534 | static void Refresh(CPpmd8 *p, CTX_PTR ctx, unsigned oldNU, unsigned scale) | ||
535 | { | ||
536 | unsigned i = ctx->NumStats, escFreq, sumFreq, flags; | ||
537 | CPpmd_State *s = (CPpmd_State *)ShrinkUnits(p, STATS(ctx), oldNU, (i + 2) >> 1); | ||
538 | ctx->Union4.Stats = REF(s); | ||
539 | |||
540 | // #ifdef PPMD8_FREEZE_SUPPORT | ||
541 | /* | ||
542 | (ctx->Union2.SummFreq >= ((UInt32)1 << 15)) can be in FREEZE mode for some files. | ||
543 | It's not good for range coder. So new versions of support fix: | ||
544 | - original PPMdI code rev.1 | ||
545 | + original PPMdI code rev.2 | ||
546 | - 7-Zip default ((PPMD8_FREEZE_SUPPORT is not defined) | ||
547 | + 7-Zip (p->RestoreMethod >= PPMD8_RESTORE_METHOD_FREEZE) | ||
548 | if we use that fixed line, we can lose compatibility with some files created before fix | ||
549 | if we don't use that fixed line, the program can work incorrectly in FREEZE mode in rare case. | ||
550 | */ | ||
551 | // if (p->RestoreMethod >= PPMD8_RESTORE_METHOD_FREEZE) | ||
552 | { | ||
553 | scale |= (ctx->Union2.SummFreq >= ((UInt32)1 << 15)); | ||
554 | } | ||
555 | // #endif | ||
556 | |||
557 | |||
558 | |||
559 | flags = HiBits_Prepare(s->Symbol); | ||
560 | { | ||
561 | unsigned freq = s->Freq; | ||
562 | escFreq = ctx->Union2.SummFreq - freq; | ||
563 | freq = (freq + scale) >> scale; | ||
564 | sumFreq = freq; | ||
565 | s->Freq = (Byte)freq; | ||
566 | } | ||
567 | |||
568 | do | ||
569 | { | ||
570 | unsigned freq = (++s)->Freq; | ||
571 | escFreq -= freq; | ||
572 | freq = (freq + scale) >> scale; | ||
573 | sumFreq += freq; | ||
574 | s->Freq = (Byte)freq; | ||
575 | flags |= HiBits_Prepare(s->Symbol); | ||
576 | } | ||
577 | while (--i); | ||
578 | |||
579 | ctx->Union2.SummFreq = (UInt16)(sumFreq + ((escFreq + scale) >> scale)); | ||
580 | ctx->Flags = (Byte)((ctx->Flags & (FLAG_PREV_HIGH + FLAG_RESCALED * scale)) + HiBits_Convert_3(flags)); | ||
581 | } | ||
582 | |||
583 | |||
584 | static void SwapStates(CPpmd_State *t1, CPpmd_State *t2) | ||
585 | { | ||
586 | CPpmd_State tmp = *t1; | ||
587 | *t1 = *t2; | ||
588 | *t2 = tmp; | ||
589 | } | ||
590 | |||
591 | |||
592 | /* | ||
593 | CutOff() reduces contexts: | ||
594 | It conversts Successors at MaxOrder to another Contexts to NULL-Successors | ||
595 | It removes RAW-Successors and NULL-Successors that are not Order-0 | ||
596 | and it removes contexts when it has no Successors. | ||
597 | if the (Union4.Stats) is close to (UnitsStart), it moves it up. | ||
598 | */ | ||
599 | |||
600 | static CPpmd_Void_Ref CutOff(CPpmd8 *p, CTX_PTR ctx, unsigned order) | ||
601 | { | ||
602 | int ns = ctx->NumStats; | ||
603 | unsigned nu; | ||
604 | CPpmd_State *stats; | ||
605 | |||
606 | if (ns == 0) | ||
607 | { | ||
608 | CPpmd_State *s = ONE_STATE(ctx); | ||
609 | CPpmd_Void_Ref successor = SUCCESSOR(s); | ||
610 | if ((Byte *)Ppmd8_GetPtr(p, successor) >= p->UnitsStart) | ||
611 | { | ||
612 | if (order < p->MaxOrder) | ||
613 | successor = CutOff(p, CTX(successor), order + 1); | ||
614 | else | ||
615 | successor = 0; | ||
616 | SetSuccessor(s, successor); | ||
617 | if (successor || order <= 9) /* O_BOUND */ | ||
618 | return REF(ctx); | ||
619 | } | ||
620 | SpecialFreeUnit(p, ctx); | ||
621 | return 0; | ||
622 | } | ||
623 | |||
624 | nu = ((unsigned)ns + 2) >> 1; | ||
625 | // ctx->Union4.Stats = STATS_REF(MoveUnitsUp(p, STATS(ctx), nu)); | ||
626 | { | ||
627 | unsigned indx = U2I(nu); | ||
628 | stats = STATS(ctx); | ||
629 | |||
630 | if ((UInt32)((Byte *)stats - p->UnitsStart) <= (1 << 14) | ||
631 | && (CPpmd_Void_Ref)ctx->Union4.Stats <= p->FreeList[indx]) | ||
632 | { | ||
633 | void *ptr = RemoveNode(p, indx); | ||
634 | ctx->Union4.Stats = STATS_REF(ptr); | ||
635 | MyMem12Cpy(ptr, (const void *)stats, nu); | ||
636 | if ((Byte *)stats != p->UnitsStart) | ||
637 | InsertNode(p, stats, indx); | ||
638 | else | ||
639 | p->UnitsStart += U2B(I2U(indx)); | ||
640 | stats = ptr; | ||
641 | } | ||
642 | } | ||
643 | |||
644 | { | ||
645 | CPpmd_State *s = stats + (unsigned)ns; | ||
646 | do | ||
647 | { | ||
648 | CPpmd_Void_Ref successor = SUCCESSOR(s); | ||
649 | if ((Byte *)Ppmd8_GetPtr(p, successor) < p->UnitsStart) | ||
650 | { | ||
651 | CPpmd_State *s2 = stats + (unsigned)(ns--); | ||
652 | if (order) | ||
653 | { | ||
654 | if (s != s2) | ||
655 | *s = *s2; | ||
656 | } | ||
657 | else | ||
658 | { | ||
659 | SwapStates(s, s2); | ||
660 | SetSuccessor(s2, 0); | ||
661 | } | ||
662 | } | ||
663 | else | ||
664 | { | ||
665 | if (order < p->MaxOrder) | ||
666 | SetSuccessor(s, CutOff(p, CTX(successor), order + 1)); | ||
667 | else | ||
668 | SetSuccessor(s, 0); | ||
669 | } | ||
670 | } | ||
671 | while (--s >= stats); | ||
672 | } | ||
673 | |||
674 | if (ns != ctx->NumStats && order) | ||
675 | { | ||
676 | if (ns < 0) | ||
677 | { | ||
678 | FreeUnits(p, stats, nu); | ||
679 | SpecialFreeUnit(p, ctx); | ||
680 | return 0; | ||
681 | } | ||
682 | ctx->NumStats = (Byte)ns; | ||
683 | if (ns == 0) | ||
684 | { | ||
685 | const Byte sym = stats->Symbol; | ||
686 | ctx->Flags = (Byte)((ctx->Flags & FLAG_PREV_HIGH) + PPMD8_HiBitsFlag_3(sym)); | ||
687 | // *ONE_STATE(ctx) = *stats; | ||
688 | ctx->Union2.State2.Symbol = sym; | ||
689 | ctx->Union2.State2.Freq = (Byte)(((unsigned)stats->Freq + 11) >> 3); | ||
690 | ctx->Union4.State4.Successor_0 = stats->Successor_0; | ||
691 | ctx->Union4.State4.Successor_1 = stats->Successor_1; | ||
692 | FreeUnits(p, stats, nu); | ||
693 | } | ||
694 | else | ||
695 | { | ||
696 | Refresh(p, ctx, nu, ctx->Union2.SummFreq > 16 * (unsigned)ns); | ||
697 | } | ||
698 | } | ||
699 | |||
700 | return REF(ctx); | ||
701 | } | ||
702 | |||
703 | |||
704 | |||
705 | #ifdef PPMD8_FREEZE_SUPPORT | ||
706 | |||
707 | /* | ||
708 | RemoveBinContexts() | ||
709 | It conversts Successors at MaxOrder to another Contexts to NULL-Successors | ||
710 | It changes RAW-Successors to NULL-Successors | ||
711 | removes Bin Context without Successor, if suffix of that context is also binary. | ||
712 | */ | ||
713 | |||
714 | static CPpmd_Void_Ref RemoveBinContexts(CPpmd8 *p, CTX_PTR ctx, unsigned order) | ||
715 | { | ||
716 | if (!ctx->NumStats) | ||
717 | { | ||
718 | CPpmd_State *s = ONE_STATE(ctx); | ||
719 | CPpmd_Void_Ref successor = SUCCESSOR(s); | ||
720 | if ((Byte *)Ppmd8_GetPtr(p, successor) >= p->UnitsStart && order < p->MaxOrder) | ||
721 | successor = RemoveBinContexts(p, CTX(successor), order + 1); | ||
722 | else | ||
723 | successor = 0; | ||
724 | SetSuccessor(s, successor); | ||
725 | /* Suffix context can be removed already, since different (high-order) | ||
726 | Successors may refer to same context. So we check Flags == 0xFF (Stamp == EMPTY_NODE) */ | ||
727 | if (!successor && (!SUFFIX(ctx)->NumStats || SUFFIX(ctx)->Flags == 0xFF)) | ||
728 | { | ||
729 | FreeUnits(p, ctx, 1); | ||
730 | return 0; | ||
731 | } | ||
732 | } | ||
733 | else | ||
734 | { | ||
735 | CPpmd_State *s = STATS(ctx) + ctx->NumStats; | ||
736 | do | ||
737 | { | ||
738 | CPpmd_Void_Ref successor = SUCCESSOR(s); | ||
739 | if ((Byte *)Ppmd8_GetPtr(p, successor) >= p->UnitsStart && order < p->MaxOrder) | ||
740 | SetSuccessor(s, RemoveBinContexts(p, CTX(successor), order + 1)); | ||
741 | else | ||
742 | SetSuccessor(s, 0); | ||
743 | } | ||
744 | while (--s >= STATS(ctx)); | ||
745 | } | ||
746 | |||
747 | return REF(ctx); | ||
748 | } | ||
749 | |||
750 | #endif | ||
751 | |||
752 | |||
753 | |||
754 | static UInt32 GetUsedMemory(const CPpmd8 *p) | ||
755 | { | ||
756 | UInt32 v = 0; | ||
757 | unsigned i; | ||
758 | for (i = 0; i < PPMD_NUM_INDEXES; i++) | ||
759 | v += p->Stamps[i] * I2U(i); | ||
760 | return p->Size - (UInt32)(p->HiUnit - p->LoUnit) - (UInt32)(p->UnitsStart - p->Text) - U2B(v); | ||
761 | } | ||
762 | |||
763 | #ifdef PPMD8_FREEZE_SUPPORT | ||
764 | #define RESTORE_MODEL(c1, fSuccessor) RestoreModel(p, c1, fSuccessor) | ||
765 | #else | ||
766 | #define RESTORE_MODEL(c1, fSuccessor) RestoreModel(p, c1) | ||
767 | #endif | ||
768 | |||
769 | |||
770 | static void RestoreModel(CPpmd8 *p, CTX_PTR ctxError | ||
771 | #ifdef PPMD8_FREEZE_SUPPORT | ||
772 | , CTX_PTR fSuccessor | ||
773 | #endif | ||
774 | ) | ||
775 | { | ||
776 | CTX_PTR c; | ||
777 | CPpmd_State *s; | ||
778 | RESET_TEXT(0); | ||
779 | |||
780 | // we go here in cases of error of allocation for context (c1) | ||
781 | // Order(MinContext) < Order(ctxError) <= Order(MaxContext) | ||
782 | |||
783 | // We remove last symbol from each of contexts [p->MaxContext ... ctxError) contexts | ||
784 | // So we rollback all created (symbols) before error. | ||
785 | for (c = p->MaxContext; c != ctxError; c = SUFFIX(c)) | ||
786 | if (--(c->NumStats) == 0) | ||
787 | { | ||
788 | s = STATS(c); | ||
789 | c->Flags = (Byte)((c->Flags & FLAG_PREV_HIGH) + PPMD8_HiBitsFlag_3(s->Symbol)); | ||
790 | // *ONE_STATE(c) = *s; | ||
791 | c->Union2.State2.Symbol = s->Symbol; | ||
792 | c->Union2.State2.Freq = (Byte)(((unsigned)s->Freq + 11) >> 3); | ||
793 | c->Union4.State4.Successor_0 = s->Successor_0; | ||
794 | c->Union4.State4.Successor_1 = s->Successor_1; | ||
795 | |||
796 | SpecialFreeUnit(p, s); | ||
797 | } | ||
798 | else | ||
799 | { | ||
800 | /* Refresh() can increase Escape_Freq on value of Freq of last symbol, that was added before error. | ||
801 | so the largest possible increase for Escape_Freq is (8) from value before ModelUpoadet() */ | ||
802 | Refresh(p, c, ((unsigned)c->NumStats + 3) >> 1, 0); | ||
803 | } | ||
804 | |||
805 | // increase Escape Freq for context [ctxError ... p->MinContext) | ||
806 | for (; c != p->MinContext; c = SUFFIX(c)) | ||
807 | if (c->NumStats == 0) | ||
808 | { | ||
809 | // ONE_STATE(c) | ||
810 | c->Union2.State2.Freq = (Byte)(((unsigned)c->Union2.State2.Freq + 1) >> 1); | ||
811 | } | ||
812 | else if ((c->Union2.SummFreq = (UInt16)(c->Union2.SummFreq + 4)) > 128 + 4 * c->NumStats) | ||
813 | Refresh(p, c, ((unsigned)c->NumStats + 2) >> 1, 1); | ||
814 | |||
815 | #ifdef PPMD8_FREEZE_SUPPORT | ||
816 | if (p->RestoreMethod > PPMD8_RESTORE_METHOD_FREEZE) | ||
817 | { | ||
818 | p->MaxContext = fSuccessor; | ||
819 | p->GlueCount += !(p->Stamps[1] & 1); // why? | ||
820 | } | ||
821 | else if (p->RestoreMethod == PPMD8_RESTORE_METHOD_FREEZE) | ||
822 | { | ||
823 | while (p->MaxContext->Suffix) | ||
824 | p->MaxContext = SUFFIX(p->MaxContext); | ||
825 | RemoveBinContexts(p, p->MaxContext, 0); | ||
826 | // we change the current mode to (PPMD8_RESTORE_METHOD_FREEZE + 1) | ||
827 | p->RestoreMethod = PPMD8_RESTORE_METHOD_FREEZE + 1; | ||
828 | p->GlueCount = 0; | ||
829 | p->OrderFall = p->MaxOrder; | ||
830 | } | ||
831 | else | ||
832 | #endif | ||
833 | if (p->RestoreMethod == PPMD8_RESTORE_METHOD_RESTART || GetUsedMemory(p) < (p->Size >> 1)) | ||
834 | RestartModel(p); | ||
835 | else | ||
836 | { | ||
837 | while (p->MaxContext->Suffix) | ||
838 | p->MaxContext = SUFFIX(p->MaxContext); | ||
839 | do | ||
840 | { | ||
841 | CutOff(p, p->MaxContext, 0); | ||
842 | ExpandTextArea(p); | ||
843 | } | ||
844 | while (GetUsedMemory(p) > 3 * (p->Size >> 2)); | ||
845 | p->GlueCount = 0; | ||
846 | p->OrderFall = p->MaxOrder; | ||
847 | } | ||
848 | p->MinContext = p->MaxContext; | ||
849 | } | ||
850 | |||
851 | |||
852 | |||
853 | MY_NO_INLINE | ||
854 | static CTX_PTR CreateSuccessors(CPpmd8 *p, BoolInt skip, CPpmd_State *s1, CTX_PTR c) | ||
855 | { | ||
856 | |||
857 | CPpmd_Byte_Ref upBranch = (CPpmd_Byte_Ref)SUCCESSOR(p->FoundState); | ||
858 | Byte newSym, newFreq, flags; | ||
859 | unsigned numPs = 0; | ||
860 | CPpmd_State *ps[PPMD8_MAX_ORDER + 1]; /* fixed over Shkarin's code. Maybe it could work without + 1 too. */ | ||
861 | |||
862 | if (!skip) | ||
863 | ps[numPs++] = p->FoundState; | ||
864 | |||
865 | while (c->Suffix) | ||
866 | { | ||
867 | CPpmd_Void_Ref successor; | ||
868 | CPpmd_State *s; | ||
869 | c = SUFFIX(c); | ||
870 | |||
871 | if (s1) { s = s1; s1 = NULL; } | ||
872 | else if (c->NumStats != 0) | ||
873 | { | ||
874 | Byte sym = p->FoundState->Symbol; | ||
875 | for (s = STATS(c); s->Symbol != sym; s++); | ||
876 | if (s->Freq < MAX_FREQ - 9) { s->Freq++; c->Union2.SummFreq++; } | ||
877 | } | ||
878 | else | ||
879 | { | ||
880 | s = ONE_STATE(c); | ||
881 | s->Freq = (Byte)(s->Freq + (!SUFFIX(c)->NumStats & (s->Freq < 24))); | ||
882 | } | ||
883 | successor = SUCCESSOR(s); | ||
884 | if (successor != upBranch) | ||
885 | { | ||
886 | |||
887 | c = CTX(successor); | ||
888 | if (numPs == 0) | ||
889 | { | ||
890 | |||
891 | |||
892 | return c; | ||
893 | } | ||
894 | break; | ||
895 | } | ||
896 | ps[numPs++] = s; | ||
897 | } | ||
898 | |||
899 | |||
900 | |||
901 | |||
902 | |||
903 | newSym = *(const Byte *)Ppmd8_GetPtr(p, upBranch); | ||
904 | upBranch++; | ||
905 | flags = (Byte)(PPMD8_HiBitsFlag_4(p->FoundState->Symbol) + PPMD8_HiBitsFlag_3(newSym)); | ||
906 | |||
907 | if (c->NumStats == 0) | ||
908 | newFreq = c->Union2.State2.Freq; | ||
909 | else | ||
910 | { | ||
911 | UInt32 cf, s0; | ||
912 | CPpmd_State *s; | ||
913 | for (s = STATS(c); s->Symbol != newSym; s++); | ||
914 | cf = (UInt32)s->Freq - 1; | ||
915 | s0 = (UInt32)c->Union2.SummFreq - c->NumStats - cf; | ||
916 | /* | ||
917 | |||
918 | |||
919 | max(newFreq)= (s->Freq - 1), when (s0 == 1) | ||
920 | |||
921 | |||
922 | */ | ||
923 | newFreq = (Byte)(1 + ((2 * cf <= s0) ? (5 * cf > s0) : ((cf + 2 * s0 - 3) / s0))); | ||
924 | } | ||
925 | |||
926 | |||
927 | |||
928 | do | ||
929 | { | ||
930 | CTX_PTR c1; | ||
931 | /* = AllocContext(p); */ | ||
932 | if (p->HiUnit != p->LoUnit) | ||
933 | c1 = (CTX_PTR)(void *)(p->HiUnit -= UNIT_SIZE); | ||
934 | else if (p->FreeList[0] != 0) | ||
935 | c1 = (CTX_PTR)RemoveNode(p, 0); | ||
936 | else | ||
937 | { | ||
938 | c1 = (CTX_PTR)AllocUnitsRare(p, 0); | ||
939 | if (!c1) | ||
940 | return NULL; | ||
941 | } | ||
942 | c1->Flags = flags; | ||
943 | c1->NumStats = 0; | ||
944 | c1->Union2.State2.Symbol = newSym; | ||
945 | c1->Union2.State2.Freq = newFreq; | ||
946 | SetSuccessor(ONE_STATE(c1), upBranch); | ||
947 | c1->Suffix = REF(c); | ||
948 | SetSuccessor(ps[--numPs], REF(c1)); | ||
949 | c = c1; | ||
950 | } | ||
951 | while (numPs != 0); | ||
952 | |||
953 | return c; | ||
954 | } | ||
955 | |||
956 | |||
957 | static CTX_PTR ReduceOrder(CPpmd8 *p, CPpmd_State *s1, CTX_PTR c) | ||
958 | { | ||
959 | CPpmd_State *s = NULL; | ||
960 | CTX_PTR c1 = c; | ||
961 | CPpmd_Void_Ref upBranch = REF(p->Text); | ||
962 | |||
963 | #ifdef PPMD8_FREEZE_SUPPORT | ||
964 | /* The BUG in Shkarin's code was fixed: ps could overflow in CUT_OFF mode. */ | ||
965 | CPpmd_State *ps[PPMD8_MAX_ORDER + 1]; | ||
966 | unsigned numPs = 0; | ||
967 | ps[numPs++] = p->FoundState; | ||
968 | #endif | ||
969 | |||
970 | SetSuccessor(p->FoundState, upBranch); | ||
971 | p->OrderFall++; | ||
972 | |||
973 | for (;;) | ||
974 | { | ||
975 | if (s1) | ||
976 | { | ||
977 | c = SUFFIX(c); | ||
978 | s = s1; | ||
979 | s1 = NULL; | ||
980 | } | ||
981 | else | ||
982 | { | ||
983 | if (!c->Suffix) | ||
984 | { | ||
985 | #ifdef PPMD8_FREEZE_SUPPORT | ||
986 | if (p->RestoreMethod > PPMD8_RESTORE_METHOD_FREEZE) | ||
987 | { | ||
988 | do { SetSuccessor(ps[--numPs], REF(c)); } while (numPs); | ||
989 | RESET_TEXT(1); | ||
990 | p->OrderFall = 1; | ||
991 | } | ||
992 | #endif | ||
993 | return c; | ||
994 | } | ||
995 | c = SUFFIX(c); | ||
996 | if (c->NumStats) | ||
997 | { | ||
998 | if ((s = STATS(c))->Symbol != p->FoundState->Symbol) | ||
999 | do { s++; } while (s->Symbol != p->FoundState->Symbol); | ||
1000 | if (s->Freq < MAX_FREQ - 9) | ||
1001 | { | ||
1002 | s->Freq = (Byte)(s->Freq + 2); | ||
1003 | c->Union2.SummFreq = (UInt16)(c->Union2.SummFreq + 2); | ||
1004 | } | ||
1005 | } | ||
1006 | else | ||
1007 | { | ||
1008 | s = ONE_STATE(c); | ||
1009 | s->Freq = (Byte)(s->Freq + (s->Freq < 32)); | ||
1010 | } | ||
1011 | } | ||
1012 | if (SUCCESSOR(s)) | ||
1013 | break; | ||
1014 | #ifdef PPMD8_FREEZE_SUPPORT | ||
1015 | ps[numPs++] = s; | ||
1016 | #endif | ||
1017 | SetSuccessor(s, upBranch); | ||
1018 | p->OrderFall++; | ||
1019 | } | ||
1020 | |||
1021 | #ifdef PPMD8_FREEZE_SUPPORT | ||
1022 | if (p->RestoreMethod > PPMD8_RESTORE_METHOD_FREEZE) | ||
1023 | { | ||
1024 | c = CTX(SUCCESSOR(s)); | ||
1025 | do { SetSuccessor(ps[--numPs], REF(c)); } while (numPs); | ||
1026 | RESET_TEXT(1); | ||
1027 | p->OrderFall = 1; | ||
1028 | return c; | ||
1029 | } | ||
1030 | else | ||
1031 | #endif | ||
1032 | if (SUCCESSOR(s) <= upBranch) | ||
1033 | { | ||
1034 | CTX_PTR successor; | ||
1035 | CPpmd_State *s2 = p->FoundState; | ||
1036 | p->FoundState = s; | ||
1037 | |||
1038 | successor = CreateSuccessors(p, False, NULL, c); | ||
1039 | if (!successor) | ||
1040 | SetSuccessor(s, 0); | ||
1041 | else | ||
1042 | SetSuccessor(s, REF(successor)); | ||
1043 | p->FoundState = s2; | ||
1044 | } | ||
1045 | |||
1046 | { | ||
1047 | CPpmd_Void_Ref successor = SUCCESSOR(s); | ||
1048 | if (p->OrderFall == 1 && c1 == p->MaxContext) | ||
1049 | { | ||
1050 | SetSuccessor(p->FoundState, successor); | ||
1051 | p->Text--; | ||
1052 | } | ||
1053 | if (successor == 0) | ||
1054 | return NULL; | ||
1055 | return CTX(successor); | ||
1056 | } | ||
1057 | } | ||
1058 | |||
1059 | |||
1060 | |||
1061 | void Ppmd8_UpdateModel(CPpmd8 *p); | ||
1062 | MY_NO_INLINE | ||
1063 | void Ppmd8_UpdateModel(CPpmd8 *p) | ||
1064 | { | ||
1065 | CPpmd_Void_Ref maxSuccessor, minSuccessor = SUCCESSOR(p->FoundState); | ||
1066 | CTX_PTR c; | ||
1067 | unsigned s0, ns, fFreq = p->FoundState->Freq; | ||
1068 | Byte flag, fSymbol = p->FoundState->Symbol; | ||
1069 | { | ||
1070 | CPpmd_State *s = NULL; | ||
1071 | if (p->FoundState->Freq < MAX_FREQ / 4 && p->MinContext->Suffix != 0) | ||
1072 | { | ||
1073 | /* Update Freqs in Suffix Context */ | ||
1074 | |||
1075 | c = SUFFIX(p->MinContext); | ||
1076 | |||
1077 | if (c->NumStats == 0) | ||
1078 | { | ||
1079 | s = ONE_STATE(c); | ||
1080 | if (s->Freq < 32) | ||
1081 | s->Freq++; | ||
1082 | } | ||
1083 | else | ||
1084 | { | ||
1085 | Byte sym = p->FoundState->Symbol; | ||
1086 | s = STATS(c); | ||
1087 | |||
1088 | if (s->Symbol != sym) | ||
1089 | { | ||
1090 | do | ||
1091 | { | ||
1092 | |||
1093 | s++; | ||
1094 | } | ||
1095 | while (s->Symbol != sym); | ||
1096 | |||
1097 | if (s[0].Freq >= s[-1].Freq) | ||
1098 | { | ||
1099 | SwapStates(&s[0], &s[-1]); | ||
1100 | s--; | ||
1101 | } | ||
1102 | } | ||
1103 | |||
1104 | if (s->Freq < MAX_FREQ - 9) | ||
1105 | { | ||
1106 | s->Freq = (Byte)(s->Freq + 2); | ||
1107 | c->Union2.SummFreq = (UInt16)(c->Union2.SummFreq + 2); | ||
1108 | } | ||
1109 | } | ||
1110 | } | ||
1111 | |||
1112 | c = p->MaxContext; | ||
1113 | if (p->OrderFall == 0 && minSuccessor) | ||
1114 | { | ||
1115 | CTX_PTR cs = CreateSuccessors(p, True, s, p->MinContext); | ||
1116 | if (!cs) | ||
1117 | { | ||
1118 | SetSuccessor(p->FoundState, 0); | ||
1119 | RESTORE_MODEL(c, CTX(minSuccessor)); | ||
1120 | return; | ||
1121 | } | ||
1122 | SetSuccessor(p->FoundState, REF(cs)); | ||
1123 | p->MinContext = p->MaxContext = cs; | ||
1124 | return; | ||
1125 | } | ||
1126 | |||
1127 | |||
1128 | |||
1129 | |||
1130 | { | ||
1131 | Byte *text = p->Text; | ||
1132 | *text++ = p->FoundState->Symbol; | ||
1133 | p->Text = text; | ||
1134 | if (text >= p->UnitsStart) | ||
1135 | { | ||
1136 | RESTORE_MODEL(c, CTX(minSuccessor)); /* check it */ | ||
1137 | return; | ||
1138 | } | ||
1139 | maxSuccessor = REF(text); | ||
1140 | } | ||
1141 | |||
1142 | if (!minSuccessor) | ||
1143 | { | ||
1144 | CTX_PTR cs = ReduceOrder(p, s, p->MinContext); | ||
1145 | if (!cs) | ||
1146 | { | ||
1147 | RESTORE_MODEL(c, NULL); | ||
1148 | return; | ||
1149 | } | ||
1150 | minSuccessor = REF(cs); | ||
1151 | } | ||
1152 | else if ((Byte *)Ppmd8_GetPtr(p, minSuccessor) < p->UnitsStart) | ||
1153 | { | ||
1154 | CTX_PTR cs = CreateSuccessors(p, False, s, p->MinContext); | ||
1155 | if (!cs) | ||
1156 | { | ||
1157 | RESTORE_MODEL(c, NULL); | ||
1158 | return; | ||
1159 | } | ||
1160 | minSuccessor = REF(cs); | ||
1161 | } | ||
1162 | |||
1163 | if (--p->OrderFall == 0) | ||
1164 | { | ||
1165 | maxSuccessor = minSuccessor; | ||
1166 | p->Text -= (p->MaxContext != p->MinContext); | ||
1167 | } | ||
1168 | #ifdef PPMD8_FREEZE_SUPPORT | ||
1169 | else if (p->RestoreMethod > PPMD8_RESTORE_METHOD_FREEZE) | ||
1170 | { | ||
1171 | maxSuccessor = minSuccessor; | ||
1172 | RESET_TEXT(0); | ||
1173 | p->OrderFall = 0; | ||
1174 | } | ||
1175 | #endif | ||
1176 | } | ||
1177 | |||
1178 | |||
1179 | |||
1180 | |||
1181 | |||
1182 | |||
1183 | |||
1184 | |||
1185 | |||
1186 | |||
1187 | |||
1188 | |||
1189 | |||
1190 | |||
1191 | |||
1192 | |||
1193 | |||
1194 | |||
1195 | |||
1196 | |||
1197 | |||
1198 | |||
1199 | |||
1200 | |||
1201 | |||
1202 | |||
1203 | |||
1204 | |||
1205 | flag = (Byte)(PPMD8_HiBitsFlag_3(fSymbol)); | ||
1206 | s0 = p->MinContext->Union2.SummFreq - (ns = p->MinContext->NumStats) - fFreq; | ||
1207 | |||
1208 | for (; c != p->MinContext; c = SUFFIX(c)) | ||
1209 | { | ||
1210 | unsigned ns1; | ||
1211 | UInt32 sum; | ||
1212 | |||
1213 | if ((ns1 = c->NumStats) != 0) | ||
1214 | { | ||
1215 | if ((ns1 & 1) != 0) | ||
1216 | { | ||
1217 | /* Expand for one UNIT */ | ||
1218 | unsigned oldNU = (ns1 + 1) >> 1; | ||
1219 | unsigned i = U2I(oldNU); | ||
1220 | if (i != U2I((size_t)oldNU + 1)) | ||
1221 | { | ||
1222 | void *ptr = AllocUnits(p, i + 1); | ||
1223 | void *oldPtr; | ||
1224 | if (!ptr) | ||
1225 | { | ||
1226 | RESTORE_MODEL(c, CTX(minSuccessor)); | ||
1227 | return; | ||
1228 | } | ||
1229 | oldPtr = STATS(c); | ||
1230 | MyMem12Cpy(ptr, oldPtr, oldNU); | ||
1231 | InsertNode(p, oldPtr, i); | ||
1232 | c->Union4.Stats = STATS_REF(ptr); | ||
1233 | } | ||
1234 | } | ||
1235 | sum = c->Union2.SummFreq; | ||
1236 | /* max increase of Escape_Freq is 1 here. | ||
1237 | an average increase is 1/3 per symbol */ | ||
1238 | sum += (3 * ns1 + 1 < ns); | ||
1239 | /* original PPMdH uses 16-bit variable for (sum) here. | ||
1240 | But (sum < ???). Do we need to truncate (sum) to 16-bit */ | ||
1241 | // sum = (UInt16)sum; | ||
1242 | } | ||
1243 | else | ||
1244 | { | ||
1245 | |||
1246 | CPpmd_State *s = (CPpmd_State*)AllocUnits(p, 0); | ||
1247 | if (!s) | ||
1248 | { | ||
1249 | RESTORE_MODEL(c, CTX(minSuccessor)); | ||
1250 | return; | ||
1251 | } | ||
1252 | { | ||
1253 | unsigned freq = c->Union2.State2.Freq; | ||
1254 | // s = *ONE_STATE(c); | ||
1255 | s->Symbol = c->Union2.State2.Symbol; | ||
1256 | s->Successor_0 = c->Union4.State4.Successor_0; | ||
1257 | s->Successor_1 = c->Union4.State4.Successor_1; | ||
1258 | // SetSuccessor(s, c->Union4.Stats); // call it only for debug purposes to check the order of | ||
1259 | // (Successor_0 and Successor_1) in LE/BE. | ||
1260 | c->Union4.Stats = REF(s); | ||
1261 | if (freq < MAX_FREQ / 4 - 1) | ||
1262 | freq <<= 1; | ||
1263 | else | ||
1264 | freq = MAX_FREQ - 4; | ||
1265 | |||
1266 | s->Freq = (Byte)freq; | ||
1267 | |||
1268 | sum = freq + p->InitEsc + (ns > 2); // Ppmd8 (> 2) | ||
1269 | } | ||
1270 | } | ||
1271 | |||
1272 | { | ||
1273 | CPpmd_State *s = STATS(c) + ns1 + 1; | ||
1274 | UInt32 cf = 2 * (sum + 6) * (UInt32)fFreq; | ||
1275 | UInt32 sf = (UInt32)s0 + sum; | ||
1276 | s->Symbol = fSymbol; | ||
1277 | c->NumStats = (Byte)(ns1 + 1); | ||
1278 | SetSuccessor(s, maxSuccessor); | ||
1279 | c->Flags |= flag; | ||
1280 | if (cf < 6 * sf) | ||
1281 | { | ||
1282 | cf = (unsigned)1 + (cf > sf) + (cf >= 4 * sf); | ||
1283 | sum += 4; | ||
1284 | /* It can add (1, 2, 3) to Escape_Freq */ | ||
1285 | } | ||
1286 | else | ||
1287 | { | ||
1288 | cf = (unsigned)4 + (cf > 9 * sf) + (cf > 12 * sf) + (cf > 15 * sf); | ||
1289 | sum += cf; | ||
1290 | } | ||
1291 | |||
1292 | c->Union2.SummFreq = (UInt16)sum; | ||
1293 | s->Freq = (Byte)cf; | ||
1294 | } | ||
1295 | |||
1296 | } | ||
1297 | p->MaxContext = p->MinContext = CTX(minSuccessor); | ||
1298 | } | ||
1299 | |||
1300 | |||
1301 | |||
1302 | MY_NO_INLINE | ||
1303 | static void Rescale(CPpmd8 *p) | ||
1304 | { | ||
1305 | unsigned i, adder, sumFreq, escFreq; | ||
1306 | CPpmd_State *stats = STATS(p->MinContext); | ||
1307 | CPpmd_State *s = p->FoundState; | ||
1308 | |||
1309 | /* Sort the list by Freq */ | ||
1310 | if (s != stats) | ||
1311 | { | ||
1312 | CPpmd_State tmp = *s; | ||
1313 | do | ||
1314 | s[0] = s[-1]; | ||
1315 | while (--s != stats); | ||
1316 | *s = tmp; | ||
1317 | } | ||
1318 | |||
1319 | sumFreq = s->Freq; | ||
1320 | escFreq = p->MinContext->Union2.SummFreq - sumFreq; | ||
1321 | |||
1322 | |||
1323 | |||
1324 | |||
1325 | |||
1326 | |||
1327 | adder = (p->OrderFall != 0); | ||
1328 | |||
1329 | #ifdef PPMD8_FREEZE_SUPPORT | ||
1330 | adder |= (p->RestoreMethod > PPMD8_RESTORE_METHOD_FREEZE); | ||
1331 | #endif | ||
1332 | |||
1333 | sumFreq = (sumFreq + 4 + adder) >> 1; | ||
1334 | i = p->MinContext->NumStats; | ||
1335 | s->Freq = (Byte)sumFreq; | ||
1336 | |||
1337 | do | ||
1338 | { | ||
1339 | unsigned freq = (++s)->Freq; | ||
1340 | escFreq -= freq; | ||
1341 | freq = (freq + adder) >> 1; | ||
1342 | sumFreq += freq; | ||
1343 | s->Freq = (Byte)freq; | ||
1344 | if (freq > s[-1].Freq) | ||
1345 | { | ||
1346 | CPpmd_State tmp = *s; | ||
1347 | CPpmd_State *s1 = s; | ||
1348 | do | ||
1349 | { | ||
1350 | s1[0] = s1[-1]; | ||
1351 | } | ||
1352 | while (--s1 != stats && freq > s1[-1].Freq); | ||
1353 | *s1 = tmp; | ||
1354 | } | ||
1355 | } | ||
1356 | while (--i); | ||
1357 | |||
1358 | if (s->Freq == 0) | ||
1359 | { | ||
1360 | /* Remove all items with Freq == 0 */ | ||
1361 | CPpmd8_Context *mc; | ||
1362 | unsigned numStats, numStatsNew, n0, n1; | ||
1363 | |||
1364 | i = 0; do { i++; } while ((--s)->Freq == 0); | ||
1365 | |||
1366 | |||
1367 | |||
1368 | |||
1369 | escFreq += i; | ||
1370 | mc = p->MinContext; | ||
1371 | numStats = mc->NumStats; | ||
1372 | numStatsNew = numStats - i; | ||
1373 | mc->NumStats = (Byte)(numStatsNew); | ||
1374 | n0 = (numStats + 2) >> 1; | ||
1375 | |||
1376 | if (numStatsNew == 0) | ||
1377 | { | ||
1378 | |||
1379 | unsigned freq = (2 * (unsigned)stats->Freq + escFreq - 1) / escFreq; | ||
1380 | if (freq > MAX_FREQ / 3) | ||
1381 | freq = MAX_FREQ / 3; | ||
1382 | mc->Flags = (Byte)((mc->Flags & FLAG_PREV_HIGH) + PPMD8_HiBitsFlag_3(stats->Symbol)); | ||
1383 | |||
1384 | |||
1385 | |||
1386 | |||
1387 | |||
1388 | s = ONE_STATE(mc); | ||
1389 | *s = *stats; | ||
1390 | s->Freq = (Byte)freq; | ||
1391 | p->FoundState = s; | ||
1392 | InsertNode(p, stats, U2I(n0)); | ||
1393 | return; | ||
1394 | } | ||
1395 | |||
1396 | n1 = (numStatsNew + 2) >> 1; | ||
1397 | if (n0 != n1) | ||
1398 | mc->Union4.Stats = STATS_REF(ShrinkUnits(p, stats, n0, n1)); | ||
1399 | { | ||
1400 | // here we are for max order only. So Ppmd8_MakeEscFreq() doesn't use mc->Flags | ||
1401 | // but we still need current (Flags & FLAG_PREV_HIGH), if we will convert context to 1-symbol context later. | ||
1402 | /* | ||
1403 | unsigned flags = HiBits_Prepare((s = STATS(mc))->Symbol); | ||
1404 | i = mc->NumStats; | ||
1405 | do { flags |= HiBits_Prepare((++s)->Symbol); } while (--i); | ||
1406 | mc->Flags = (Byte)((mc->Flags & ~FLAG_SYM_HIGH) + HiBits_Convert_3(flags)); | ||
1407 | */ | ||
1408 | } | ||
1409 | } | ||
1410 | |||
1411 | |||
1412 | |||
1413 | |||
1414 | |||
1415 | |||
1416 | { | ||
1417 | CPpmd8_Context *mc = p->MinContext; | ||
1418 | mc->Union2.SummFreq = (UInt16)(sumFreq + escFreq - (escFreq >> 1)); | ||
1419 | mc->Flags |= FLAG_RESCALED; | ||
1420 | p->FoundState = STATS(mc); | ||
1421 | } | ||
1422 | } | ||
1423 | |||
1424 | |||
1425 | CPpmd_See *Ppmd8_MakeEscFreq(CPpmd8 *p, unsigned numMasked1, UInt32 *escFreq) | ||
1426 | { | ||
1427 | CPpmd_See *see; | ||
1428 | const CPpmd8_Context *mc = p->MinContext; | ||
1429 | unsigned numStats = mc->NumStats; | ||
1430 | if (numStats != 0xFF) | ||
1431 | { | ||
1432 | // (3 <= numStats + 2 <= 256) (3 <= NS2Indx[3] and NS2Indx[256] === 26) | ||
1433 | see = p->See[(size_t)(unsigned)p->NS2Indx[(size_t)numStats + 2] - 3] | ||
1434 | + (mc->Union2.SummFreq > 11 * (numStats + 1)) | ||
1435 | + 2 * (unsigned)(2 * numStats < ((unsigned)SUFFIX(mc)->NumStats + numMasked1)) | ||
1436 | + mc->Flags; | ||
1437 | |||
1438 | { | ||
1439 | // if (see->Summ) field is larger than 16-bit, we need only low 16 bits of Summ | ||
1440 | unsigned summ = (UInt16)see->Summ; // & 0xFFFF | ||
1441 | unsigned r = (summ >> see->Shift); | ||
1442 | see->Summ = (UInt16)(summ - r); | ||
1443 | *escFreq = r + (r == 0); | ||
1444 | } | ||
1445 | } | ||
1446 | else | ||
1447 | { | ||
1448 | see = &p->DummySee; | ||
1449 | *escFreq = 1; | ||
1450 | } | ||
1451 | return see; | ||
1452 | } | ||
1453 | |||
1454 | |||
1455 | static void NextContext(CPpmd8 *p) | ||
1456 | { | ||
1457 | CTX_PTR c = CTX(SUCCESSOR(p->FoundState)); | ||
1458 | if (p->OrderFall == 0 && (const Byte *)c >= p->UnitsStart) | ||
1459 | p->MaxContext = p->MinContext = c; | ||
1460 | else | ||
1461 | Ppmd8_UpdateModel(p); | ||
1462 | } | ||
1463 | |||
1464 | |||
1465 | void Ppmd8_Update1(CPpmd8 *p) | ||
1466 | { | ||
1467 | CPpmd_State *s = p->FoundState; | ||
1468 | unsigned freq = s->Freq; | ||
1469 | freq += 4; | ||
1470 | p->MinContext->Union2.SummFreq = (UInt16)(p->MinContext->Union2.SummFreq + 4); | ||
1471 | s->Freq = (Byte)freq; | ||
1472 | if (freq > s[-1].Freq) | ||
1473 | { | ||
1474 | SwapStates(s, &s[-1]); | ||
1475 | p->FoundState = --s; | ||
1476 | if (freq > MAX_FREQ) | ||
1477 | Rescale(p); | ||
1478 | } | ||
1479 | NextContext(p); | ||
1480 | } | ||
1481 | |||
1482 | |||
1483 | void Ppmd8_Update1_0(CPpmd8 *p) | ||
1484 | { | ||
1485 | CPpmd_State *s = p->FoundState; | ||
1486 | CPpmd8_Context *mc = p->MinContext; | ||
1487 | unsigned freq = s->Freq; | ||
1488 | unsigned summFreq = mc->Union2.SummFreq; | ||
1489 | p->PrevSuccess = (2 * freq >= summFreq); // Ppmd8 (>=) | ||
1490 | p->RunLength += (int)p->PrevSuccess; | ||
1491 | mc->Union2.SummFreq = (UInt16)(summFreq + 4); | ||
1492 | freq += 4; | ||
1493 | s->Freq = (Byte)freq; | ||
1494 | if (freq > MAX_FREQ) | ||
1495 | Rescale(p); | ||
1496 | NextContext(p); | ||
1497 | } | ||
1498 | |||
1499 | |||
1500 | /* | ||
1501 | void Ppmd8_UpdateBin(CPpmd8 *p) | ||
1502 | { | ||
1503 | unsigned freq = p->FoundState->Freq; | ||
1504 | p->FoundState->Freq = (Byte)(freq + (freq < 196)); // Ppmd8 (196) | ||
1505 | p->PrevSuccess = 1; | ||
1506 | p->RunLength++; | ||
1507 | NextContext(p); | ||
1508 | } | ||
1509 | */ | ||
1510 | |||
1511 | void Ppmd8_Update2(CPpmd8 *p) | ||
1512 | { | ||
1513 | CPpmd_State *s = p->FoundState; | ||
1514 | unsigned freq = s->Freq; | ||
1515 | freq += 4; | ||
1516 | p->RunLength = p->InitRL; | ||
1517 | p->MinContext->Union2.SummFreq = (UInt16)(p->MinContext->Union2.SummFreq + 4); | ||
1518 | s->Freq = (Byte)freq; | ||
1519 | if (freq > MAX_FREQ) | ||
1520 | Rescale(p); | ||
1521 | Ppmd8_UpdateModel(p); | ||
1522 | } | ||
1523 | |||
1524 | /* H->I changes: | ||
1525 | NS2Indx | ||
1526 | GlueCount, and Glue method | ||
1527 | BinSum | ||
1528 | See / EscFreq | ||
1529 | CreateSuccessors updates more suffix contexts | ||
1530 | Ppmd8_UpdateModel consts. | ||
1531 | PrevSuccess Update | ||
1532 | |||
1533 | Flags: | ||
1534 | (1 << 2) - the Context was Rescaled | ||
1535 | (1 << 3) - there is symbol in Stats with (sym >= 0x40) in | ||
1536 | (1 << 4) - main symbol of context is (sym >= 0x40) | ||
1537 | */ | ||