diff options
author | Igor Pavlov <87184205+ip7z@users.noreply.github.com> | 2024-05-14 00:00:00 +0000 |
---|---|---|
committer | Igor Pavlov <87184205+ip7z@users.noreply.github.com> | 2024-05-15 23:55:04 +0500 |
commit | fc662341e6f85da78ada0e443f6116b978f79f22 (patch) | |
tree | 1be1cc402a7a9cbc18d4eeea6b141354c2d559e3 /C/7zCrc.c | |
parent | 5b39dc76f1bc82f941d5c800ab9f34407a06b53a (diff) | |
download | 7zip-24.05.tar.gz 7zip-24.05.tar.bz2 7zip-24.05.zip |
24.0524.05
Diffstat (limited to 'C/7zCrc.c')
-rw-r--r-- | C/7zCrc.c | 508 |
1 files changed, 294 insertions, 214 deletions
@@ -1,93 +1,96 @@ | |||
1 | /* 7zCrc.c -- CRC32 calculation and init | 1 | /* 7zCrc.c -- CRC32 calculation and init |
2 | 2023-04-02 : Igor Pavlov : Public domain */ | 2 | 2024-03-01 : Igor Pavlov : Public domain */ |
3 | 3 | ||
4 | #include "Precomp.h" | 4 | #include "Precomp.h" |
5 | 5 | ||
6 | #include "7zCrc.h" | 6 | #include "7zCrc.h" |
7 | #include "CpuArch.h" | 7 | #include "CpuArch.h" |
8 | 8 | ||
9 | #define kCrcPoly 0xEDB88320 | 9 | // for debug: |
10 | // #define __ARM_FEATURE_CRC32 1 | ||
10 | 11 | ||
11 | #ifdef MY_CPU_LE | 12 | #ifdef __ARM_FEATURE_CRC32 |
12 | #define CRC_NUM_TABLES 8 | 13 | // #pragma message("__ARM_FEATURE_CRC32") |
13 | #else | 14 | #define Z7_CRC_HW_FORCE |
14 | #define CRC_NUM_TABLES 9 | 15 | #endif |
15 | 16 | ||
16 | UInt32 Z7_FASTCALL CrcUpdateT1_BeT4(UInt32 v, const void *data, size_t size, const UInt32 *table); | 17 | // #define Z7_CRC_DEBUG_BE |
17 | UInt32 Z7_FASTCALL CrcUpdateT1_BeT8(UInt32 v, const void *data, size_t size, const UInt32 *table); | 18 | #ifdef Z7_CRC_DEBUG_BE |
19 | #undef MY_CPU_LE | ||
20 | #define MY_CPU_BE | ||
18 | #endif | 21 | #endif |
19 | 22 | ||
20 | #ifndef MY_CPU_BE | 23 | #ifdef Z7_CRC_HW_FORCE |
21 | UInt32 Z7_FASTCALL CrcUpdateT4(UInt32 v, const void *data, size_t size, const UInt32 *table); | 24 | #define Z7_CRC_NUM_TABLES_USE 1 |
22 | UInt32 Z7_FASTCALL CrcUpdateT8(UInt32 v, const void *data, size_t size, const UInt32 *table); | 25 | #else |
26 | #ifdef Z7_CRC_NUM_TABLES | ||
27 | #define Z7_CRC_NUM_TABLES_USE Z7_CRC_NUM_TABLES | ||
28 | #else | ||
29 | #define Z7_CRC_NUM_TABLES_USE 12 | ||
30 | #endif | ||
23 | #endif | 31 | #endif |
24 | 32 | ||
25 | /* | 33 | #if Z7_CRC_NUM_TABLES_USE < 1 |
26 | extern | 34 | #error Stop_Compiling_Bad_Z7_CRC_NUM_TABLES |
27 | CRC_FUNC g_CrcUpdateT4; | 35 | #endif |
28 | CRC_FUNC g_CrcUpdateT4; | ||
29 | */ | ||
30 | extern | ||
31 | CRC_FUNC g_CrcUpdateT8; | ||
32 | CRC_FUNC g_CrcUpdateT8; | ||
33 | extern | ||
34 | CRC_FUNC g_CrcUpdateT0_32; | ||
35 | CRC_FUNC g_CrcUpdateT0_32; | ||
36 | extern | ||
37 | CRC_FUNC g_CrcUpdateT0_64; | ||
38 | CRC_FUNC g_CrcUpdateT0_64; | ||
39 | extern | ||
40 | CRC_FUNC g_CrcUpdate; | ||
41 | CRC_FUNC g_CrcUpdate; | ||
42 | |||
43 | UInt32 g_CrcTable[256 * CRC_NUM_TABLES]; | ||
44 | |||
45 | UInt32 Z7_FASTCALL CrcUpdate(UInt32 v, const void *data, size_t size) | ||
46 | { | ||
47 | return g_CrcUpdate(v, data, size, g_CrcTable); | ||
48 | } | ||
49 | 36 | ||
50 | UInt32 Z7_FASTCALL CrcCalc(const void *data, size_t size) | 37 | #if defined(MY_CPU_LE) || (Z7_CRC_NUM_TABLES_USE == 1) |
51 | { | 38 | #define Z7_CRC_NUM_TABLES_TOTAL Z7_CRC_NUM_TABLES_USE |
52 | return g_CrcUpdate(CRC_INIT_VAL, data, size, g_CrcTable) ^ CRC_INIT_VAL; | 39 | #else |
53 | } | 40 | #define Z7_CRC_NUM_TABLES_TOTAL (Z7_CRC_NUM_TABLES_USE + 1) |
41 | #endif | ||
54 | 42 | ||
55 | #if CRC_NUM_TABLES < 4 \ | 43 | #ifndef Z7_CRC_HW_FORCE |
56 | || (CRC_NUM_TABLES == 4 && defined(MY_CPU_BE)) \ | 44 | |
45 | #if Z7_CRC_NUM_TABLES_USE == 1 \ | ||
57 | || (!defined(MY_CPU_LE) && !defined(MY_CPU_BE)) | 46 | || (!defined(MY_CPU_LE) && !defined(MY_CPU_BE)) |
58 | #define CRC_UPDATE_BYTE_2(crc, b) (table[((crc) ^ (b)) & 0xFF] ^ ((crc) >> 8)) | 47 | #define CRC_UPDATE_BYTE_2(crc, b) (table[((crc) ^ (b)) & 0xFF] ^ ((crc) >> 8)) |
59 | UInt32 Z7_FASTCALL CrcUpdateT1(UInt32 v, const void *data, size_t size, const UInt32 *table); | 48 | #define Z7_CRC_UPDATE_T1_FUNC_NAME CrcUpdateGT1 |
60 | UInt32 Z7_FASTCALL CrcUpdateT1(UInt32 v, const void *data, size_t size, const UInt32 *table) | 49 | static UInt32 Z7_FASTCALL Z7_CRC_UPDATE_T1_FUNC_NAME(UInt32 v, const void *data, size_t size) |
61 | { | 50 | { |
51 | const UInt32 *table = g_CrcTable; | ||
62 | const Byte *p = (const Byte *)data; | 52 | const Byte *p = (const Byte *)data; |
63 | const Byte *pEnd = p + size; | 53 | const Byte *lim = p + size; |
64 | for (; p != pEnd; p++) | 54 | for (; p != lim; p++) |
65 | v = CRC_UPDATE_BYTE_2(v, *p); | 55 | v = CRC_UPDATE_BYTE_2(v, *p); |
66 | return v; | 56 | return v; |
67 | } | 57 | } |
68 | #endif | 58 | #endif |
69 | 59 | ||
60 | |||
61 | #if Z7_CRC_NUM_TABLES_USE != 1 | ||
62 | #ifndef MY_CPU_BE | ||
63 | #define FUNC_NAME_LE_2(s) CrcUpdateT ## s | ||
64 | #define FUNC_NAME_LE_1(s) FUNC_NAME_LE_2(s) | ||
65 | #define FUNC_NAME_LE FUNC_NAME_LE_1(Z7_CRC_NUM_TABLES_USE) | ||
66 | UInt32 Z7_FASTCALL FUNC_NAME_LE (UInt32 v, const void *data, size_t size, const UInt32 *table); | ||
67 | #endif | ||
68 | #ifndef MY_CPU_LE | ||
69 | #define FUNC_NAME_BE_2(s) CrcUpdateT1_BeT ## s | ||
70 | #define FUNC_NAME_BE_1(s) FUNC_NAME_BE_2(s) | ||
71 | #define FUNC_NAME_BE FUNC_NAME_BE_1(Z7_CRC_NUM_TABLES_USE) | ||
72 | UInt32 Z7_FASTCALL FUNC_NAME_BE (UInt32 v, const void *data, size_t size, const UInt32 *table); | ||
73 | #endif | ||
74 | #endif | ||
75 | |||
76 | #endif // Z7_CRC_HW_FORCE | ||
77 | |||
70 | /* ---------- hardware CRC ---------- */ | 78 | /* ---------- hardware CRC ---------- */ |
71 | 79 | ||
72 | #ifdef MY_CPU_LE | 80 | #ifdef MY_CPU_LE |
73 | 81 | ||
74 | #if defined(MY_CPU_ARM_OR_ARM64) | 82 | #if defined(MY_CPU_ARM_OR_ARM64) |
75 | |||
76 | // #pragma message("ARM*") | 83 | // #pragma message("ARM*") |
77 | 84 | ||
78 | #if defined(_MSC_VER) | 85 | #if (defined(__clang__) && (__clang_major__ >= 3)) \ |
79 | #if defined(MY_CPU_ARM64) | 86 | || defined(__GNUC__) && (__GNUC__ >= 6) && defined(MY_CPU_ARM64) \ |
80 | #if (_MSC_VER >= 1910) | 87 | || defined(__GNUC__) && (__GNUC__ >= 8) |
81 | #ifndef __clang__ | ||
82 | #define USE_ARM64_CRC | ||
83 | #include <intrin.h> | ||
84 | #endif | ||
85 | #endif | ||
86 | #endif | ||
87 | #elif (defined(__clang__) && (__clang_major__ >= 3)) \ | ||
88 | || (defined(__GNUC__) && (__GNUC__ > 4)) | ||
89 | #if !defined(__ARM_FEATURE_CRC32) | 88 | #if !defined(__ARM_FEATURE_CRC32) |
89 | // #pragma message("!defined(__ARM_FEATURE_CRC32)") | ||
90 | Z7_DIAGNOSTIC_IGNORE_BEGIN_RESERVED_MACRO_IDENTIFIER | ||
90 | #define __ARM_FEATURE_CRC32 1 | 91 | #define __ARM_FEATURE_CRC32 1 |
92 | Z7_DIAGNOSTIC_IGNORE_END_RESERVED_MACRO_IDENTIFIER | ||
93 | #define Z7_ARM_FEATURE_CRC32_WAS_SET | ||
91 | #if defined(__clang__) | 94 | #if defined(__clang__) |
92 | #if defined(MY_CPU_ARM64) | 95 | #if defined(MY_CPU_ARM64) |
93 | #define ATTRIB_CRC __attribute__((__target__("crc"))) | 96 | #define ATTRIB_CRC __attribute__((__target__("crc"))) |
@@ -96,100 +99,120 @@ UInt32 Z7_FASTCALL CrcUpdateT1(UInt32 v, const void *data, size_t size, const UI | |||
96 | #endif | 99 | #endif |
97 | #else | 100 | #else |
98 | #if defined(MY_CPU_ARM64) | 101 | #if defined(MY_CPU_ARM64) |
102 | #if !defined(Z7_GCC_VERSION) || (Z7_GCC_VERSION >= 60000) | ||
99 | #define ATTRIB_CRC __attribute__((__target__("+crc"))) | 103 | #define ATTRIB_CRC __attribute__((__target__("+crc"))) |
104 | #endif | ||
100 | #else | 105 | #else |
106 | #if !defined(Z7_GCC_VERSION) || (__GNUC__ >= 8) | ||
107 | #if defined(__ARM_FP) && __GNUC__ >= 8 | ||
108 | // for -mfloat-abi=hard: similar to <arm_acle.h> | ||
109 | #define ATTRIB_CRC __attribute__((__target__("arch=armv8-a+crc+simd"))) | ||
110 | #else | ||
101 | #define ATTRIB_CRC __attribute__((__target__("arch=armv8-a+crc"))) | 111 | #define ATTRIB_CRC __attribute__((__target__("arch=armv8-a+crc"))) |
112 | #endif | ||
113 | #endif | ||
102 | #endif | 114 | #endif |
103 | #endif | 115 | #endif |
104 | #endif | 116 | #endif |
105 | #if defined(__ARM_FEATURE_CRC32) | 117 | #if defined(__ARM_FEATURE_CRC32) |
106 | #define USE_ARM64_CRC | 118 | // #pragma message("<arm_acle.h>") |
119 | /* | ||
120 | arm_acle.h (GGC): | ||
121 | before Nov 17, 2017: | ||
122 | #ifdef __ARM_FEATURE_CRC32 | ||
123 | |||
124 | Nov 17, 2017: gcc10.0 (gcc 9.2.0) checked" | ||
125 | #if __ARM_ARCH >= 8 | ||
126 | #pragma GCC target ("arch=armv8-a+crc") | ||
127 | |||
128 | Aug 22, 2019: GCC 8.4?, 9.2.1, 10.1: | ||
129 | #ifdef __ARM_FEATURE_CRC32 | ||
130 | #ifdef __ARM_FP | ||
131 | #pragma GCC target ("arch=armv8-a+crc+simd") | ||
132 | #else | ||
133 | #pragma GCC target ("arch=armv8-a+crc") | ||
134 | #endif | ||
135 | */ | ||
136 | #if defined(__ARM_ARCH) && __ARM_ARCH < 8 | ||
137 | #if defined(Z7_GCC_VERSION) && (__GNUC__ == 8) && (Z7_GCC_VERSION < 80400) \ | ||
138 | || defined(Z7_GCC_VERSION) && (__GNUC__ == 9) && (Z7_GCC_VERSION < 90201) \ | ||
139 | || defined(Z7_GCC_VERSION) && (__GNUC__ == 10) && (Z7_GCC_VERSION < 100100) | ||
140 | Z7_DIAGNOSTIC_IGNORE_BEGIN_RESERVED_MACRO_IDENTIFIER | ||
141 | // #pragma message("#define __ARM_ARCH 8") | ||
142 | #undef __ARM_ARCH | ||
143 | #define __ARM_ARCH 8 | ||
144 | Z7_DIAGNOSTIC_IGNORE_END_RESERVED_MACRO_IDENTIFIER | ||
145 | #endif | ||
146 | #endif | ||
147 | #define Z7_CRC_HW_USE | ||
107 | #include <arm_acle.h> | 148 | #include <arm_acle.h> |
108 | #endif | 149 | #endif |
150 | #elif defined(_MSC_VER) | ||
151 | #if defined(MY_CPU_ARM64) | ||
152 | #if (_MSC_VER >= 1910) | ||
153 | #ifdef __clang__ | ||
154 | // #define Z7_CRC_HW_USE | ||
155 | // #include <arm_acle.h> | ||
156 | #else | ||
157 | #define Z7_CRC_HW_USE | ||
158 | #include <intrin.h> | ||
159 | #endif | ||
160 | #endif | ||
161 | #endif | ||
109 | #endif | 162 | #endif |
110 | 163 | ||
111 | #else | 164 | #else // non-ARM* |
112 | |||
113 | // no hardware CRC | ||
114 | |||
115 | // #define USE_CRC_EMU | ||
116 | |||
117 | #ifdef USE_CRC_EMU | ||
118 | |||
119 | #pragma message("ARM64 CRC emulation") | ||
120 | |||
121 | Z7_FORCE_INLINE | ||
122 | UInt32 __crc32b(UInt32 v, UInt32 data) | ||
123 | { | ||
124 | const UInt32 *table = g_CrcTable; | ||
125 | v = CRC_UPDATE_BYTE_2(v, (Byte)data); | ||
126 | return v; | ||
127 | } | ||
128 | 165 | ||
129 | Z7_FORCE_INLINE | 166 | // #define Z7_CRC_HW_USE // for debug : we can test HW-branch of code |
130 | UInt32 __crc32w(UInt32 v, UInt32 data) | 167 | #ifdef Z7_CRC_HW_USE |
131 | { | 168 | #include "7zCrcEmu.h" |
132 | const UInt32 *table = g_CrcTable; | 169 | #endif |
133 | v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8; | ||
134 | v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8; | ||
135 | v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8; | ||
136 | v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8; | ||
137 | return v; | ||
138 | } | ||
139 | 170 | ||
140 | Z7_FORCE_INLINE | 171 | #endif // non-ARM* |
141 | UInt32 __crc32d(UInt32 v, UInt64 data) | ||
142 | { | ||
143 | const UInt32 *table = g_CrcTable; | ||
144 | v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8; | ||
145 | v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8; | ||
146 | v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8; | ||
147 | v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8; | ||
148 | v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8; | ||
149 | v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8; | ||
150 | v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8; | ||
151 | v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8; | ||
152 | return v; | ||
153 | } | ||
154 | 172 | ||
155 | #endif // USE_CRC_EMU | ||
156 | 173 | ||
157 | #endif // defined(MY_CPU_ARM64) && defined(MY_CPU_LE) | ||
158 | 174 | ||
175 | #if defined(Z7_CRC_HW_USE) | ||
159 | 176 | ||
177 | // #pragma message("USE ARM HW CRC") | ||
160 | 178 | ||
161 | #if defined(USE_ARM64_CRC) || defined(USE_CRC_EMU) | 179 | #ifdef MY_CPU_64BIT |
180 | #define CRC_HW_WORD_TYPE UInt64 | ||
181 | #define CRC_HW_WORD_FUNC __crc32d | ||
182 | #else | ||
183 | #define CRC_HW_WORD_TYPE UInt32 | ||
184 | #define CRC_HW_WORD_FUNC __crc32w | ||
185 | #endif | ||
162 | 186 | ||
163 | #define T0_32_UNROLL_BYTES (4 * 4) | 187 | #define CRC_HW_UNROLL_BYTES (sizeof(CRC_HW_WORD_TYPE) * 4) |
164 | #define T0_64_UNROLL_BYTES (4 * 8) | ||
165 | 188 | ||
166 | #ifndef ATTRIB_CRC | 189 | #ifdef ATTRIB_CRC |
167 | #define ATTRIB_CRC | 190 | ATTRIB_CRC |
168 | #endif | 191 | #endif |
169 | // #pragma message("USE ARM HW CRC") | 192 | Z7_NO_INLINE |
170 | 193 | #ifdef Z7_CRC_HW_FORCE | |
171 | ATTRIB_CRC | 194 | UInt32 Z7_FASTCALL CrcUpdate |
172 | UInt32 Z7_FASTCALL CrcUpdateT0_32(UInt32 v, const void *data, size_t size, const UInt32 *table); | 195 | #else |
173 | ATTRIB_CRC | 196 | static UInt32 Z7_FASTCALL CrcUpdate_HW |
174 | UInt32 Z7_FASTCALL CrcUpdateT0_32(UInt32 v, const void *data, size_t size, const UInt32 *table) | 197 | #endif |
198 | (UInt32 v, const void *data, size_t size) | ||
175 | { | 199 | { |
176 | const Byte *p = (const Byte *)data; | 200 | const Byte *p = (const Byte *)data; |
177 | UNUSED_VAR(table); | 201 | for (; size != 0 && ((unsigned)(ptrdiff_t)p & (CRC_HW_UNROLL_BYTES - 1)) != 0; size--) |
178 | |||
179 | for (; size != 0 && ((unsigned)(ptrdiff_t)p & (T0_32_UNROLL_BYTES - 1)) != 0; size--) | ||
180 | v = __crc32b(v, *p++); | 202 | v = __crc32b(v, *p++); |
181 | 203 | if (size >= CRC_HW_UNROLL_BYTES) | |
182 | if (size >= T0_32_UNROLL_BYTES) | ||
183 | { | 204 | { |
184 | const Byte *lim = p + size; | 205 | const Byte *lim = p + size; |
185 | size &= (T0_32_UNROLL_BYTES - 1); | 206 | size &= CRC_HW_UNROLL_BYTES - 1; |
186 | lim -= size; | 207 | lim -= size; |
187 | do | 208 | do |
188 | { | 209 | { |
189 | v = __crc32w(v, *(const UInt32 *)(const void *)(p)); | 210 | v = CRC_HW_WORD_FUNC(v, *(const CRC_HW_WORD_TYPE *)(const void *)(p)); |
190 | v = __crc32w(v, *(const UInt32 *)(const void *)(p + 4)); p += 2 * 4; | 211 | v = CRC_HW_WORD_FUNC(v, *(const CRC_HW_WORD_TYPE *)(const void *)(p + sizeof(CRC_HW_WORD_TYPE))); |
191 | v = __crc32w(v, *(const UInt32 *)(const void *)(p)); | 212 | p += 2 * sizeof(CRC_HW_WORD_TYPE); |
192 | v = __crc32w(v, *(const UInt32 *)(const void *)(p + 4)); p += 2 * 4; | 213 | v = CRC_HW_WORD_FUNC(v, *(const CRC_HW_WORD_TYPE *)(const void *)(p)); |
214 | v = CRC_HW_WORD_FUNC(v, *(const CRC_HW_WORD_TYPE *)(const void *)(p + sizeof(CRC_HW_WORD_TYPE))); | ||
215 | p += 2 * sizeof(CRC_HW_WORD_TYPE); | ||
193 | } | 216 | } |
194 | while (p != lim); | 217 | while (p != lim); |
195 | } | 218 | } |
@@ -200,46 +223,86 @@ UInt32 Z7_FASTCALL CrcUpdateT0_32(UInt32 v, const void *data, size_t size, const | |||
200 | return v; | 223 | return v; |
201 | } | 224 | } |
202 | 225 | ||
203 | ATTRIB_CRC | 226 | #ifdef Z7_ARM_FEATURE_CRC32_WAS_SET |
204 | UInt32 Z7_FASTCALL CrcUpdateT0_64(UInt32 v, const void *data, size_t size, const UInt32 *table); | 227 | Z7_DIAGNOSTIC_IGNORE_BEGIN_RESERVED_MACRO_IDENTIFIER |
205 | ATTRIB_CRC | 228 | #undef __ARM_FEATURE_CRC32 |
206 | UInt32 Z7_FASTCALL CrcUpdateT0_64(UInt32 v, const void *data, size_t size, const UInt32 *table) | 229 | Z7_DIAGNOSTIC_IGNORE_END_RESERVED_MACRO_IDENTIFIER |
230 | #undef Z7_ARM_FEATURE_CRC32_WAS_SET | ||
231 | #endif | ||
232 | |||
233 | #endif // defined(Z7_CRC_HW_USE) | ||
234 | #endif // MY_CPU_LE | ||
235 | |||
236 | |||
237 | |||
238 | #ifndef Z7_CRC_HW_FORCE | ||
239 | |||
240 | #if defined(Z7_CRC_HW_USE) || defined(Z7_CRC_UPDATE_T1_FUNC_NAME) | ||
241 | /* | ||
242 | typedef UInt32 (Z7_FASTCALL *Z7_CRC_UPDATE_WITH_TABLE_FUNC) | ||
243 | (UInt32 v, const void *data, size_t size, const UInt32 *table); | ||
244 | Z7_CRC_UPDATE_WITH_TABLE_FUNC g_CrcUpdate; | ||
245 | */ | ||
246 | static unsigned g_Crc_Algo; | ||
247 | #if (!defined(MY_CPU_LE) && !defined(MY_CPU_BE)) | ||
248 | static unsigned g_Crc_Be; | ||
249 | #endif | ||
250 | #endif // defined(Z7_CRC_HW_USE) || defined(Z7_CRC_UPDATE_T1_FUNC_NAME) | ||
251 | |||
252 | |||
253 | |||
254 | Z7_NO_INLINE | ||
255 | #ifdef Z7_CRC_HW_USE | ||
256 | static UInt32 Z7_FASTCALL CrcUpdate_Base | ||
257 | #else | ||
258 | UInt32 Z7_FASTCALL CrcUpdate | ||
259 | #endif | ||
260 | (UInt32 crc, const void *data, size_t size) | ||
207 | { | 261 | { |
208 | const Byte *p = (const Byte *)data; | 262 | #if Z7_CRC_NUM_TABLES_USE == 1 |
209 | UNUSED_VAR(table); | 263 | return Z7_CRC_UPDATE_T1_FUNC_NAME(crc, data, size); |
264 | #else // Z7_CRC_NUM_TABLES_USE != 1 | ||
265 | #ifdef Z7_CRC_UPDATE_T1_FUNC_NAME | ||
266 | if (g_Crc_Algo == 1) | ||
267 | return Z7_CRC_UPDATE_T1_FUNC_NAME(crc, data, size); | ||
268 | #endif | ||
210 | 269 | ||
211 | for (; size != 0 && ((unsigned)(ptrdiff_t)p & (T0_64_UNROLL_BYTES - 1)) != 0; size--) | 270 | #ifdef MY_CPU_LE |
212 | v = __crc32b(v, *p++); | 271 | return FUNC_NAME_LE(crc, data, size, g_CrcTable); |
272 | #elif defined(MY_CPU_BE) | ||
273 | return FUNC_NAME_BE(crc, data, size, g_CrcTable); | ||
274 | #else | ||
275 | if (g_Crc_Be) | ||
276 | return FUNC_NAME_BE(crc, data, size, g_CrcTable); | ||
277 | else | ||
278 | return FUNC_NAME_LE(crc, data, size, g_CrcTable); | ||
279 | #endif | ||
280 | #endif // Z7_CRC_NUM_TABLES_USE != 1 | ||
281 | } | ||
213 | 282 | ||
214 | if (size >= T0_64_UNROLL_BYTES) | ||
215 | { | ||
216 | const Byte *lim = p + size; | ||
217 | size &= (T0_64_UNROLL_BYTES - 1); | ||
218 | lim -= size; | ||
219 | do | ||
220 | { | ||
221 | v = __crc32d(v, *(const UInt64 *)(const void *)(p)); | ||
222 | v = __crc32d(v, *(const UInt64 *)(const void *)(p + 8)); p += 2 * 8; | ||
223 | v = __crc32d(v, *(const UInt64 *)(const void *)(p)); | ||
224 | v = __crc32d(v, *(const UInt64 *)(const void *)(p + 8)); p += 2 * 8; | ||
225 | } | ||
226 | while (p != lim); | ||
227 | } | ||
228 | |||
229 | for (; size != 0; size--) | ||
230 | v = __crc32b(v, *p++); | ||
231 | 283 | ||
232 | return v; | 284 | #ifdef Z7_CRC_HW_USE |
285 | Z7_NO_INLINE | ||
286 | UInt32 Z7_FASTCALL CrcUpdate(UInt32 crc, const void *data, size_t size) | ||
287 | { | ||
288 | if (g_Crc_Algo == 0) | ||
289 | return CrcUpdate_HW(crc, data, size); | ||
290 | return CrcUpdate_Base(crc, data, size); | ||
233 | } | 291 | } |
292 | #endif | ||
234 | 293 | ||
235 | #undef T0_32_UNROLL_BYTES | 294 | #endif // !defined(Z7_CRC_HW_FORCE) |
236 | #undef T0_64_UNROLL_BYTES | ||
237 | 295 | ||
238 | #endif // defined(USE_ARM64_CRC) || defined(USE_CRC_EMU) | ||
239 | 296 | ||
240 | #endif // MY_CPU_LE | 297 | |
298 | UInt32 Z7_FASTCALL CrcCalc(const void *data, size_t size) | ||
299 | { | ||
300 | return CrcUpdate(CRC_INIT_VAL, data, size) ^ CRC_INIT_VAL; | ||
301 | } | ||
241 | 302 | ||
242 | 303 | ||
304 | MY_ALIGN(64) | ||
305 | UInt32 g_CrcTable[256 * Z7_CRC_NUM_TABLES_TOTAL]; | ||
243 | 306 | ||
244 | 307 | ||
245 | void Z7_FASTCALL CrcGenerateTable(void) | 308 | void Z7_FASTCALL CrcGenerateTable(void) |
@@ -247,94 +310,111 @@ void Z7_FASTCALL CrcGenerateTable(void) | |||
247 | UInt32 i; | 310 | UInt32 i; |
248 | for (i = 0; i < 256; i++) | 311 | for (i = 0; i < 256; i++) |
249 | { | 312 | { |
313 | #if defined(Z7_CRC_HW_FORCE) | ||
314 | g_CrcTable[i] = __crc32b(i, 0); | ||
315 | #else | ||
316 | #define kCrcPoly 0xEDB88320 | ||
250 | UInt32 r = i; | 317 | UInt32 r = i; |
251 | unsigned j; | 318 | unsigned j; |
252 | for (j = 0; j < 8; j++) | 319 | for (j = 0; j < 8; j++) |
253 | r = (r >> 1) ^ (kCrcPoly & ((UInt32)0 - (r & 1))); | 320 | r = (r >> 1) ^ (kCrcPoly & ((UInt32)0 - (r & 1))); |
254 | g_CrcTable[i] = r; | 321 | g_CrcTable[i] = r; |
322 | #endif | ||
255 | } | 323 | } |
256 | for (i = 256; i < 256 * CRC_NUM_TABLES; i++) | 324 | for (i = 256; i < 256 * Z7_CRC_NUM_TABLES_USE; i++) |
257 | { | 325 | { |
258 | const UInt32 r = g_CrcTable[(size_t)i - 256]; | 326 | const UInt32 r = g_CrcTable[(size_t)i - 256]; |
259 | g_CrcTable[i] = g_CrcTable[r & 0xFF] ^ (r >> 8); | 327 | g_CrcTable[i] = g_CrcTable[r & 0xFF] ^ (r >> 8); |
260 | } | 328 | } |
261 | 329 | ||
262 | #if CRC_NUM_TABLES < 4 | 330 | #if !defined(Z7_CRC_HW_FORCE) && \ |
263 | g_CrcUpdate = CrcUpdateT1; | 331 | (defined(Z7_CRC_HW_USE) || defined(Z7_CRC_UPDATE_T1_FUNC_NAME) || defined(MY_CPU_BE)) |
264 | #elif defined(MY_CPU_LE) | 332 | |
265 | // g_CrcUpdateT4 = CrcUpdateT4; | 333 | #if Z7_CRC_NUM_TABLES_USE <= 1 |
266 | #if CRC_NUM_TABLES < 8 | 334 | g_Crc_Algo = 1; |
267 | g_CrcUpdate = CrcUpdateT4; | 335 | #else // Z7_CRC_NUM_TABLES_USE <= 1 |
268 | #else // CRC_NUM_TABLES >= 8 | 336 | |
269 | g_CrcUpdateT8 = CrcUpdateT8; | 337 | #if defined(MY_CPU_LE) |
270 | /* | 338 | g_Crc_Algo = Z7_CRC_NUM_TABLES_USE; |
271 | #ifdef MY_CPU_X86_OR_AMD64 | 339 | #else // !defined(MY_CPU_LE) |
272 | if (!CPU_Is_InOrder()) | ||
273 | #endif | ||
274 | */ | ||
275 | g_CrcUpdate = CrcUpdateT8; | ||
276 | #endif | ||
277 | #else | ||
278 | { | 340 | { |
279 | #ifndef MY_CPU_BE | 341 | #ifndef MY_CPU_BE |
280 | UInt32 k = 0x01020304; | 342 | UInt32 k = 0x01020304; |
281 | const Byte *p = (const Byte *)&k; | 343 | const Byte *p = (const Byte *)&k; |
282 | if (p[0] == 4 && p[1] == 3) | 344 | if (p[0] == 4 && p[1] == 3) |
283 | { | 345 | g_Crc_Algo = Z7_CRC_NUM_TABLES_USE; |
284 | #if CRC_NUM_TABLES < 8 | ||
285 | // g_CrcUpdateT4 = CrcUpdateT4; | ||
286 | g_CrcUpdate = CrcUpdateT4; | ||
287 | #else // CRC_NUM_TABLES >= 8 | ||
288 | g_CrcUpdateT8 = CrcUpdateT8; | ||
289 | g_CrcUpdate = CrcUpdateT8; | ||
290 | #endif | ||
291 | } | ||
292 | else if (p[0] != 1 || p[1] != 2) | 346 | else if (p[0] != 1 || p[1] != 2) |
293 | g_CrcUpdate = CrcUpdateT1; | 347 | g_Crc_Algo = 1; |
294 | else | 348 | else |
295 | #endif // MY_CPU_BE | 349 | #endif // MY_CPU_BE |
296 | { | 350 | { |
297 | for (i = 256 * CRC_NUM_TABLES - 1; i >= 256; i--) | 351 | for (i = 256 * Z7_CRC_NUM_TABLES_TOTAL - 1; i >= 256; i--) |
298 | { | 352 | { |
299 | const UInt32 x = g_CrcTable[(size_t)i - 256]; | 353 | const UInt32 x = g_CrcTable[(size_t)i - 256]; |
300 | g_CrcTable[i] = Z7_BSWAP32(x); | 354 | g_CrcTable[i] = Z7_BSWAP32(x); |
301 | } | 355 | } |
302 | #if CRC_NUM_TABLES <= 4 | 356 | #if defined(Z7_CRC_UPDATE_T1_FUNC_NAME) |
303 | g_CrcUpdate = CrcUpdateT1; | 357 | g_Crc_Algo = Z7_CRC_NUM_TABLES_USE; |
304 | #elif CRC_NUM_TABLES <= 8 | 358 | #endif |
305 | // g_CrcUpdateT4 = CrcUpdateT1_BeT4; | 359 | #if (!defined(MY_CPU_LE) && !defined(MY_CPU_BE)) |
306 | g_CrcUpdate = CrcUpdateT1_BeT4; | 360 | g_Crc_Be = 1; |
307 | #else // CRC_NUM_TABLES > 8 | 361 | #endif |
308 | g_CrcUpdateT8 = CrcUpdateT1_BeT8; | ||
309 | g_CrcUpdate = CrcUpdateT1_BeT8; | ||
310 | #endif | ||
311 | } | 362 | } |
312 | } | 363 | } |
313 | #endif // CRC_NUM_TABLES < 4 | 364 | #endif // !defined(MY_CPU_LE) |
314 | 365 | ||
315 | #ifdef MY_CPU_LE | 366 | #ifdef MY_CPU_LE |
316 | #ifdef USE_ARM64_CRC | 367 | #ifdef Z7_CRC_HW_USE |
317 | if (CPU_IsSupported_CRC32()) | 368 | if (CPU_IsSupported_CRC32()) |
318 | { | 369 | g_Crc_Algo = 0; |
319 | g_CrcUpdateT0_32 = CrcUpdateT0_32; | 370 | #endif // Z7_CRC_HW_USE |
320 | g_CrcUpdateT0_64 = CrcUpdateT0_64; | 371 | #endif // MY_CPU_LE |
321 | g_CrcUpdate = | 372 | |
322 | #if defined(MY_CPU_ARM) | 373 | #endif // Z7_CRC_NUM_TABLES_USE <= 1 |
323 | CrcUpdateT0_32; | 374 | #endif // g_Crc_Algo was declared |
324 | #else | 375 | } |
325 | CrcUpdateT0_64; | 376 | |
326 | #endif | 377 | Z7_CRC_UPDATE_FUNC z7_GetFunc_CrcUpdate(unsigned algo) |
327 | } | 378 | { |
328 | #endif | 379 | if (algo == 0) |
329 | 380 | return &CrcUpdate; | |
330 | #ifdef USE_CRC_EMU | 381 | |
331 | g_CrcUpdateT0_32 = CrcUpdateT0_32; | 382 | #if defined(Z7_CRC_HW_USE) |
332 | g_CrcUpdateT0_64 = CrcUpdateT0_64; | 383 | if (algo == sizeof(CRC_HW_WORD_TYPE) * 8) |
333 | g_CrcUpdate = CrcUpdateT0_64; | 384 | { |
334 | #endif | 385 | #ifdef Z7_CRC_HW_FORCE |
386 | return &CrcUpdate; | ||
387 | #else | ||
388 | if (g_Crc_Algo == 0) | ||
389 | return &CrcUpdate_HW; | ||
390 | #endif | ||
391 | } | ||
392 | #endif | ||
393 | |||
394 | #ifndef Z7_CRC_HW_FORCE | ||
395 | if (algo == Z7_CRC_NUM_TABLES_USE) | ||
396 | return | ||
397 | #ifdef Z7_CRC_HW_USE | ||
398 | &CrcUpdate_Base; | ||
399 | #else | ||
400 | &CrcUpdate; | ||
335 | #endif | 401 | #endif |
402 | #endif | ||
403 | |||
404 | return NULL; | ||
336 | } | 405 | } |
337 | 406 | ||
338 | #undef kCrcPoly | 407 | #undef kCrcPoly |
339 | #undef CRC64_NUM_TABLES | 408 | #undef Z7_CRC_NUM_TABLES_USE |
409 | #undef Z7_CRC_NUM_TABLES_TOTAL | ||
340 | #undef CRC_UPDATE_BYTE_2 | 410 | #undef CRC_UPDATE_BYTE_2 |
411 | #undef FUNC_NAME_LE_2 | ||
412 | #undef FUNC_NAME_LE_1 | ||
413 | #undef FUNC_NAME_LE | ||
414 | #undef FUNC_NAME_BE_2 | ||
415 | #undef FUNC_NAME_BE_1 | ||
416 | #undef FUNC_NAME_BE | ||
417 | |||
418 | #undef CRC_HW_UNROLL_BYTES | ||
419 | #undef CRC_HW_WORD_FUNC | ||
420 | #undef CRC_HW_WORD_TYPE | ||