23.0123.01

1 files changed, 206 insertions, 142 deletions

diff --git a/C/AesOpt.c b/C/AesOpt.c
index 8be8ff6..cfa6413 100644
--- a/C/AesOpt.c
+++ b/C/AesOpt.c
@@ -1,39 +1,33 @@
 /* AesOpt.c -- AES optimized code for x86 AES hardware instructions
-2021-04-01 : Igor Pavlov : Public domain */
+2023-04-02 : Igor Pavlov : Public domain */
 
 #include "Precomp.h"
 
+#include "Aes.h"
 #include "CpuArch.h"
 
 #ifdef MY_CPU_X86_OR_AMD64
 
-  #if defined(__clang__)
-    #if __clang_major__ > 3 || (__clang_major__ == 3 && __clang_minor__ >= 8)
-      #define USE_INTEL_AES
-        #define ATTRIB_AES __attribute__((__target__("aes")))
-      #if (__clang_major__ >= 8)
-        #define USE_INTEL_VAES
-        #define ATTRIB_VAES __attribute__((__target__("aes,vaes,avx2")))
-      #endif
-    #endif
-  #elif defined(__GNUC__)
-    #if (__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 4)
-      #define USE_INTEL_AES
-      #ifndef __AES__
-        #define ATTRIB_AES __attribute__((__target__("aes")))
-      #endif
-      #if (__GNUC__ >= 8)
-        #define USE_INTEL_VAES
-        #define ATTRIB_VAES __attribute__((__target__("aes,vaes,avx2")))
-      #endif
-    #endif
-  #elif defined(__INTEL_COMPILER)
+  #if defined(__INTEL_COMPILER)
     #if (__INTEL_COMPILER >= 1110)
       #define USE_INTEL_AES
       #if (__INTEL_COMPILER >= 1900)
         #define USE_INTEL_VAES
       #endif
     #endif
+  #elif defined(__clang__) && (__clang_major__ > 3 || __clang_major__ == 3 && __clang_minor__ >= 8) \
+       || defined(__GNUC__) && (__GNUC__ > 4 || __GNUC__ == 4 && __GNUC_MINOR__ >= 4)
+        #define USE_INTEL_AES
+        #if !defined(__AES__)
+          #define ATTRIB_AES __attribute__((__target__("aes")))
+        #endif
+      #if defined(__clang__) && (__clang_major__ >= 8) \
+          || defined(__GNUC__) && (__GNUC__ >= 8)
+        #define USE_INTEL_VAES
+        #if !defined(__AES__) || !defined(__VAES__) || !defined(__AVX__) || !defined(__AVX2__)
+          #define ATTRIB_VAES __attribute__((__target__("aes,vaes,avx,avx2")))
+        #endif
+      #endif
   #elif defined(_MSC_VER)
     #if (_MSC_VER > 1500) || (_MSC_FULL_VER >= 150030729)
       #define USE_INTEL_AES
@@ -56,12 +50,15 @@
 #include <wmmintrin.h>
 
 #ifndef USE_INTEL_VAES
-#define AES_TYPE_keys __m128i
-#define AES_TYPE_data __m128i
+#define AES_TYPE_keys UInt32
+#define AES_TYPE_data Byte
+// #define AES_TYPE_keys __m128i
+// #define AES_TYPE_data __m128i
 #endif
 
 #define AES_FUNC_START(name) \
-    void MY_FAST_CALL name(__m128i *p, __m128i *data, size_t numBlocks)
+    void Z7_FASTCALL name(UInt32 *ivAes, Byte *data8, size_t numBlocks)
+    // void Z7_FASTCALL name(__m128i *p, __m128i *data, size_t numBlocks)
 
 #define AES_FUNC_START2(name) \
 AES_FUNC_START (name); \
@@ -69,14 +66,16 @@ ATTRIB_AES \
 AES_FUNC_START (name)
 
 #define MM_OP(op, dest, src)  dest = op(dest, src);
-#define MM_OP_m(op, src)      MM_OP(op, m, src);
+#define MM_OP_m(op, src)      MM_OP(op, m, src)
 
-#define MM_XOR( dest, src)    MM_OP(_mm_xor_si128,    dest, src);
-#define AVX_XOR(dest, src)    MM_OP(_mm256_xor_si256, dest, src);
+#define MM_XOR( dest, src)    MM_OP(_mm_xor_si128,    dest, src)
+#define AVX_XOR(dest, src)    MM_OP(_mm256_xor_si256, dest, src)
 
 
 AES_FUNC_START2 (AesCbc_Encode_HW)
 {
+  __m128i *p = (__m128i *)(void *)ivAes;
+  __m128i *data = (__m128i *)(void *)data8;
   __m128i m = *p;
   const __m128i k0 = p[2];
   const __m128i k1 = p[3];
@@ -86,17 +85,17 @@ AES_FUNC_START2 (AesCbc_Encode_HW)
     UInt32 r = numRounds2;
     const __m128i *w = p + 4;
     __m128i temp = *data;
-    MM_XOR (temp, k0);
-    MM_XOR (m, temp);
-    MM_OP_m (_mm_aesenc_si128, k1);
+    MM_XOR (temp, k0)
+    MM_XOR (m, temp)
+    MM_OP_m (_mm_aesenc_si128, k1)
     do
     {
-      MM_OP_m (_mm_aesenc_si128, w[0]);
-      MM_OP_m (_mm_aesenc_si128, w[1]);
+      MM_OP_m (_mm_aesenc_si128, w[0])
+      MM_OP_m (_mm_aesenc_si128, w[1])
       w += 2;
     }
     while (--r);
-    MM_OP_m (_mm_aesenclast_si128, w[0]);
+    MM_OP_m (_mm_aesenclast_si128, w[0])
     *data = m;
   }
   *p = m;
@@ -104,14 +103,14 @@ AES_FUNC_START2 (AesCbc_Encode_HW)
 
 
 #define WOP_1(op)
-#define WOP_2(op)   WOP_1 (op)  op (m1, 1);
-#define WOP_3(op)   WOP_2 (op)  op (m2, 2);
-#define WOP_4(op)   WOP_3 (op)  op (m3, 3);
+#define WOP_2(op)   WOP_1 (op)  op (m1, 1)
+#define WOP_3(op)   WOP_2 (op)  op (m2, 2)
+#define WOP_4(op)   WOP_3 (op)  op (m3, 3)
 #ifdef MY_CPU_AMD64
-#define WOP_5(op)   WOP_4 (op)  op (m4, 4);
-#define WOP_6(op)   WOP_5 (op)  op (m5, 5);
-#define WOP_7(op)   WOP_6 (op)  op (m6, 6);
-#define WOP_8(op)   WOP_7 (op)  op (m7, 7);
+#define WOP_5(op)   WOP_4 (op)  op (m4, 4)
+#define WOP_6(op)   WOP_5 (op)  op (m5, 5)
+#define WOP_7(op)   WOP_6 (op)  op (m6, 6)
+#define WOP_8(op)   WOP_7 (op)  op (m7, 7)
 #endif
 /*
 #define WOP_9(op)   WOP_8 (op)  op (m8, 8);
@@ -130,20 +129,20 @@ AES_FUNC_START2 (AesCbc_Encode_HW)
   #define WOP_M1    WOP_4
 #endif
 
-#define WOP(op)  op (m0, 0);  WOP_M1(op)
+#define WOP(op)  op (m0, 0)  WOP_M1(op)
 
 
-#define DECLARE_VAR(reg, ii)  __m128i reg
+#define DECLARE_VAR(reg, ii)  __m128i reg;
 #define LOAD_data(  reg, ii)  reg = data[ii];
 #define STORE_data( reg, ii)  data[ii] = reg;
 #if (NUM_WAYS > 1)
-#define XOR_data_M1(reg, ii)  MM_XOR (reg, data[ii- 1]);
+#define XOR_data_M1(reg, ii)  MM_XOR (reg, data[ii- 1])
 #endif
 
-#define AVX__DECLARE_VAR(reg, ii)  __m256i reg
-#define AVX__LOAD_data(  reg, ii)  reg = ((const __m256i *)(const void *)data)[ii];
-#define AVX__STORE_data( reg, ii)  ((__m256i *)(void *)data)[ii] = reg;
-#define AVX__XOR_data_M1(reg, ii)  AVX_XOR (reg, (((const __m256i *)(const void *)(data - 1))[ii]));
+#define AVX_DECLARE_VAR(reg, ii)  __m256i reg;
+#define AVX_LOAD_data(  reg, ii)  reg = ((const __m256i *)(const void *)data)[ii];
+#define AVX_STORE_data( reg, ii)  ((__m256i *)(void *)data)[ii] = reg;
+#define AVX_XOR_data_M1(reg, ii)  AVX_XOR (reg, (((const __m256i *)(const void *)(data - 1))[ii]))
 
 #define MM_OP_key(op, reg)  MM_OP(op, reg, key);
 
@@ -154,23 +153,23 @@ AES_FUNC_START2 (AesCbc_Encode_HW)
 #define AES_XOR(      reg, ii)   MM_OP_key (_mm_xor_si128,        reg)
 
 
-#define AVX__AES_DEC(      reg, ii)   MM_OP_key (_mm256_aesdec_epi128,     reg)
-#define AVX__AES_DEC_LAST( reg, ii)   MM_OP_key (_mm256_aesdeclast_epi128, reg)
-#define AVX__AES_ENC(      reg, ii)   MM_OP_key (_mm256_aesenc_epi128,     reg)
-#define AVX__AES_ENC_LAST( reg, ii)   MM_OP_key (_mm256_aesenclast_epi128, reg)
-#define AVX__AES_XOR(      reg, ii)   MM_OP_key (_mm256_xor_si256,         reg)
+#define AVX_AES_DEC(      reg, ii)   MM_OP_key (_mm256_aesdec_epi128,     reg)
+#define AVX_AES_DEC_LAST( reg, ii)   MM_OP_key (_mm256_aesdeclast_epi128, reg)
+#define AVX_AES_ENC(      reg, ii)   MM_OP_key (_mm256_aesenc_epi128,     reg)
+#define AVX_AES_ENC_LAST( reg, ii)   MM_OP_key (_mm256_aesenclast_epi128, reg)
+#define AVX_AES_XOR(      reg, ii)   MM_OP_key (_mm256_xor_si256,         reg)
 
-#define CTR_START(reg, ii)  MM_OP (_mm_add_epi64, ctr, one); reg = ctr;
-#define CTR_END(  reg, ii)  MM_XOR (data[ii], reg);
+#define CTR_START(reg, ii)  MM_OP (_mm_add_epi64, ctr, one)  reg = ctr;
+#define CTR_END(  reg, ii)  MM_XOR (data[ii], reg)
 
-#define AVX__CTR_START(reg, ii)  MM_OP (_mm256_add_epi64, ctr2, two); reg = _mm256_xor_si256(ctr2, key);
-#define AVX__CTR_END(  reg, ii)  AVX_XOR (((__m256i *)(void *)data)[ii], reg);
+#define AVX_CTR_START(reg, ii)  MM_OP (_mm256_add_epi64, ctr2, two)  reg = _mm256_xor_si256(ctr2, key);
+#define AVX_CTR_END(  reg, ii)  AVX_XOR (((__m256i *)(void *)data)[ii], reg)
 
 #define WOP_KEY(op, n) { \
     const __m128i key = w[n]; \
     WOP(op); }
 
-#define AVX__WOP_KEY(op, n) { \
+#define AVX_WOP_KEY(op, n) { \
     const __m256i key = w[n]; \
     WOP(op); }
 
@@ -218,6 +217,8 @@ AES_FUNC_START2 (AesCbc_Encode_HW)
 
 AES_FUNC_START2 (AesCbc_Decode_HW)
 {
+  __m128i *p = (__m128i *)(void *)ivAes;
+  __m128i *data = (__m128i *)(void *)data8;
   __m128i iv = *p;
   const __m128i *wStart = p + *(const UInt32 *)(p + 1) * 2 + 2 - 1;
   const __m128i *dataEnd;
@@ -228,7 +229,7 @@ AES_FUNC_START2 (AesCbc_Decode_HW)
     const __m128i *w = wStart;
     
     WOP (DECLARE_VAR)
-    WOP (LOAD_data);
+    WOP (LOAD_data)
     WOP_KEY (AES_XOR, 1)
 
     do
@@ -239,10 +240,10 @@ AES_FUNC_START2 (AesCbc_Decode_HW)
     while (w != p);
     WOP_KEY (AES_DEC_LAST, 0)
 
-    MM_XOR (m0, iv);
+    MM_XOR (m0, iv)
     WOP_M1 (XOR_data_M1)
     iv = data[NUM_WAYS - 1];
-    WOP (STORE_data);
+    WOP (STORE_data)
   }
   WIDE_LOOP_END
 
@@ -252,15 +253,15 @@ AES_FUNC_START2 (AesCbc_Decode_HW)
     __m128i m = _mm_xor_si128 (w[2], *data);
     do
     {
-      MM_OP_m (_mm_aesdec_si128, w[1]);
-      MM_OP_m (_mm_aesdec_si128, w[0]);
+      MM_OP_m (_mm_aesdec_si128, w[1])
+      MM_OP_m (_mm_aesdec_si128, w[0])
       w -= 2;
     }
     while (w != p);
-    MM_OP_m (_mm_aesdec_si128,     w[1]);
-    MM_OP_m (_mm_aesdeclast_si128, w[0]);
+    MM_OP_m (_mm_aesdec_si128,     w[1])
+    MM_OP_m (_mm_aesdeclast_si128, w[0])
 
-    MM_XOR (m, iv);
+    MM_XOR (m, iv)
     iv = *data;
     *data = m;
   }
@@ -271,6 +272,8 @@ AES_FUNC_START2 (AesCbc_Decode_HW)
 
 AES_FUNC_START2 (AesCtr_Code_HW)
 {
+  __m128i *p = (__m128i *)(void *)ivAes;
+  __m128i *data = (__m128i *)(void *)data8;
   __m128i ctr = *p;
   UInt32 numRoundsMinus2 = *(const UInt32 *)(p + 1) * 2 - 1;
   const __m128i *dataEnd;
@@ -283,7 +286,7 @@ AES_FUNC_START2 (AesCtr_Code_HW)
     const __m128i *w = p;
     UInt32 r = numRoundsMinus2;
     WOP (DECLARE_VAR)
-    WOP (CTR_START);
+    WOP (CTR_START)
     WOP_KEY (AES_XOR, 0)
     w += 1;
     do
@@ -294,7 +297,7 @@ AES_FUNC_START2 (AesCtr_Code_HW)
     while (--r);
     WOP_KEY (AES_ENC_LAST, 0)
    
-    WOP (CTR_END);
+    WOP (CTR_END)
   }
   WIDE_LOOP_END
 
@@ -303,19 +306,19 @@ AES_FUNC_START2 (AesCtr_Code_HW)
     UInt32 numRounds2 = *(const UInt32 *)(p - 2 + 1) - 1;
     const __m128i *w = p;
     __m128i m;
-    MM_OP (_mm_add_epi64, ctr, one);
+    MM_OP (_mm_add_epi64, ctr, one)
     m = _mm_xor_si128 (ctr, p[0]);
     w += 1;
     do
     {
-      MM_OP_m (_mm_aesenc_si128, w[0]);
-      MM_OP_m (_mm_aesenc_si128, w[1]);
+      MM_OP_m (_mm_aesenc_si128, w[0])
+      MM_OP_m (_mm_aesenc_si128, w[1])
       w += 2;
     }
     while (--numRounds2);
-    MM_OP_m (_mm_aesenc_si128,     w[0]);
-    MM_OP_m (_mm_aesenclast_si128, w[1]);
-    MM_XOR (*data, m);
+    MM_OP_m (_mm_aesenc_si128,     w[0])
+    MM_OP_m (_mm_aesenclast_si128, w[1])
+    MM_XOR (*data, m)
   }
   
   p[-2] = ctr;
@@ -325,17 +328,58 @@ AES_FUNC_START2 (AesCtr_Code_HW)
 
 #ifdef USE_INTEL_VAES
 
+/*
+GCC before 2013-Jun:
+  <immintrin.h>:
+    #ifdef __AVX__
+     #include <avxintrin.h>
+    #endif
+GCC after 2013-Jun:
+  <immintrin.h>:
+    #include <avxintrin.h>
+CLANG 3.8+:
+{
+  <immintrin.h>:
+    #if !defined(_MSC_VER) || defined(__AVX__)
+      #include <avxintrin.h>
+    #endif
+
+  if (the compiler is clang for Windows and if global arch is not set for __AVX__)
+    [ if (defined(_MSC_VER) && !defined(__AVX__)) ]
+  {
+    <immintrin.h> doesn't include <avxintrin.h>
+    and we have 2 ways to fix it:
+      1) we can define required __AVX__ before <immintrin.h>
+      or
+      2) we can include <avxintrin.h> after <immintrin.h>
+  }
+}
+
+If we include <avxintrin.h> manually for GCC/CLANG, it's
+required that <immintrin.h> must be included before <avxintrin.h>.
+*/
+
+/*
 #if defined(__clang__) && defined(_MSC_VER)
-#define __SSE4_2__
-#define __AES__
 #define __AVX__
 #define __AVX2__
 #define __VAES__
-#define __AVX512F__
-#define __AVX512VL__
 #endif
+*/
 
 #include <immintrin.h>
+#if defined(__clang__) && defined(_MSC_VER)
+  #if !defined(__AVX__)
+    #include <avxintrin.h>
+  #endif
+  #if !defined(__AVX2__)
+    #include <avx2intrin.h>
+  #endif
+  #if !defined(__VAES__)
+    #include <vaesintrin.h>
+  #endif
+#endif  // __clang__ && _MSC_VER
+
 
 #define VAES_FUNC_START2(name) \
 AES_FUNC_START (name); \
@@ -344,6 +388,8 @@ AES_FUNC_START (name)
 
 VAES_FUNC_START2 (AesCbc_Decode_HW_256)
 {
+  __m128i *p = (__m128i *)(void *)ivAes;
+  __m128i *data = (__m128i *)(void *)data8;
   __m128i iv = *p;
   const __m128i *dataEnd;
   UInt32 numRounds = *(const UInt32 *)(p + 1) * 2 + 1;
@@ -353,22 +399,22 @@ VAES_FUNC_START2 (AesCbc_Decode_HW_256)
   {
     const __m256i *w = keys + numRounds - 2;
     
-    WOP (AVX__DECLARE_VAR)
-    WOP (AVX__LOAD_data);
-    AVX__WOP_KEY (AVX__AES_XOR, 1)
+    WOP (AVX_DECLARE_VAR)
+    WOP (AVX_LOAD_data)
+    AVX_WOP_KEY (AVX_AES_XOR, 1)
 
     do
     {
-      AVX__WOP_KEY (AVX__AES_DEC, 0)
+      AVX_WOP_KEY (AVX_AES_DEC, 0)
       w--;
     }
     while (w != keys);
-    AVX__WOP_KEY (AVX__AES_DEC_LAST, 0)
+    AVX_WOP_KEY (AVX_AES_DEC_LAST, 0)
 
-    AVX_XOR (m0, _mm256_setr_m128i(iv, data[0]));
-    WOP_M1 (AVX__XOR_data_M1)
+    AVX_XOR (m0, _mm256_setr_m128i(iv, data[0]))
+    WOP_M1 (AVX_XOR_data_M1)
     iv = data[NUM_WAYS * 2 - 1];
-    WOP (AVX__STORE_data);
+    WOP (AVX_STORE_data)
   }
   WIDE_LOOP_END_AVX(;)
 
@@ -378,15 +424,15 @@ VAES_FUNC_START2 (AesCbc_Decode_HW_256)
     __m128i m = _mm_xor_si128 (w[2], *data);
     do
     {
-      MM_OP_m (_mm_aesdec_si128, w[1]);
-      MM_OP_m (_mm_aesdec_si128, w[0]);
+      MM_OP_m (_mm_aesdec_si128, w[1])
+      MM_OP_m (_mm_aesdec_si128, w[0])
       w -= 2;
     }
     while (w != p);
-    MM_OP_m (_mm_aesdec_si128,     w[1]);
-    MM_OP_m (_mm_aesdeclast_si128, w[0]);
+    MM_OP_m (_mm_aesdec_si128,     w[1])
+    MM_OP_m (_mm_aesdeclast_si128, w[0])
 
-    MM_XOR (m, iv);
+    MM_XOR (m, iv)
     iv = *data;
     *data = m;
   }
@@ -403,18 +449,20 @@ AVX2: _mm256_add_epi64             : vpaddq ymm, ymm, ymm
       _mm256_broadcastsi128_si256  : vbroadcasti128
 */
 
-#define AVX__CTR_LOOP_START  \
+#define AVX_CTR_LOOP_START  \
     ctr2 = _mm256_setr_m128i(_mm_sub_epi64(ctr, one), ctr); \
     two = _mm256_setr_m128i(one, one); \
     two = _mm256_add_epi64(two, two); \
 
 // two = _mm256_setr_epi64x(2, 0, 2, 0);
   
-#define AVX__CTR_LOOP_ENC  \
+#define AVX_CTR_LOOP_ENC  \
     ctr = _mm256_extracti128_si256 (ctr2, 1); \
  
 VAES_FUNC_START2 (AesCtr_Code_HW_256)
 {
+  __m128i *p = (__m128i *)(void *)ivAes;
+  __m128i *data = (__m128i *)(void *)data8;
   __m128i ctr = *p;
   UInt32 numRounds = *(const UInt32 *)(p + 1) * 2 + 1;
   const __m128i *dataEnd;
@@ -422,44 +470,44 @@ VAES_FUNC_START2 (AesCtr_Code_HW_256)
   __m256i ctr2, two;
   p += 2;
   
-  WIDE_LOOP_START_AVX (AVX__CTR_LOOP_START)
+  WIDE_LOOP_START_AVX (AVX_CTR_LOOP_START)
   {
     const __m256i *w = keys;
     UInt32 r = numRounds - 2;
-    WOP (AVX__DECLARE_VAR)
-    AVX__WOP_KEY (AVX__CTR_START, 0);
+    WOP (AVX_DECLARE_VAR)
+    AVX_WOP_KEY (AVX_CTR_START, 0)
 
     w += 1;
     do
     {
-      AVX__WOP_KEY (AVX__AES_ENC, 0)
+      AVX_WOP_KEY (AVX_AES_ENC, 0)
       w += 1;
     }
     while (--r);
-    AVX__WOP_KEY (AVX__AES_ENC_LAST, 0)
+    AVX_WOP_KEY (AVX_AES_ENC_LAST, 0)
    
-    WOP (AVX__CTR_END);
+    WOP (AVX_CTR_END)
   }
-  WIDE_LOOP_END_AVX (AVX__CTR_LOOP_ENC)
+  WIDE_LOOP_END_AVX (AVX_CTR_LOOP_ENC)
   
   SINGLE_LOOP
   {
     UInt32 numRounds2 = *(const UInt32 *)(p - 2 + 1) - 1;
     const __m128i *w = p;
     __m128i m;
-    MM_OP (_mm_add_epi64, ctr, one);
+    MM_OP (_mm_add_epi64, ctr, one)
     m = _mm_xor_si128 (ctr, p[0]);
     w += 1;
     do
     {
-      MM_OP_m (_mm_aesenc_si128, w[0]);
-      MM_OP_m (_mm_aesenc_si128, w[1]);
+      MM_OP_m (_mm_aesenc_si128, w[0])
+      MM_OP_m (_mm_aesenc_si128, w[1])
       w += 2;
     }
     while (--numRounds2);
-    MM_OP_m (_mm_aesenc_si128,     w[0]);
-    MM_OP_m (_mm_aesenclast_si128, w[1]);
-    MM_XOR (*data, m);
+    MM_OP_m (_mm_aesenc_si128,     w[0])
+    MM_OP_m (_mm_aesenclast_si128, w[1])
+    MM_XOR (*data, m)
   }
 
   p[-2] = ctr;
@@ -477,7 +525,7 @@ VAES_FUNC_START2 (AesCtr_Code_HW_256)
 #define AES_TYPE_data Byte
 
 #define AES_FUNC_START(name) \
-    void MY_FAST_CALL name(UInt32 *p, Byte *data, size_t numBlocks) \
+    void Z7_FASTCALL name(UInt32 *p, Byte *data, size_t numBlocks) \
 
 #define AES_COMPAT_STUB(name) \
     AES_FUNC_START(name); \
@@ -496,8 +544,8 @@ AES_COMPAT_STUB (AesCtr_Code)
 #pragma message("VAES HW_SW stub was used")
 
 #define VAES_COMPAT_STUB(name) \
-    void MY_FAST_CALL name ## _256(UInt32 *p, Byte *data, size_t numBlocks); \
-    void MY_FAST_CALL name ## _256(UInt32 *p, Byte *data, size_t numBlocks) \
+    void Z7_FASTCALL name ## _256(UInt32 *p, Byte *data, size_t numBlocks); \
+    void Z7_FASTCALL name ## _256(UInt32 *p, Byte *data, size_t numBlocks) \
     { name((AES_TYPE_keys *)(void *)p, (AES_TYPE_data *)(void *)data, numBlocks); }
 
 VAES_COMPAT_STUB (AesCbc_Decode_HW)
@@ -551,7 +599,8 @@ VAES_COMPAT_STUB (AesCtr_Code_HW)
 typedef uint8x16_t v128;
 
 #define AES_FUNC_START(name) \
-    void MY_FAST_CALL name(v128 *p, v128 *data, size_t numBlocks)
+    void Z7_FASTCALL name(UInt32 *ivAes, Byte *data8, size_t numBlocks)
+    // void Z7_FASTCALL name(v128 *p, v128 *data, size_t numBlocks)
 
 #define AES_FUNC_START2(name) \
 AES_FUNC_START (name); \
@@ -559,18 +608,20 @@ ATTRIB_AES \
 AES_FUNC_START (name)
 
 #define MM_OP(op, dest, src)  dest = op(dest, src);
-#define MM_OP_m(op, src)      MM_OP(op, m, src);
+#define MM_OP_m(op, src)      MM_OP(op, m, src)
 #define MM_OP1_m(op)          m = op(m);
 
-#define MM_XOR( dest, src)    MM_OP(veorq_u8, dest, src);
-#define MM_XOR_m( src)        MM_XOR(m, src);
+#define MM_XOR( dest, src)    MM_OP(veorq_u8, dest, src)
+#define MM_XOR_m( src)        MM_XOR(m, src)
 
-#define AES_E_m(k)     MM_OP_m (vaeseq_u8, k);
-#define AES_E_MC_m(k)  AES_E_m (k);  MM_OP1_m(vaesmcq_u8);
+#define AES_E_m(k)     MM_OP_m (vaeseq_u8, k)
+#define AES_E_MC_m(k)  AES_E_m (k)  MM_OP1_m(vaesmcq_u8)
 
 
 AES_FUNC_START2 (AesCbc_Encode_HW)
 {
+  v128 *p = (v128*)(void*)ivAes;
+  v128 *data = (v128*)(void*)data8;
   v128 m = *p;
   const v128 k0 = p[2];
   const v128 k1 = p[3];
@@ -608,7 +659,7 @@ AES_FUNC_START2 (AesCbc_Encode_HW)
         AES_E_MC_m (p[14])
       }
     }
-    AES_E_m  (k_z1);
+    AES_E_m  (k_z1)
     MM_XOR_m (k_z0);
     *data = m;
   }
@@ -617,44 +668,44 @@ AES_FUNC_START2 (AesCbc_Encode_HW)
 
 
 #define WOP_1(op)
-#define WOP_2(op)   WOP_1 (op)  op (m1, 1);
-#define WOP_3(op)   WOP_2 (op)  op (m2, 2);
-#define WOP_4(op)   WOP_3 (op)  op (m3, 3);
-#define WOP_5(op)   WOP_4 (op)  op (m4, 4);
-#define WOP_6(op)   WOP_5 (op)  op (m5, 5);
-#define WOP_7(op)   WOP_6 (op)  op (m6, 6);
-#define WOP_8(op)   WOP_7 (op)  op (m7, 7);
+#define WOP_2(op)   WOP_1 (op)  op (m1, 1)
+#define WOP_3(op)   WOP_2 (op)  op (m2, 2)
+#define WOP_4(op)   WOP_3 (op)  op (m3, 3)
+#define WOP_5(op)   WOP_4 (op)  op (m4, 4)
+#define WOP_6(op)   WOP_5 (op)  op (m5, 5)
+#define WOP_7(op)   WOP_6 (op)  op (m6, 6)
+#define WOP_8(op)   WOP_7 (op)  op (m7, 7)
 
   #define NUM_WAYS      8
   #define WOP_M1    WOP_8
 
-#define WOP(op)  op (m0, 0);  WOP_M1(op)
+#define WOP(op)  op (m0, 0)   WOP_M1(op)
 
-#define DECLARE_VAR(reg, ii)  v128 reg
+#define DECLARE_VAR(reg, ii)  v128 reg;
 #define LOAD_data(  reg, ii)  reg = data[ii];
 #define STORE_data( reg, ii)  data[ii] = reg;
 #if (NUM_WAYS > 1)
-#define XOR_data_M1(reg, ii)  MM_XOR (reg, data[ii- 1]);
+#define XOR_data_M1(reg, ii)  MM_XOR (reg, data[ii- 1])
 #endif
 
-#define MM_OP_key(op, reg)  MM_OP (op, reg, key);
+#define MM_OP_key(op, reg)  MM_OP (op, reg, key)
 
-#define AES_D_m(k)      MM_OP_m (vaesdq_u8, k);
-#define AES_D_IMC_m(k)  AES_D_m (k);  MM_OP1_m (vaesimcq_u8);
+#define AES_D_m(k)      MM_OP_m (vaesdq_u8, k)
+#define AES_D_IMC_m(k)  AES_D_m (k)  MM_OP1_m (vaesimcq_u8)
 
 #define AES_XOR(   reg, ii)  MM_OP_key (veorq_u8,  reg)
 #define AES_D(     reg, ii)  MM_OP_key (vaesdq_u8, reg)
 #define AES_E(     reg, ii)  MM_OP_key (vaeseq_u8, reg)
 
-#define AES_D_IMC( reg, ii)  AES_D (reg, ii);  reg = vaesimcq_u8(reg)
-#define AES_E_MC(  reg, ii)  AES_E (reg, ii);  reg = vaesmcq_u8(reg)
+#define AES_D_IMC( reg, ii)  AES_D (reg, ii)  reg = vaesimcq_u8(reg);
+#define AES_E_MC(  reg, ii)  AES_E (reg, ii)  reg = vaesmcq_u8(reg);
 
-#define CTR_START(reg, ii)  MM_OP (vaddq_u64, ctr, one);  reg = vreinterpretq_u8_u64(ctr);
-#define CTR_END(  reg, ii)  MM_XOR (data[ii], reg);
+#define CTR_START(reg, ii)  MM_OP (vaddq_u64, ctr, one)  reg = vreinterpretq_u8_u64(ctr);
+#define CTR_END(  reg, ii)  MM_XOR (data[ii], reg)
 
 #define WOP_KEY(op, n) { \
     const v128 key = w[n]; \
-    WOP(op); }
+    WOP(op) }
 
 #define WIDE_LOOP_START  \
     dataEnd = data + numBlocks;  \
@@ -672,6 +723,8 @@ AES_FUNC_START2 (AesCbc_Encode_HW)
 
 AES_FUNC_START2 (AesCbc_Decode_HW)
 {
+  v128 *p = (v128*)(void*)ivAes;
+  v128 *data = (v128*)(void*)data8;
   v128 iv = *p;
   const v128 *wStart = p + ((size_t)*(const UInt32 *)(p + 1)) * 2;
   const v128 *dataEnd;
@@ -681,7 +734,7 @@ AES_FUNC_START2 (AesCbc_Decode_HW)
   {
     const v128 *w = wStart;
     WOP (DECLARE_VAR)
-    WOP (LOAD_data);
+    WOP (LOAD_data)
     WOP_KEY (AES_D_IMC, 2)
     do
     {
@@ -695,7 +748,7 @@ AES_FUNC_START2 (AesCbc_Decode_HW)
     MM_XOR (m0, iv);
     WOP_M1 (XOR_data_M1)
     iv = data[NUM_WAYS - 1];
-    WOP (STORE_data);
+    WOP (STORE_data)
   }
   WIDE_LOOP_END
 
@@ -724,6 +777,8 @@ AES_FUNC_START2 (AesCbc_Decode_HW)
 
 AES_FUNC_START2 (AesCtr_Code_HW)
 {
+  v128 *p = (v128*)(void*)ivAes;
+  v128 *data = (v128*)(void*)data8;
   uint64x2_t ctr = vreinterpretq_u64_u8(*p);
   const v128 *wEnd = p + ((size_t)*(const UInt32 *)(p + 1)) * 2;
   const v128 *dataEnd;
@@ -735,7 +790,7 @@ AES_FUNC_START2 (AesCtr_Code_HW)
   {
     const v128 *w = p;
     WOP (DECLARE_VAR)
-    WOP (CTR_START);
+    WOP (CTR_START)
     do
     {
       WOP_KEY (AES_E_MC, 0)
@@ -746,7 +801,7 @@ AES_FUNC_START2 (AesCtr_Code_HW)
     WOP_KEY (AES_E_MC, 0)
     WOP_KEY (AES_E,    1)
     WOP_KEY (AES_XOR,  2)
-    WOP (CTR_END);
+    WOP (CTR_END)
   }
   WIDE_LOOP_END
 
@@ -762,10 +817,10 @@ AES_FUNC_START2 (AesCtr_Code_HW)
       w += 2;
     }
     while (w != wEnd);
-    AES_E_MC_m (w[0]);
-    AES_E_m    (w[1]);
-    MM_XOR_m   (w[2]);
-    CTR_END (m, 0);
+    AES_E_MC_m (w[0])
+    AES_E_m    (w[1])
+    MM_XOR_m   (w[2])
+    CTR_END (m, 0)
   }
   
   p[-2] = vreinterpretq_u8_u64(ctr);
@@ -774,3 +829,12 @@ AES_FUNC_START2 (AesCtr_Code_HW)
 #endif // USE_HW_AES
 
 #endif // MY_CPU_ARM_OR_ARM64
+
+#undef NUM_WAYS
+#undef WOP_M1
+#undef WOP
+#undef DECLARE_VAR
+#undef LOAD_data
+#undef STORE_data
+#undef USE_INTEL_AES
+#undef USE_HW_AES