24.0524.05

1 files changed, 215 insertions, 137 deletions

diff --git a/Asm/x86/7zCrcOpt.asm b/Asm/x86/7zCrcOpt.asm
index 0fee206..c5de808 100644
--- a/Asm/x86/7zCrcOpt.asm
+++ b/Asm/x86/7zCrcOpt.asm
@@ -1,180 +1,258 @@
 ; 7zCrcOpt.asm -- CRC32 calculation : optimized version
-; 2021-02-07 : Igor Pavlov : Public domain
+; 2023-12-08 : Igor Pavlov : Public domain
 
 include 7zAsm.asm
 
 MY_ASM_START
 
-rD   equ  r2
-rN   equ  r7
-rT   equ  r5
+NUM_WORDS       equ     3
+UNROLL_CNT      equ     2
 
-ifdef x64
-    num_VAR     equ r8
-    table_VAR   equ r9
-else
-  if (IS_CDECL gt 0)
-    crc_OFFS    equ (REG_SIZE * 5)
-    data_OFFS   equ (REG_SIZE + crc_OFFS)
-    size_OFFS   equ (REG_SIZE + data_OFFS)
-  else
-    size_OFFS   equ (REG_SIZE * 5)
-  endif
-    table_OFFS  equ (REG_SIZE + size_OFFS)
-    num_VAR     equ [r4 + size_OFFS]
-    table_VAR   equ [r4 + table_OFFS]
+if (NUM_WORDS lt 1) or (NUM_WORDS gt 64)
+.err <NUM_WORDS_IS_INCORRECT>
+endif
+if (UNROLL_CNT lt 1)
+.err <UNROLL_CNT_IS_INCORRECT>
 endif
 
-SRCDAT  equ  rD + rN * 1 + 4 *
+rD      equ  r2
+rD_x    equ  x2
+rN      equ  r7
+rT      equ  r5
+
+ifndef x64
+    if (IS_CDECL gt 0)
+        crc_OFFS    equ (REG_SIZE * 5)
+        data_OFFS   equ (REG_SIZE + crc_OFFS)
+        size_OFFS   equ (REG_SIZE + data_OFFS)
+    else
+        size_OFFS   equ (REG_SIZE * 5)
+    endif
+        table_OFFS  equ (REG_SIZE + size_OFFS)
+endif
+
+; rN + rD is same speed as rD, but we reduce one instruction in loop
+SRCDAT_1        equ     rN + rD * 1 + 1 *
+SRCDAT_4        equ     rN + rD * 1 + 4 *
 
 CRC macro op:req, dest:req, src:req, t:req
-    op      dest, DWORD PTR [rT + src * 4 + 0400h * t]
+        op      dest, dword ptr [rT + @CatStr(src, _R) * 4 + 0400h * (t)]
 endm
 
 CRC_XOR macro dest:req, src:req, t:req
-    CRC xor, dest, src, t
+        CRC     xor, dest, src, t
 endm
 
 CRC_MOV macro dest:req, src:req, t:req
-    CRC mov, dest, src, t
+        CRC     mov, dest, src, t
+endm
+
+MOVZXLO macro dest:req, src:req
+        movzx   dest, @CatStr(src, _L)
+endm
+
+MOVZXHI macro dest:req, src:req
+        movzx   dest, @CatStr(src, _H)
 endm
 
+; movzx x0, x0_L - is slow in some cpus (ivb), if same register for src and dest
+; movzx x3, x0_L sometimes is 0   cycles latency (not always)
+; movzx x3, x0_L sometimes is 0.5 cycles latency
+; movzx x3, x0_H is 2 cycles latency in some cpus
+
 CRC1b macro
-    movzx   x6, BYTE PTR [rD]
-    inc     rD
-    movzx   x3, x0_L
-    xor     x6, x3
-    shr     x0, 8
-    CRC     xor, x0, r6, 0
-    dec     rN
+        movzx   x6, byte ptr [rD]
+        MOVZXLO x3, x0
+        inc     rD
+        shr     x0, 8
+        xor     x6, x3
+        CRC_XOR x0, x6, 0
+        dec     rN
+endm
+
+LOAD_1 macro dest:req, t:req, iter:req, index:req
+        movzx   dest, byte ptr [SRCDAT_1 (4 * (NUM_WORDS - 1 - t + iter * NUM_WORDS) + index)]
+endm
+
+LOAD_2 macro dest:req, t:req, iter:req, index:req
+        movzx   dest, word ptr [SRCDAT_1 (4 * (NUM_WORDS - 1 - t + iter * NUM_WORDS) + index)]
+endm
+
+CRC_QUAD macro nn, t:req, iter:req
+ifdef x64
+        ; paired memory loads give 1-3% speed gain, but it uses more registers
+        LOAD_2  x3, t, iter, 0
+        LOAD_2  x9, t, iter, 2
+        MOVZXLO x6, x3
+        shr     x3, 8
+        CRC_XOR nn, x6, t * 4 + 3
+        MOVZXLO x6, x9
+        shr     x9, 8
+        CRC_XOR nn, x3, t * 4 + 2
+        CRC_XOR nn, x6, t * 4 + 1
+        CRC_XOR nn, x9, t * 4 + 0
+elseif 0
+        LOAD_2  x3, t, iter, 0
+        MOVZXLO x6, x3
+        shr     x3, 8
+        CRC_XOR nn, x6, t * 4 + 3
+        CRC_XOR nn, x3, t * 4 + 2
+        LOAD_2  x3, t, iter, 2
+        MOVZXLO x6, x3
+        shr     x3, 8
+        CRC_XOR nn, x6, t * 4 + 1
+        CRC_XOR nn, x3, t * 4 + 0
+elseif 0
+        LOAD_1  x3, t, iter, 0
+        LOAD_1  x6, t, iter, 1
+        CRC_XOR nn, x3, t * 4 + 3
+        CRC_XOR nn, x6, t * 4 + 2
+        LOAD_1  x3, t, iter, 2
+        LOAD_1  x6, t, iter, 3
+        CRC_XOR nn, x3, t * 4 + 1
+        CRC_XOR nn, x6, t * 4 + 0
+else
+        ; 32-bit load is better if there is only one read port (core2)
+        ; but that code can be slower if there are 2 read ports (snb)
+        mov     x3, dword ptr [SRCDAT_1 (4 * (NUM_WORDS - 1 - t + iter *  NUM_WORDS) + 0)]
+        MOVZXLO x6, x3
+        CRC_XOR nn, x6, t * 4 + 3
+        MOVZXHI x6, x3
+        shr     x3, 16
+        CRC_XOR nn, x6, t * 4 + 2
+        MOVZXLO x6, x3
+        shr     x3, 8
+        CRC_XOR nn, x6, t * 4 + 1
+        CRC_XOR nn, x3, t * 4 + 0
+endif
 endm
 
-MY_PROLOG macro crc_end:req
 
+LAST    equ     (4 * (NUM_WORDS - 1))
+
+CRC_ITER macro qq, nn, iter
+        mov     nn, [SRCDAT_4 (NUM_WORDS * (1 + iter))]
+
+    i = 0
+    rept NUM_WORDS - 1
+        CRC_QUAD nn, i, iter
+        i = i + 1
+    endm
+
+        MOVZXLO x6, qq
+        mov     x3, qq
+        shr     x3, 24
+        CRC_XOR nn, x6, LAST + 3
+        CRC_XOR nn, x3, LAST + 0
+        ror     qq, 16
+        MOVZXLO x6, qq
+        shr     qq, 24
+        CRC_XOR nn, x6, LAST + 1
+if ((UNROLL_CNT and 1) eq 1) and (iter eq (UNROLL_CNT - 1))
+        CRC_MOV qq, qq, LAST + 2
+        xor     qq, nn
+else
+        CRC_XOR nn, qq, LAST + 2
+endif
+endm
+
+
+; + 4 for prefetching next 4-bytes after current iteration
+NUM_BYTES_LIMIT equ     (NUM_WORDS * 4 * UNROLL_CNT + 4)
+ALIGN_MASK      equ     3
+
+
+; MY_PROC @CatStr(CrcUpdateT, 12), 4
+MY_PROC @CatStr(CrcUpdateT, %(NUM_WORDS * 4)), 4
+        MY_PUSH_PRESERVED_ABI_REGS_UP_TO_INCLUDING_R11
     ifdef x64
+        mov     x0, REG_ABI_PARAM_0_x   ; x0 = x1(win) / x7(linux)
+        mov     rT, REG_ABI_PARAM_3     ; r5 = r9(win) / x1(linux)
+        mov     rN, REG_ABI_PARAM_2     ; r7 = r8(win) / r2(linux)
+        ; mov     rD, REG_ABI_PARAM_1     ; r2 = r2(win)
       if  (IS_LINUX gt 0)
-        MY_PUSH_2_REGS
-        mov     x0, REG_ABI_PARAM_0_x   ; x0 = x7
-        mov     rT, REG_ABI_PARAM_3     ; r5 = r1
-        mov     rN, REG_ABI_PARAM_2     ; r7 = r2
         mov     rD, REG_ABI_PARAM_1     ; r2 = r6
-      else
-        MY_PUSH_4_REGS
-        mov     x0, REG_ABI_PARAM_0_x   ; x0 = x1
-        mov     rT, REG_ABI_PARAM_3     ; r5 = r9
-        mov     rN, REG_ABI_PARAM_2     ; r7 = r8
-        ; mov     rD, REG_ABI_PARAM_1     ; r2 = r2
       endif
     else
-        MY_PUSH_4_REGS
       if  (IS_CDECL gt 0)
         mov     x0, [r4 + crc_OFFS]
         mov     rD, [r4 + data_OFFS]
       else
         mov     x0, REG_ABI_PARAM_0_x
       endif
-        mov     rN, num_VAR
-        mov     rT, table_VAR
+        mov     rN, [r4 + size_OFFS]
+        mov     rT, [r4 + table_OFFS]
     endif
     
-    test    rN, rN
-    jz      crc_end
-  @@:
-    test    rD, 7
-    jz      @F
-    CRC1b
-    jnz     @B
-  @@:
-    cmp     rN, 16
-    jb      crc_end
-    add     rN, rD
-    mov     num_VAR, rN
-    sub     rN, 8
-    and     rN, NOT 7
-    sub     rD, rN
-    xor     x0, [SRCDAT 0]
-endm
+        cmp     rN, NUM_BYTES_LIMIT + ALIGN_MASK
+        jb      crc_end
+@@:
+        test    rD_x, ALIGN_MASK    ; test    rD, ALIGN_MASK
+        jz      @F
+        CRC1b
+        jmp     @B
+@@:
+        xor     x0, dword ptr [rD]
+        lea     rN, [rD + rN * 1 - (NUM_BYTES_LIMIT - 1)]
+        sub     rD, rN
 
-MY_EPILOG macro crc_end:req
-    xor     x0, [SRCDAT 0]
-    mov     rD, rN
-    mov     rN, num_VAR
-    sub     rN, rD
-  crc_end:
-    test    rN, rN
-    jz      @F
-    CRC1b
-    jmp     crc_end
-  @@:
-      if (IS_X64 gt 0) and (IS_LINUX gt 0)
-        MY_POP_2_REGS
-      else
-        MY_POP_4_REGS
-      endif
+align 16
+@@:
+unr_index = 0
+while unr_index lt UNROLL_CNT
+    if (unr_index and 1) eq 0
+        CRC_ITER x0, x1, unr_index
+    else
+        CRC_ITER x1, x0, unr_index
+    endif
+        unr_index = unr_index + 1
 endm
 
-MY_PROC CrcUpdateT8, 4
-    MY_PROLOG crc_end_8
-    mov     x1, [SRCDAT 1]
-    align 16
-  main_loop_8:
-    mov     x6, [SRCDAT 2]
-    movzx   x3, x1_L
-    CRC_XOR x6, r3, 3
-    movzx   x3, x1_H
-    CRC_XOR x6, r3, 2
-    shr     x1, 16
-    movzx   x3, x1_L
-    movzx   x1, x1_H
-    CRC_XOR x6, r3, 1
-    movzx   x3, x0_L
-    CRC_XOR x6, r1, 0
-
-    mov     x1, [SRCDAT 3]
-    CRC_XOR x6, r3, 7
-    movzx   x3, x0_H
-    shr     x0, 16
-    CRC_XOR x6, r3, 6
-    movzx   x3, x0_L
-    CRC_XOR x6, r3, 5
-    movzx   x3, x0_H
-    CRC_MOV x0, r3, 4
-    xor     x0, x6
-    add     rD, 8
-    jnz     main_loop_8
-
-    MY_EPILOG crc_end_8
-MY_ENDP
+        add     rD, NUM_WORDS * 4 * UNROLL_CNT
+        jnc     @B
+
+if 0
+        ; byte verson
+        add     rD, rN
+        xor     x0, dword ptr [rD]
+        add     rN, NUM_BYTES_LIMIT - 1
+else
+        ; 4-byte version
+        add     rN, 4 * NUM_WORDS * UNROLL_CNT
+        sub     rD, 4 * NUM_WORDS * UNROLL_CNT
+@@:
+        MOVZXLO x3, x0
+        MOVZXHI x1, x0
+        shr     x0, 16
+        MOVZXLO x6, x0
+        shr     x0, 8
+        CRC_MOV x0, x0, 0
+        CRC_XOR x0, x3, 3
+        CRC_XOR x0, x1, 2
+        CRC_XOR x0, x6, 1
+
+        add     rD, 4
+if (NUM_WORDS * UNROLL_CNT) ne 1
+        jc      @F
+        xor     x0, [SRCDAT_4 0]
+        jmp     @B
+@@:
+endif
+        add     rD, rN
+        add     rN, 4 - 1
+        
+endif
+        
+        sub     rN, rD
+crc_end:
+        test    rN, rN
+        jz      func_end
+@@:
+        CRC1b
+        jnz     @B
 
-MY_PROC CrcUpdateT4, 4
-    MY_PROLOG crc_end_4
-    align 16
-  main_loop_4:
-    movzx   x1, x0_L
-    movzx   x3, x0_H
-    shr     x0, 16
-    movzx   x6, x0_H
-    and     x0, 0FFh
-    CRC_MOV x1, r1, 3
-    xor     x1, [SRCDAT 1]
-    CRC_XOR x1, r3, 2
-    CRC_XOR x1, r6, 0
-    CRC_XOR x1, r0, 1
- 
-    movzx   x0, x1_L
-    movzx   x3, x1_H
-    shr     x1, 16
-    movzx   x6, x1_H
-    and     x1, 0FFh
-    CRC_MOV x0, r0, 3
-    xor     x0, [SRCDAT 2]
-    CRC_XOR x0, r3, 2
-    CRC_XOR x0, r6, 0
-    CRC_XOR x0, r1, 1
-    add     rD, 8
-    jnz     main_loop_4
-
-    MY_EPILOG crc_end_4
+func_end:
+        MY_POP_PRESERVED_ABI_REGS_UP_TO_INCLUDING_R11
 MY_ENDP
 
 end