libbb/yescrypt: make it possible to set constant parameters, and set YESCRYPT_RW

function                                             old     new   delta
yescrypt_kdf32_body                                 1052    1420    +368
yescrypt_r                                          1133    1084     -49
static.smix                                          762       -    -762
------------------------------------------------------------------------------
(add/remove: 0/1 grow/shrink: 1/1 up/down: 368/-811)         Total: -443 bytes

Signed-off-by: Denys Vlasenko <vda.linux@googlemail.com>

3 files changed, 141 insertions, 95 deletions

diff --git a/libbb/yescrypt/alg-yescrypt-common.c b/libbb/yescrypt/alg-yescrypt-common.c
index 5bdf1893e..db6e098c7 100644
--- a/libbb/yescrypt/alg-yescrypt-common.c
+++ b/libbb/yescrypt/alg-yescrypt-common.c
@@ -144,7 +144,7 @@ char *yescrypt_r(
         char *dst;
         const uint8_t *src, *saltstr, *saltend;
         size_t need, prefixlen, saltstrlen;
-        uint32_t flavor, N_log2;
+        uint32_t u32;
 
         memset(yctx, 0, sizeof(yctx));
         yctx->param.p = 1;
@@ -152,43 +152,34 @@ char *yescrypt_r(
         /* we assume setting starts with "$y$" (caller must ensure this) */
         src = setting + 3;
 
-        src = decode64_uint32(&flavor, src, 0);
+        src = decode64_uint32(&yctx->param.flags, src, 0);
         /* "j9T" returns: 0x2f */
-        dbg("yescrypt flavor=0x%x YESCRYPT_RW:%u", (unsigned)flavor, !!(flavor & YESCRYPT_RW));
         //if (!src)
         //      goto fail;
 
-        if (flavor < YESCRYPT_RW) {
-                yctx->param.flags = flavor;
-        } else if (flavor <= YESCRYPT_RW + (YESCRYPT_RW_FLAVOR_MASK >> 2)) {
+        if (yctx->param.flags < YESCRYPT_RW) {
+                dbg("yctx->param.flags=0x%x", (unsigned)yctx->param.flags);
+                goto fail; // bbox: we don't support scrypt - only yescrypt
+        } else if (yctx->param.flags <= YESCRYPT_RW + (YESCRYPT_RW_FLAVOR_MASK >> 2)) {
                 /* "j9T" sets flags to 0xb6 */
-                yctx->param.flags = YESCRYPT_RW + ((flavor - YESCRYPT_RW) << 2);
+                yctx->param.flags = YESCRYPT_RW + ((yctx->param.flags - YESCRYPT_RW) << 2);
                 dbg("yctx->param.flags=0x%x", (unsigned)yctx->param.flags);
-                dbg(" YESCRYPT_RW:%u"       , !!(yctx->param.flags & YESCRYPT_RW       ));
-                dbg(" YESCRYPT_ROUNDS_6:%u" , !!(yctx->param.flags & YESCRYPT_ROUNDS_6 ));
-                dbg(" YESCRYPT_GATHER_2:%u" , !!(yctx->param.flags & YESCRYPT_GATHER_2 ));
-                dbg(" YESCRYPT_GATHER_4:%u" , !!(yctx->param.flags & YESCRYPT_GATHER_4 ));
-                dbg(" YESCRYPT_GATHER_8:%u" , !!(yctx->param.flags & YESCRYPT_GATHER_8 ));
-                dbg(" YESCRYPT_SIMPLE_2:%u" , !!(yctx->param.flags & YESCRYPT_SIMPLE_2 ));
-                dbg(" YESCRYPT_SIMPLE_4:%u" , !!(yctx->param.flags & YESCRYPT_SIMPLE_4 ));
-                dbg(" YESCRYPT_SIMPLE_8:%u" , !!(yctx->param.flags & YESCRYPT_SIMPLE_8 ));
-                dbg(" YESCRYPT_SBOX_12K:%u" , !!(yctx->param.flags & YESCRYPT_SBOX_12K ));
-                dbg(" YESCRYPT_SBOX_24K:%u" , !!(yctx->param.flags & YESCRYPT_SBOX_24K ));
-                dbg(" YESCRYPT_SBOX_48K:%u" , !!(yctx->param.flags & YESCRYPT_SBOX_48K ));
-                dbg(" YESCRYPT_SBOX_96K:%u" , !!(yctx->param.flags & YESCRYPT_SBOX_96K ));
-                dbg(" YESCRYPT_SBOX_192K:%u", !!(yctx->param.flags & YESCRYPT_SBOX_192K));
-                dbg(" YESCRYPT_SBOX_384K:%u", !!(yctx->param.flags & YESCRYPT_SBOX_384K));
-                dbg(" YESCRYPT_SBOX_768K:%u", !!(yctx->param.flags & YESCRYPT_SBOX_768K));
+                dbg(" YESCRYPT_RW:%u", !!(yctx->param.flags & YESCRYPT_RW));
+                dbg((yctx->param.flags & YESCRYPT_RW_FLAVOR_MASK) ==
+                       (YESCRYPT_ROUNDS_6 | YESCRYPT_GATHER_4 | YESCRYPT_SIMPLE_2 | YESCRYPT_SBOX_12K)
+                    ? " YESCRYPT_ROUNDS_6 | YESCRYPT_GATHER_4 | YESCRYPT_SIMPLE_2 | YESCRYPT_SBOX_12K"
+                    : " flags are not standard"
+                );
         } else {
                 goto fail;
         }
 
-        src = decode64_uint32(&N_log2, src, 1);
-        if (/*!src ||*/ N_log2 > 63)
+        src = decode64_uint32(&u32, src, 1);
+        if (/*!src ||*/ u32 > 63)
                 goto fail;
-        yctx->param.N = (uint64_t)1 << N_log2;
+        yctx->param.N = (uint64_t)1 << u32;
         /* "j9T" sets to 4096 (1<<12) */
-        dbg("yctx->param.N=%llu (1<<%u)", (unsigned long long)yctx->param.N, (unsigned)N_log2);
+        dbg("yctx->param.N=%llu (1<<%u)", (unsigned long long)yctx->param.N, (unsigned)u32);
 
         src = decode64_uint32(&yctx->param.r, src, 1);
         /* "j9T" sets to 32 */
@@ -197,21 +188,19 @@ char *yescrypt_r(
         if (!src)
                 goto fail;
         if (*src != '$') {
-                uint32_t have;
-                src = decode64_uint32(&have, src, 1);
+                src = decode64_uint32(&u32, src, 1);
                 dbg("yescrypt has extended params:0x%x", (unsigned)have);
-                if (have & 1)
+                if (u32 & 1)
                         src = decode64_uint32(&yctx->param.p, src, 2);
-                if (have & 2)
+                if (u32 & 2)
                         src = decode64_uint32(&yctx->param.t, src, 1);
-                if (have & 4)
+                if (u32 & 4)
                         src = decode64_uint32(&yctx->param.g, src, 1);
-                if (have & 8) {
-                        uint32_t NROM_log2;
-                        src = decode64_uint32(&NROM_log2, src, 1);
-                        if (/*!src ||*/ NROM_log2 > 63)
+                if (u32 & 8) {
+                        src = decode64_uint32(&u32, src, 1);
+                        if (/*!src ||*/ u32 > 63)
                                 goto fail;
-                        yctx->param.NROM = (uint64_t)1 << NROM_log2;
+                        yctx->param.NROM = (uint64_t)1 << u32;
                 }
                 if (!src)
                         goto fail;
diff --git a/libbb/yescrypt/alg-yescrypt-kdf.c b/libbb/yescrypt/alg-yescrypt-kdf.c
index f1f06621e..13ae62b7c 100644
--- a/libbb/yescrypt/alg-yescrypt-kdf.c
+++ b/libbb/yescrypt/alg-yescrypt-kdf.c
@@ -460,7 +460,7 @@ static inline uint32_t integerify(const salsa20_blk_t *B, size_t r)
  * to a multiple of at least 16 bytes.
  */
 static void smix1(uint8_t *B, size_t r, uint32_t N,
-                yescrypt_flags_t flags,
+                uint32_t flags,
                 salsa20_blk_t *V,
                 uint32_t NROM, const salsa20_blk_t *VROM,
                 salsa20_blk_t *XY,
@@ -513,6 +513,7 @@ static void smix1(uint8_t *B, size_t r, uint32_t N,
                 V_j = &VROM[j * s];
                 blockmix_xor(Y, V_j, XY, r, ctx);
         } else if (flags & YESCRYPT_RW) {
+//can't use flags___YESCRYPT_RW, smix1() may be called with flags = 0
                 uint32_t n;
                 salsa20_blk_t *V_j;
 
@@ -580,7 +581,7 @@ static void smix1(uint8_t *B, size_t r, uint32_t N,
  * 64 bytes, and arrays B and XY to a multiple of at least 16 bytes.
  */
 static void smix2(uint8_t *B, size_t r, uint32_t N, uint64_t Nloop,
-                yescrypt_flags_t flags,
+                uint32_t flags,
                 salsa20_blk_t *V,
                 uint32_t NROM, const salsa20_blk_t *VROM,
                 salsa20_blk_t *XY,
@@ -610,6 +611,7 @@ static void smix2(uint8_t *B, size_t r, uint32_t N, uint64_t Nloop,
  * because our SMix resets YESCRYPT_RW for the smix2() calls operating on the
  * entire V when p > 1.
  */
+//and this is why bbox can't use flags___YESCRYPT_RW in this function
         if (VROM && (flags & YESCRYPT_RW)) {
                 do {
                         salsa20_blk_t *V_j = &V[j * s];
@@ -683,7 +685,7 @@ static uint64_t p2floor(uint64_t x)
  * might also result in cache bank conflicts).
  */
 static void smix(uint8_t *B, size_t r, uint32_t N, uint32_t p, uint32_t t,
-                yescrypt_flags_t flags,
+                uint32_t flags,
                 salsa20_blk_t *V,
                 uint32_t NROM, const salsa20_blk_t *VROM,
                 salsa20_blk_t *XY,
@@ -696,7 +698,7 @@ static void smix(uint8_t *B, size_t r, uint32_t N, uint32_t p, uint32_t t,
 
         Nchunk = N / p;
         Nloop_all = Nchunk;
-        if (flags & YESCRYPT_RW) {
+        if (flags___YESCRYPT_RW) {
                 if (t <= 1) {
                         if (t)
                                 Nloop_all *= 2; /* 2/3 */
@@ -711,7 +713,7 @@ static void smix(uint8_t *B, size_t r, uint32_t N, uint32_t p, uint32_t t,
         }
 
         Nloop_rw = 0;
-        if (flags & YESCRYPT_RW)
+        if (flags___YESCRYPT_RW)
                 Nloop_rw = Nloop_all / p;
 
         Nchunk &= ~(uint32_t)1; /* round down to even */
@@ -725,7 +727,7 @@ static void smix(uint8_t *B, size_t r, uint32_t N, uint32_t p, uint32_t t,
                 salsa20_blk_t *Vp = &V[Vchunk * s];
                 salsa20_blk_t *XYp = XY;
                 pwxform_ctx_t *ctx_i = NULL;
-                if (flags & YESCRYPT_RW) {
+                if (flags___YESCRYPT_RW) {
                         uint8_t *Si = S + i * Salloc;
                         smix1(Bp, 1, Sbytes / 128, 0 /* no flags */,
                             (salsa20_blk_t *)Si, 0, NULL, XYp, NULL);
@@ -752,12 +754,12 @@ static void smix(uint8_t *B, size_t r, uint32_t N, uint32_t p, uint32_t t,
                         uint8_t *Bp = &B[128 * r * i];
                         salsa20_blk_t *XYp = XY;
                         pwxform_ctx_t *ctx_i = NULL;
-                        if (flags & YESCRYPT_RW) {
+                        if (flags___YESCRYPT_RW) {
                                 uint8_t *Si = S + i * Salloc;
                                 ctx_i = (pwxform_ctx_t *)(Si + Sbytes);
                         }
                         smix2(Bp, r, N, Nloop_all - Nloop_rw,
-                              flags & (yescrypt_flags_t)~YESCRYPT_RW,
+                              flags & (uint32_t)~YESCRYPT_RW,
                               V, NROM, VROM, XYp, ctx_i);
                 }
         }
@@ -812,7 +814,7 @@ static void free_region(yescrypt_region_t *region)
 static int yescrypt_kdf32_body(
                 yescrypt_ctx_t *yctx,
                 const uint8_t *passwd, size_t passwdlen,
-                yescrypt_flags_t flags, uint64_t N, uint32_t t,
+                uint32_t flags, uint64_t N, uint32_t t,
                 uint8_t *buf32)
 {
         const salsa20_blk_t *VROM;
@@ -823,13 +825,13 @@ static int yescrypt_kdf32_body(
         uint8_t dk[sizeof(sha256)], *dkp = buf32;
 
         /* Sanity-check parameters */
-        switch (flags & YESCRYPT_MODE_MASK) {
+        switch (flags___YESCRYPT_MODE_MASK) {
         case 0: /* classic scrypt - can't have anything non-standard */
-                if (flags || t || yctx->param.NROM)
+                if (flags || t || YCTX_param_NROM)
                         goto out_EINVAL;
                 break;
         case YESCRYPT_WORM:
-                if (flags != YESCRYPT_WORM || yctx->param.NROM)
+                if (flags != YESCRYPT_WORM || YCTX_param_NROM)
                         goto out_EINVAL;
                 break;
         case YESCRYPT_RW:
@@ -852,8 +854,8 @@ static int yescrypt_kdf32_body(
                 goto out_EINVAL;
 #endif
     {
-        const uint32_t r = yctx->param.r;
-        const uint32_t p = yctx->param.p;
+        const uint32_t r = YCTX_param_r;
+        const uint32_t p = YCTX_param_p;
         if ((uint64_t)r * (uint64_t)p >= 1 << 30)
                 goto out_EINVAL;
         if (N > UINT32_MAX)
@@ -863,7 +865,7 @@ static int yescrypt_kdf32_body(
         if (r > SIZE_MAX / 256 / p ||
             N > SIZE_MAX / 128 / r)
                 goto out_EINVAL;
-        if (flags & YESCRYPT_RW) {
+        if (flags___YESCRYPT_RW) {
                 /* p cannot be greater than SIZE_MAX/Salloc on 64-bit systems,
                    but it can on 32-bit systems.  */
 #pragma GCC diagnostic push
@@ -874,7 +876,7 @@ static int yescrypt_kdf32_body(
         }
 
         VROM = NULL;
-        if (yctx->param.NROM)
+        if (YCTX_param_NROM)
                 goto out_EINVAL;
 
         /* Allocate memory */
@@ -889,7 +891,7 @@ static int yescrypt_kdf32_body(
         need += XY_size;
         if (need < XY_size)
                 goto out_EINVAL;
-        if (flags & YESCRYPT_RW) {
+        if (flags___YESCRYPT_RW) {
                 size_t S_size = (size_t)Salloc * p;
                 need += S_size;
                 if (need < S_size)
@@ -907,7 +909,7 @@ static int yescrypt_kdf32_body(
         V = (salsa20_blk_t *)((uint8_t *)B + B_size);
         XY = (salsa20_blk_t *)((uint8_t *)V + V_size);
         S = NULL;
-        if (flags & YESCRYPT_RW)
+        if (flags___YESCRYPT_RW)
                 S = (uint8_t *)XY + XY_size;
 
         if (flags) {
@@ -926,13 +928,13 @@ static int yescrypt_kdf32_body(
         if (flags)
                 memcpy(sha256, B, sizeof(sha256));
 
-        if (p == 1 || (flags & YESCRYPT_RW)) {
-                smix(B, r, N, p, t, flags, V, yctx->param.NROM, VROM, XY, S, sha256);
+        if (p == 1 || (flags___YESCRYPT_RW)) {
+                smix(B, r, N, p, t, flags, V, YCTX_param_NROM, VROM, XY, S, sha256);
         } else {
                 uint32_t i;
                 for (i = 0; i < p; i++) {
                         smix(&B[(size_t)128 * r * i], r, N, 1, t, flags, V,
-                            yctx->param.NROM, VROM, XY, NULL, NULL);
+                            YCTX_param_NROM, VROM, XY, NULL, NULL);
                 }
         }
 
@@ -996,12 +998,12 @@ int yescrypt_kdf32(
                 const uint8_t *passwd, size_t passwdlen,
                 uint8_t *buf32)
 {
-        yescrypt_flags_t flags = yctx->param.flags;
-        uint64_t N = yctx->param.N;
-        uint32_t r = yctx->param.r;
-        uint32_t p = yctx->param.p;
-        uint32_t t = yctx->param.t;
-        uint32_t g = yctx->param.g;
+        uint32_t flags = YCTX_param_flags;
+        uint64_t N = YCTX_param_N;
+        uint32_t r = YCTX_param_r;
+        uint32_t p = YCTX_param_p;
+        uint32_t t = YCTX_param_t;
+        uint32_t g = YCTX_param_g;
         uint8_t dk32[32];
         int retval;
 
@@ -1011,7 +1013,7 @@ int yescrypt_kdf32(
                 return -1;
         }
 
-        if ((flags & YESCRYPT_RW)
+        if ((flags___YESCRYPT_RW)
          && p >= 1
          && N / p >= 0x100
          && N / p * r >= 0x20000
diff --git a/libbb/yescrypt/alg-yescrypt.h b/libbb/yescrypt/alg-yescrypt.h
index 996af333f..97475d89f 100644
--- a/libbb/yescrypt/alg-yescrypt.h
+++ b/libbb/yescrypt/alg-yescrypt.h
@@ -42,28 +42,32 @@
  * Please refer to the description of yescrypt_kdf() below for the meaning of
  * these flags.
  */
-typedef uint32_t yescrypt_flags_t;
+/* yescrypt flags:
+ * bits pos: 7654321076543210
+ *                    ss  r w
+ *                sbox  gg y
+ */
 /* Public */
 #define YESCRYPT_WORM                   1
 #define YESCRYPT_RW                     0x002
-#define YESCRYPT_ROUNDS_3               0x000
-#define YESCRYPT_ROUNDS_6               0x004
-#define YESCRYPT_GATHER_1               0x000
-#define YESCRYPT_GATHER_2               0x008
-#define YESCRYPT_GATHER_4               0x010
-#define YESCRYPT_GATHER_8               0x018
-#define YESCRYPT_SIMPLE_1               0x000
-#define YESCRYPT_SIMPLE_2               0x020
-#define YESCRYPT_SIMPLE_4               0x040
-#define YESCRYPT_SIMPLE_8               0x060
-#define YESCRYPT_SBOX_6K                0x000
-#define YESCRYPT_SBOX_12K               0x080
-#define YESCRYPT_SBOX_24K               0x100
-#define YESCRYPT_SBOX_48K               0x180
-#define YESCRYPT_SBOX_96K               0x200
-#define YESCRYPT_SBOX_192K              0x280
-#define YESCRYPT_SBOX_384K              0x300
-#define YESCRYPT_SBOX_768K              0x380
+#define YESCRYPT_ROUNDS_3               0x000 //r=0
+#define YESCRYPT_ROUNDS_6               0x004 //r=1
+#define YESCRYPT_GATHER_1               0x000 //gg=00
+#define YESCRYPT_GATHER_2               0x008 //gg=01
+#define YESCRYPT_GATHER_4               0x010 //gg=10
+#define YESCRYPT_GATHER_8               0x018 //gg=11
+#define YESCRYPT_SIMPLE_1               0x000 //ss=00
+#define YESCRYPT_SIMPLE_2               0x020 //ss=01
+#define YESCRYPT_SIMPLE_4               0x040 //ss=11
+#define YESCRYPT_SIMPLE_8               0x060 //ss=11
+#define YESCRYPT_SBOX_6K                0x000 //sbox=0000
+#define YESCRYPT_SBOX_12K               0x080 //sbox=0001
+#define YESCRYPT_SBOX_24K               0x100 //sbox=0010
+#define YESCRYPT_SBOX_48K               0x180 //sbox=0011
+#define YESCRYPT_SBOX_96K               0x200 //sbox=0100
+#define YESCRYPT_SBOX_192K              0x280 //sbox=0101
+#define YESCRYPT_SBOX_384K              0x300 //sbox=0110
+#define YESCRYPT_SBOX_768K              0x380 //sbox=0111
 
 #ifdef YESCRYPT_INTERNAL
 /* Private */
@@ -86,6 +90,19 @@ typedef uint32_t yescrypt_flags_t;
         YESCRYPT_ALLOC_ONLY | YESCRYPT_PREHASH)
 #endif
 
+/* How many chars base-64 encoded bytes require? */
+#define YESCRYPT_BYTES2CHARS(bytes) ((((bytes) * 8) + 5) / 6)
+/* The /etc/passwd-style hash is "<prefix>$<hash><NUL>" */
+/*
+ * "$y$", up to 8 params of up to 6 chars each, '$', salt
+ * Alternatively, but that's smaller:
+ * "$7$", 3 params encoded as 1+5+5 chars, salt
+ */
+#define YESCRYPT_PREFIX_LEN (3 + 8 * 6 + 1 + YESCRYPT_BYTES2CHARS(32))
+
+#define YESCRYPT_HASH_SIZE 32
+#define YESCRYPT_HASH_LEN  YESCRYPT_BYTES2CHARS(YESCRYPT_HASH_SIZE)
+
 /**
  * Internal type used by the memory allocator.  Please do not use it directly.
  * Use yescrypt_shared_t and yescrypt_local_t as appropriate instead, since
@@ -104,7 +121,7 @@ typedef struct {
  * set.  flags, t, g, NROM are special to yescrypt.
  */
 typedef struct {
-        yescrypt_flags_t flags;
+        uint32_t flags;
         uint64_t N;
         uint32_t r, p, t, g;
         uint64_t NROM;
@@ -123,18 +140,56 @@ typedef struct {
         yescrypt_region_t local[1];
 } yescrypt_ctx_t;
 
-/* How many chars base-64 encoded bytes require? */
-#define YESCRYPT_BYTES2CHARS(bytes) ((((bytes) * 8) + 5) / 6)
-/* The /etc/passwd-style hash is "<prefix>$<hash><NUL>" */
-/*
- * "$y$", up to 8 params of up to 6 chars each, '$', salt
- * Alternatively, but that's smaller:
- * "$7$", 3 params encoded as 1+5+5 chars, salt
- */
-#define YESCRYPT_PREFIX_LEN (3 + 8 * 6 + 1 + YESCRYPT_BYTES2CHARS(32))
+// How much can save by forcing "standard" value by commenting the next line:
+//  160 bytes
+//#define YCTX_param_flags yctx->param.flags
+//  260 bytes
+//#define flags___YESCRYPT_RW (flags & YESCRYPT_RW)
+//  140 bytes
+//#define flags___YESCRYPT_MODE_MASK (flags & YESCRYPT_MODE_MASK)
+// ^^^^ forcing the above since the code already requires (checks for) this
+//   50 bytes
+#define YCTX_param_N     yctx->param.N
+// -100 bytes (negative!!!)
+#define YCTX_param_r     yctx->param.r
+//  400 bytes
+#define YCTX_param_p     yctx->param.p
+//  130 bytes
+#define YCTX_param_t     yctx->param.t
+//    2 bytes
+#define YCTX_param_g     yctx->param.g
+//    1 bytes
+// ^^^^ this looks wrong, compiler should be able to constant-propagate the fact that NROM code is dead
+#define YCTX_param_NROM  yctx->param.NROM
 
-#define YESCRYPT_HASH_SIZE 32
-#define YESCRYPT_HASH_LEN  YESCRYPT_BYTES2CHARS(YESCRYPT_HASH_SIZE)
+// standard ("j9T") values:
+#ifndef YCTX_param_flags
+#define YCTX_param_flags (YESCRYPT_RW | YESCRYPT_ROUNDS_6 | YESCRYPT_GATHER_4 | YESCRYPT_SIMPLE_2 | YESCRYPT_SBOX_12K)
+#endif
+#ifndef flags___YESCRYPT_RW
+#define flags___YESCRYPT_RW ((void)flags, YESCRYPT_RW)
+#endif
+#ifndef flags___YESCRYPT_MODE_MASK
+#define flags___YESCRYPT_MODE_MASK ((void)flags, YESCRYPT_RW)
+#endif
+#ifndef YCTX_param_N
+#define YCTX_param_N     4096
+#endif
+#ifndef YCTX_param_r
+#define YCTX_param_r     32
+#endif
+#ifndef YCTX_param_p
+#define YCTX_param_p     1
+#endif
+#ifndef YCTX_param_t
+#define YCTX_param_t     0
+#endif
+#ifndef YCTX_param_g
+#define YCTX_param_g     0
+#endif
+#ifndef YCTX_param_NROM
+#define YCTX_param_NROM  0
+#endif
 
 /**
  * yescrypt_r(shared, local, passwd, passwdlen, setting, key, buf, buflen):