diff --git a/.wolfssl_known_macro_extras b/.wolfssl_known_macro_extras index 2b022344199..e8212f3d720 100644 --- a/.wolfssl_known_macro_extras +++ b/.wolfssl_known_macro_extras @@ -833,6 +833,8 @@ WC_FIPS_ED25519CTX_NOT_APPROVED WC_FIPS_RSA_VERIFY_MIN_2048 WC_FLAG_DONT_USE_AESNI WC_FORCE_LINUXKM_FORTIFY_SOURCE +WC_FORCE_ZERO_LOOP +WC_FORCE_ZERO_SHOW_LEN WC_HASH_CUSTOM_MAX_BLOCK_SIZE WC_HASH_CUSTOM_MAX_DIGEST_SIZE WC_HASH_CUSTOM_MIN_DIGEST_SIZE @@ -848,6 +850,7 @@ WC_MLKEM_KERNEL_ASM WC_MPFS_ATHENA_NO_BASE_ADDR WC_MPFS_ATHENA_NO_HW_INIT WC_MPFS_ATHENA_NO_HW_MUTEX +WC_NO_ASSUME WC_NO_ASYNC_SLEEP WC_NO_RNG_SIMPLE WC_NO_STATIC_ASSERT @@ -1421,6 +1424,7 @@ __MINGW64__ __MSYS__ __MWERKS__ __NT__ +__OPTIMIZE__ __OS2__ __OpenBSD__ __PIC__ diff --git a/tests/api.c b/tests/api.c index a2507e33259..797bc5ff89e 100644 --- a/tests/api.c +++ b/tests/api.c @@ -32615,6 +32615,7 @@ static int test_ForceZero(void) { EXPECT_DECLS; unsigned char data[32]; + unsigned char big[32 + 300 + 16]; unsigned int i, j, len; /* Test case with 0 length */ @@ -32639,6 +32640,26 @@ static int test_ForceZero(void) } } + /* Longer wipes take other paths. Test each len from 32 to 300 bytes at + * 16 start offsets, with guard bytes on both sides. */ + for (i = 16; i < 32; i++) { + for (len = 32; len <= 300; len++) { + for (j = 0; j < sizeof(big); j++) + big[j] = (unsigned char)(j % 255 + 1); + + ForceZero(big + i, len); + + for (j = 0; j < sizeof(big); j++) { + if (j < i || j >= i + len) { + ExpectIntNE(big[j], 0x00); + } + else { + ExpectIntEQ(big[j], 0x00); + } + } + } + } + return EXPECT_RESULT(); } diff --git a/wolfcrypt/src/aes.c b/wolfcrypt/src/aes.c index fca02cf3377..e1defaaa419 100644 --- a/wolfcrypt/src/aes.c +++ b/wolfcrypt/src/aes.c @@ -16282,13 +16282,18 @@ int wc_AesCcmEncrypt(Aes* aes, byte* out, const byte* in, word32 inSz, byte* authTag, word32 authTagSz, const byte* authIn, word32 authInSz) { + /* A and B in one struct, so one wipe clears both. */ + struct { #ifdef WOLFSSL_AESNI - ALIGN128 byte A[WC_AES_BLOCK_SIZE * 4]; - ALIGN128 byte B[WC_AES_BLOCK_SIZE * 4]; + ALIGN128 byte a[WC_AES_BLOCK_SIZE * 4]; + ALIGN128 byte b[WC_AES_BLOCK_SIZE * 4]; #else - byte A[WC_AES_BLOCK_SIZE]; - byte B[WC_AES_BLOCK_SIZE]; + byte a[WC_AES_BLOCK_SIZE]; + byte b[WC_AES_BLOCK_SIZE]; #endif + } ab; + byte* A = ab.a; + byte* B = ab.b; byte lenSz; word32 i; byte mask = 0xFF; @@ -16346,7 +16351,7 @@ int wc_AesCcmEncrypt(Aes* aes, byte* out, const byte* in, word32 inSz, return MISSING_KEY; } - XMEMSET(A, 0, sizeof(A)); + XMEMSET(A, 0, sizeof(ab.a)); XMEMCPY(B+1, nonce, nonceSz); B[0] = (byte)((authInSz > 0 ? 64 : 0) @@ -16359,7 +16364,7 @@ int wc_AesCcmEncrypt(Aes* aes, byte* out, const byte* in, word32 inSz, } #ifdef WOLFSSL_CHECK_MEM_ZERO - wc_MemZero_Add("wc_AesCcmEncrypt B", B, sizeof(B)); + wc_MemZero_Add("wc_AesCcmEncrypt B", B, sizeof(ab.b)); #endif VECTOR_REGISTERS_PUSH; @@ -16369,7 +16374,7 @@ int wc_AesCcmEncrypt(Aes* aes, byte* out, const byte* in, word32 inSz, ret = wc_AesEncrypt(aes, B, A); #ifdef WOLFSSL_CHECK_MEM_ZERO if (ret == 0) - wc_MemZero_Add("wc_AesCcmEncrypt A", A, sizeof(A)); + wc_MemZero_Add("wc_AesCcmEncrypt A", A, sizeof(ab.a)); #endif if ((ret == 0) && (authInSz > 0)) @@ -16432,12 +16437,11 @@ int wc_AesCcmEncrypt(Aes* aes, byte* out, const byte* in, word32 inSz, XMEMCPY(out, A, inSz); } - ForceZero(A, sizeof(A)); - ForceZero(B, sizeof(B)); + ForceZero(&ab, sizeof(ab)); #ifdef WOLFSSL_CHECK_MEM_ZERO - wc_MemZero_Check(A, sizeof(A)); - wc_MemZero_Check(B, sizeof(B)); + wc_MemZero_Check(A, sizeof(ab.a)); + wc_MemZero_Check(B, sizeof(ab.b)); #endif VECTOR_REGISTERS_POP; @@ -16452,13 +16456,18 @@ int wc_AesCcmDecrypt(Aes* aes, byte* out, const byte* in, word32 inSz, const byte* authTag, word32 authTagSz, const byte* authIn, word32 authInSz) { + /* A and B in one struct, so one wipe clears both. */ + struct { #ifdef WOLFSSL_AESNI - ALIGN128 byte B[WC_AES_BLOCK_SIZE * 4]; - ALIGN128 byte A[WC_AES_BLOCK_SIZE * 4]; + ALIGN128 byte a[WC_AES_BLOCK_SIZE * 4]; + ALIGN128 byte b[WC_AES_BLOCK_SIZE * 4]; #else - byte A[WC_AES_BLOCK_SIZE]; - byte B[WC_AES_BLOCK_SIZE]; + byte a[WC_AES_BLOCK_SIZE]; + byte b[WC_AES_BLOCK_SIZE]; #endif + } ab; + byte* A = ab.a; + byte* B = ab.b; byte* o; byte lenSz; word32 i, oSz; @@ -16522,7 +16531,7 @@ int wc_AesCcmDecrypt(Aes* aes, byte* out, const byte* in, word32 inSz, o = out; oSz = inSz; - XMEMSET(A, 0, sizeof A); + XMEMSET(A, 0, sizeof ab.a); XMEMCPY(B+1, nonce, nonceSz); B[0] = (byte)(lenSz - 1U); @@ -16531,8 +16540,8 @@ int wc_AesCcmDecrypt(Aes* aes, byte* out, const byte* in, word32 inSz, B[15] = 1; #ifdef WOLFSSL_CHECK_MEM_ZERO - wc_MemZero_Add("wc_AesCcmEncrypt A", A, sizeof(A)); - wc_MemZero_Add("wc_AesCcmEncrypt B", B, sizeof(B)); + wc_MemZero_Add("wc_AesCcmEncrypt A", A, sizeof(ab.a)); + wc_MemZero_Add("wc_AesCcmEncrypt B", B, sizeof(ab.b)); #endif VECTOR_REGISTERS_PUSH; @@ -16627,13 +16636,12 @@ int wc_AesCcmDecrypt(Aes* aes, byte* out, const byte* in, word32 inSz, } } - ForceZero(A, sizeof(A)); - ForceZero(B, sizeof(B)); + ForceZero(&ab, sizeof(ab)); o = NULL; #ifdef WOLFSSL_CHECK_MEM_ZERO - wc_MemZero_Check(A, sizeof(A)); - wc_MemZero_Check(B, sizeof(B)); + wc_MemZero_Check(A, sizeof(ab.a)); + wc_MemZero_Check(B, sizeof(ab.b)); #endif VECTOR_REGISTERS_POP; diff --git a/wolfcrypt/src/asm.c b/wolfcrypt/src/asm.c index 8ee70539e20..a4cf0597b08 100644 --- a/wolfcrypt/src/asm.c +++ b/wolfcrypt/src/asm.c @@ -51,6 +51,20 @@ #define ECX 2 #define EDX 3 +/* misc.c may include cpuid.h, which uses these names with other values. */ +#undef CPUID_AVX1 +#undef CPUID_AVX2 +#undef CPUID_RDRAND +#undef CPUID_RDSEED +#undef CPUID_BMI2 +#undef CPUID_ADX +#undef IS_INTEL_AVX1 +#undef IS_INTEL_AVX2 +#undef IS_INTEL_BMI2 +#undef IS_INTEL_ADX +#undef IS_INTEL_RDRAND +#undef IS_INTEL_RDSEED + #define CPUID_AVX1 0x1 #define CPUID_AVX2 0x2 #define CPUID_RDRAND 0x4 diff --git a/wolfcrypt/src/asn.c b/wolfcrypt/src/asn.c index b1fbb1c902b..a0afc962c70 100644 --- a/wolfcrypt/src/asn.c +++ b/wolfcrypt/src/asn.c @@ -10046,7 +10046,8 @@ int wc_CheckPrivateKey(const byte* privKey, word32 privKeySz, WOLFSSL_ERROR_VERBOSE(ret); } } - ForceZero(privDer, privSz); + /* All of privDer, so the compiler knows the length fits. */ + ForceZero(privDer, MAX_ECC_BYTES); } } else { diff --git a/wolfcrypt/src/kdf.c b/wolfcrypt/src/kdf.c index 3f7aa7bcd28..b9356f5a5c2 100644 --- a/wolfcrypt/src/kdf.c +++ b/wolfcrypt/src/kdf.c @@ -1458,7 +1458,8 @@ int wc_KDA_KDF_onestep(const byte* z, word32 zSz, const byte* fixedInfo, if (ret == 0) { XMEMCPY(output + outIdx, hashTempBuf, derivedSecretSz - outIdx); } - ForceZero(hashTempBuf, (word32) hashOutSz); + /* All of hashTempBuf, so the compiler knows the length fits. */ + ForceZero(hashTempBuf, sizeof(hashTempBuf)); #ifdef WOLFSSL_CHECK_MEM_ZERO wc_MemZero_Check(hashTempBuf, (word32) hashOutSz); #endif diff --git a/wolfcrypt/src/misc.c b/wolfcrypt/src/misc.c index f4b81fb8373..250aee19ca5 100644 --- a/wolfcrypt/src/misc.c +++ b/wolfcrypt/src/misc.c @@ -784,38 +784,187 @@ WC_MISC_STATIC WC_INLINE void xorbuf(void* buf, const void* mask, word32 count) #endif /* !WOLFSSL_NO_XOR_OPS */ #ifndef WOLFSSL_NO_FORCE_ZERO +/* GCC and Clang on x86 zero up to WC_FORCE_ZERO_INLINE_MAX bytes with + * volatile stores, which the compiler must emit. They store from general + * registers, as SSE stores are very slow after AVX code that leaves the upper + * halves of the vector registers dirty. With USE_INTEL_SPEEDUP, x86_64 CPUs + * with AVX2 zero from WC_FORCE_ZERO_AVX2_MIN bytes with 32-byte AVX2 stores, + * which have no such penalty. Wipes over WC_FORCE_ZERO_INLINE_MAX bytes use + * XMEMSET, as libc's memset is as fast there and is tuned for each CPU. + * Other builds use XMEMSET for all lengths. WC_FORCE_ZERO_LOOP makes x86 use + * XMEMSET too. */ +#if !defined(WC_FORCE_ZERO_FAST) && !defined(WC_FORCE_ZERO_LOOP) && \ + defined(__GNUC__) && !defined(__CC_ARM) && !defined(WOLFSSL_NO_ASM) && \ + !defined(WOLFSSL_KERNEL_MODE) && \ + (defined(__x86_64__) || defined(__i386__)) + #define WC_FORCE_ZERO_FAST +#endif +#ifndef WC_FORCE_ZERO_INLINE_MAX + /* x86 zeroes up to this many bytes inline, and more with XMEMSET. Define + * to override. */ + #define WC_FORCE_ZERO_INLINE_MAX 256 +#endif + +/* The AVX2 path asks cpuid.c for the CPU features. Only library code built + * with USE_INTEL_SPEEDUP uses it, as cpuid.c is always linked there. Kernel + * code may not touch vector registers without saving them first. */ +#if defined(WC_FORCE_ZERO_FAST) && defined(__x86_64__) && \ + defined(USE_INTEL_SPEEDUP) && defined(BUILDING_WOLFSSL) && \ + !defined(WOLFSSL_NO_ASM) && !defined(WOLFSSL_KERNEL_MODE) && \ + (defined(__clang__) || __GNUC_PREREQ(4, 9)) + #include + #ifdef HAVE_CPUID_INTEL + #define WC_FORCE_ZERO_AVX2 + #endif +#endif + +#ifdef WC_FORCE_ZERO_AVX2 +#ifndef WC_FORCE_ZERO_AVX2_MIN + /* AVX2 zeroes from this many bytes. Define to override. */ + #define WC_FORCE_ZERO_AVX2_MIN 128 +#endif +#if WC_FORCE_ZERO_AVX2_MIN < 32 + #error WC_FORCE_ZERO_AVX2_MIN must be at least 32 +#endif + +#ifdef __clang__ + #define WC_FZ_AVX2_ATTR __attribute__((target("avx2"))) +#else + /* No clones: GCC would check a clone's stores against a constant buffer + * even where the length rules the call out. */ + #define WC_FZ_AVX2_ATTR __attribute__((target("avx2"), noclone)) +#endif + +/* Zeroes len bytes at zb with 32-byte AVX2 stores. len must be at least 32. */ +static WC_INLINE WC_FZ_AVX2_ATTR void ForceZero_AVX2(byte* zb, size_t len) +{ + typedef word64 MAYBE_UNALIGNED fz_v32 __attribute__((vector_size(32), + may_alias)); + const fz_v32 zv = { 0 }; + byte *ze = zb + len; + + /* The two ends cover up to 64 bytes. */ + *(volatile fz_v32 *)zb = zv; + *(volatile fz_v32 *)(ze - 32) = zv; + if (len > 64) { + /* 64 bytes per pass from the next 32-byte boundary, then 32 if + * left. */ + zb = (byte *)(((wc_ptr_t)zb + 32) & ~(wc_ptr_t)31); + while (zb < ze - 64) { + *(volatile fz_v32 *)zb = zv; + *(volatile fz_v32 *)(zb + 32) = zv; + zb += 64; + } + if (zb < ze - 32) { + *(volatile fz_v32 *)zb = zv; + } + } +} +#undef WC_FZ_AVX2_ATTR + +/* Returns whether the CPU and OS support AVX2. Each file caches the flags. + * Once they are cached, this is one atomic load. */ +static WC_INLINE int ForceZero_HaveAVX2(void) +{ + static cpuid_flags_atomic_t fz_cpuid_flags = WC_CPUID_ATOMIC_INITIALIZER; + cpuid_flags_t flags = WOLFSSL_ATOMIC_LOAD(fz_cpuid_flags); + + if (flags == WC_CPUID_INITIALIZER) { + /* Threads that race here store the same flags. */ + flags = cpuid_get_flags(); + WOLFSSL_ATOMIC_STORE(fz_cpuid_flags, flags); + } + return IS_INTEL_AVX2(flags) != 0; +} +#endif /* WC_FORCE_ZERO_AVX2 */ + /* This routine fills the first len bytes of the memory area pointed by mem with zeros. It ensures compiler optimization doesn't skip it. */ WC_MISC_STATIC WC_INLINE void ForceZero(void* mem, size_t len) { +#ifdef WC_FORCE_ZERO_FAST + typedef word64 MAYBE_UNALIGNED fz_w64 __attribute__((may_alias)); + typedef word32 MAYBE_UNALIGNED fz_w32 __attribute__((may_alias)); + typedef word16 MAYBE_UNALIGNED fz_w16 __attribute__((may_alias)); + /* WC_FZ_ZERO16(p) zeroes 16 bytes at p. */ + #define WC_FZ_ZERO16(p) do { \ + *(volatile fz_w64 *)(p) = 0; \ + *(volatile fz_w64 *)((p) + 8) = 0; \ + } while (0) byte *zb = (byte *)mem; - unsigned long *zl; + byte *ze; +#endif /* Make the compiler put the buffer's current contents at mem, so the * wipe below hits the memory that holds them and not a copy. */ WC_BARRIER_DATA(mem); - /* No early return here: a short unaligned buffer must still reach the - * trailing barrier, or its wipe can be dropped as a dead store. */ - while ((len != 0) && - ((wc_ptr_t)zb & (wc_ptr_t)(sizeof(unsigned long) - 1U))) { - *zb++ = 0; - --len; +#ifdef WC_FORCE_ZERO_FAST + /* Each branch tests len as passed in, so a known len picks one when + * ForceZero is inlined. Stores at the start and at the end may overlap. */ + if (len > WC_FORCE_ZERO_INLINE_MAX) { + #ifndef WC_FORCE_ZERO_SHOW_LEN + /* For a known len, GCC's default x86 tuning makes memset an inline + * rep stosq, much slower than libc's memset. Hiding len makes it a + * call. Define WC_FORCE_ZERO_SHOW_LEN to show len. */ + size_t n = len; + + __asm__ __volatile__("" : "+r"(n)); + XMEMSET(mem, 0, n); + #else + XMEMSET(mem, 0, len); + #endif } - - zl = (unsigned long *)zb; - - while (len >= sizeof(unsigned long)) { - *zl++ = 0; - len -= sizeof(unsigned long); + else if (len > 32) { + ze = zb + len; + #ifdef WC_FORCE_ZERO_AVX2 + if (len >= WC_FORCE_ZERO_AVX2_MIN && ForceZero_HaveAVX2()) { + ForceZero_AVX2(zb, len); + } + else + #endif + { + WC_FZ_ZERO16(zb); + WC_FZ_ZERO16(ze - 32); + WC_FZ_ZERO16(ze - 16); + /* Above 48 bytes, zero the middle from the next 16-byte boundary: + * 32 bytes per pass, then 16 if left. */ + if (len > 48) { + zb = (byte *)(((wc_ptr_t)zb + 16) & ~(wc_ptr_t)15); + while (zb + 16 < ze - 32) { + WC_FZ_ZERO16(zb); + WC_FZ_ZERO16(zb + 16); + zb += 32; + } + if (zb < ze - 32) { + WC_FZ_ZERO16(zb); + } + } + } } - - zb = (byte *)zl; - - while (len) { - *zb++ = 0; - --len; + else if (len > 16) { + WC_FZ_ZERO16(zb); + WC_FZ_ZERO16(zb + len - 16); + } + else if (len >= 8) { + *(volatile fz_w64 *)zb = 0; + *(volatile fz_w64 *)(zb + len - 8) = 0; + } + else if (len >= 4) { + *(volatile fz_w32 *)zb = 0; + *(volatile fz_w32 *)(zb + len - 4) = 0; + } + else if (len >= 2) { + *(volatile fz_w16 *)zb = 0; + *(volatile fz_w16 *)(zb + len - 2) = 0; } + else if (len == 1) { + *(volatile byte *)zb = 0; + } + #undef WC_FZ_ZERO16 +#else + XMEMSET(mem, 0, len); +#endif /* The caller is done with the buffer, so the compiler may drop the * stores above as dead. The barrier makes the buffer look read by @@ -1003,6 +1152,9 @@ WC_MISC_STATIC WC_INLINE void* ctMaskSelPtr(byte mask, void* a, void* b) volatile word32 gte_mask = (word32)ctMaskWord32GTE(a, b); word32 r = (a & ~gte_mask); r |= (b & gte_mask); + /* The volatile mask hides that r is a or b. Give the compiler the + * bound. */ + WC_ASSUME(r <= a && r <= b); return r; #else /* WOLFSSL_NO_CT_OPS */ return a > b ? b : a; @@ -1020,7 +1172,9 @@ WC_MISC_STATIC WC_INLINE void* ctMaskSelPtr(byte mask, void* a, void* b) #if !defined(WOLFSSL_NO_CT_OPS) && !defined(WOLFSSL_NO_CT_MAX_MIN) && \ defined(WORD64_AVAILABLE) volatile word32 gte_mask = (word32)ctMaskWord32GTE(a, b); - return (a & gte_mask) | (b & ~gte_mask); + word32 r = (a & gte_mask) | (b & ~gte_mask); + WC_ASSUME(r >= a && r >= b); + return r; #else /* WOLFSSL_NO_CT_OPS */ return a > b ? a : b; #endif /* WOLFSSL_NO_CT_OPS */ diff --git a/wolfcrypt/src/pwdbased.c b/wolfcrypt/src/pwdbased.c index 770b2e4c1b7..85cb00feb85 100644 --- a/wolfcrypt/src/pwdbased.c +++ b/wolfcrypt/src/pwdbased.c @@ -363,7 +363,8 @@ int wc_PBKDF2_ex(byte* output, const byte* passwd, int pLen, const byte* salt, wc_HmacFree(hmac); } - ForceZero(buffer, (word32)hLen); + /* All of buffer, so the compiler knows the length fits. */ + ForceZero(buffer, WC_MAX_DIGEST_SIZE); #if !defined(WOLFSSL_SMALL_STACK) && defined(WOLFSSL_CHECK_MEM_ZERO) wc_MemZero_Check(buffer, (word32)hLen); #endif diff --git a/wolfcrypt/src/random.c b/wolfcrypt/src/random.c index aa805e711be..bd9c355a676 100644 --- a/wolfcrypt/src/random.c +++ b/wolfcrypt/src/random.c @@ -6363,14 +6363,20 @@ static WARN_UNUSED_RESULT int PollAndReSeed(WC_RNG* rng, const byte* additional, /* newSeed is a byte[] in the plain build but a byte* in the * SMALL_STACK_CACHE build, so use the explicit buffer length (not * sizeof) to zero the whole reseed entropy buffer in both cases. */ - #ifdef WOLFSSL_CHECK_MEM_ZERO - wc_MemZero_Add("PollAndReSeed newSeed", newSeed, - SEED_SZ + SEED_BLOCK_SZ); - #endif - ForceZero(newSeed, SEED_SZ + SEED_BLOCK_SZ); - #ifdef WOLFSSL_CHECK_MEM_ZERO - wc_MemZero_Check(newSeed, SEED_SZ + SEED_BLOCK_SZ); + #ifdef WOLFSSL_SMALL_STACK_CACHE + /* newSeed is NULL when the DRBG state is missing. */ + if (newSeed != NULL) #endif + { + #ifdef WOLFSSL_CHECK_MEM_ZERO + wc_MemZero_Add("PollAndReSeed newSeed", newSeed, + SEED_SZ + SEED_BLOCK_SZ); + #endif + ForceZero(newSeed, SEED_SZ + SEED_BLOCK_SZ); + #ifdef WOLFSSL_CHECK_MEM_ZERO + wc_MemZero_Check(newSeed, SEED_SZ + SEED_BLOCK_SZ); + #endif + } #endif } /* else pass back the failure code from wc_RNG_HealthTestLocal(): diff --git a/wolfcrypt/src/wc_slhdsa.c b/wolfcrypt/src/wc_slhdsa.c index 3703a39e9de..9f43a36d664 100644 --- a/wolfcrypt/src/wc_slhdsa.c +++ b/wolfcrypt/src/wc_slhdsa.c @@ -4643,6 +4643,9 @@ static int slhdsakey_wots_pkgen_chain_c(SlhDsaKey* key, const byte* sk_seed, #else byte sk[SLHDSA_MAX_N]; + /* n is from the parameter table, so it fits sk. */ + WC_ASSUME(n <= SLHDSA_MAX_N); + /* Step 4. len consecutive addresses. */ for (i = 0; i < len; i++) { /* Step 5. Set chain address for WOTS PRF. */ @@ -5228,6 +5231,9 @@ static int slhdsakey_wots_sign(SlhDsaKey* key, const byte* m, { byte sk[SLHDSA_MAX_N]; + /* n is from the parameter table, so it fits sk. */ + WC_ASSUME(n <= SLHDSA_MAX_N); + /* sk will hold secret WOTS+ leaves; baseline-zero and register it up * front (block has a single exit through the ForceZero below). */ #ifdef WOLFSSL_CHECK_MEM_ZERO diff --git a/wolfssl/wolfcrypt/types.h b/wolfssl/wolfcrypt/types.h index 5ebba90f72b..fde1c49364f 100644 --- a/wolfssl/wolfcrypt/types.h +++ b/wolfssl/wolfcrypt/types.h @@ -2539,6 +2539,23 @@ WOLFSSL_API word32 CheckRunTimeSettings(void); struct wc_static_assert_dummy_struct #endif +/* WC_ASSUME(expr) tells an optimizing compiler that expr is always true, so it + * can use it, for example to bound a value. The compiler removes the check, so + * no code is added. expr must never be false, or behavior is undefined. It + * does nothing at -O0, where the check would stay as a branch. Define + * WC_NO_ASSUME to turn it off, or define WC_ASSUME to supply your own. */ +#if defined(WC_NO_ASSUME) + #define WC_ASSUME(expr) WC_DO_NOTHING +#elif !defined(WC_ASSUME) + #if defined(__OPTIMIZE__) && (defined(__clang__) || \ + (defined(__GNUC_PREREQ) && __GNUC_PREREQ(4, 5))) + #define WC_ASSUME(expr) \ + do { if (!(expr)) __builtin_unreachable(); } while (0) + #else + #define WC_ASSUME(expr) WC_DO_NOTHING + #endif +#endif + #ifndef WC_RELAX_LONG_LOOP #define WC_RELAX_LONG_LOOP() WC_DO_NOTHING #endif