Skip to content
C++
Domain Track
Difficulty 3/5

Security and Cryptography in C++

C++ cryptography with OpenSSL (EVP API), libsodium, secure memory handling, constant-time comparisons, TLS with Asio SSL, and common security pitfalls.

TL;DR

Never roll your own crypto. Use OpenSSL's EVP API for standard algorithms or libsodium for a safer, higher-level interface. Always check return codes, zero sensitive buffers, and use constant-time comparison for secrets.

cpp
// libsodium symmetric encryption
unsigned char key[crypto_secretbox_KEYBYTES];
unsigned char nonce[crypto_secretbox_NONCEBYTES];
randombytes_buf(key, sizeof key);
randombytes_buf(nonce, sizeof nonce);

std::vector<unsigned char> ciphertext(msg.size() + crypto_secretbox_MACBYTES);
crypto_secretbox_easy(ciphertext.data(), msg.data(), msg.size(), nonce, key);

OpenSSL EVP: AES-256-GCM Encryption

cpp
#include <openssl/evp.h>
#include <vector>
#include <stdexcept>

// Returns {ciphertext, tag}
std::pair<std::vector<unsigned char>, std::array<unsigned char,16>>
aes_gcm_encrypt(const unsigned char* key,      // 32 bytes
                const unsigned char* iv,       // 12 bytes recommended
                const unsigned char* plaintext, size_t pt_len,
                const unsigned char* aad,      // additional authenticated data
                size_t aad_len) {

    EVP_CIPHER_CTX* ctx = EVP_CIPHER_CTX_new();
    if (!ctx) throw std::runtime_error("EVP_CIPHER_CTX_new failed");

    EVP_EncryptInit_ex(ctx, EVP_aes_256_gcm(), nullptr, nullptr, nullptr);
    EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, 12, nullptr);
    EVP_EncryptInit_ex(ctx, nullptr, nullptr, key, iv);

    int len = 0;
    if (aad && aad_len > 0)
        EVP_EncryptUpdate(ctx, nullptr, &len, aad, (int)aad_len);

    std::vector<unsigned char> ciphertext(pt_len);
    EVP_EncryptUpdate(ctx, ciphertext.data(), &len, plaintext, (int)pt_len);
    int ct_len = len;

    EVP_EncryptFinal_ex(ctx, ciphertext.data() + ct_len, &len);

    std::array<unsigned char, 16> tag;
    EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, tag.data());
    EVP_CIPHER_CTX_free(ctx);

    return {ciphertext, tag};
}

OpenSSL EVP: SHA-256 Hash

cpp
#include <openssl/evp.h>
#include <array>

std::array<unsigned char, 32> sha256(const unsigned char* data, size_t len) {
    EVP_MD_CTX* ctx = EVP_MD_CTX_new();
    EVP_DigestInit_ex(ctx, EVP_sha256(), nullptr);
    EVP_DigestUpdate(ctx, data, len);

    std::array<unsigned char, 32> digest;
    unsigned int digest_len = 32;
    EVP_DigestFinal_ex(ctx, digest.data(), &digest_len);
    EVP_MD_CTX_free(ctx);
    return digest;
}

// Hex representation
std::string to_hex(std::span<const unsigned char> bytes) {
    std::string result;
    result.reserve(bytes.size() * 2);
    for (auto b : bytes)
        result += std::format("{:02x}", b);
    return result;
}

OpenSSL EVP: HMAC-SHA256

cpp
#include <openssl/hmac.h>

std::array<unsigned char, 32> hmac_sha256(
        const unsigned char* key, size_t key_len,
        const unsigned char* data, size_t data_len) {

    std::array<unsigned char, 32> result;
    unsigned int result_len = 32;

    HMAC(EVP_sha256(), key, (int)key_len,
         data, data_len,
         result.data(), &result_len);
    return result;
}

libsodium: Symmetric Encryption

cpp
#include <sodium.h>

// Initialize once at startup
if (sodium_init() < 0) throw std::runtime_error("libsodium init failed");

// Encrypt
void encrypt_message(const std::string& message) {
    unsigned char key[crypto_secretbox_KEYBYTES];   // 32 bytes
    unsigned char nonce[crypto_secretbox_NONCEBYTES]; // 24 bytes

    crypto_secretbox_keygen(key);
    randombytes_buf(nonce, sizeof nonce);

    std::vector<unsigned char> ciphertext(
        message.size() + crypto_secretbox_MACBYTES);

    crypto_secretbox_easy(
        ciphertext.data(),
        reinterpret_cast<const unsigned char*>(message.data()),
        message.size(), nonce, key);
}

// Decrypt
bool decrypt_message(
        const unsigned char* ciphertext, size_t ct_len,
        const unsigned char* nonce, const unsigned char* key,
        std::string& out) {

    std::vector<unsigned char> plaintext(ct_len - crypto_secretbox_MACBYTES);
    if (crypto_secretbox_open_easy(
            plaintext.data(), ciphertext, ct_len, nonce, key) != 0)
        return false;  // authentication failed — tampered or wrong key

    out.assign(plaintext.begin(), plaintext.end());
    return true;
}

libsodium: Ed25519 Signatures

cpp
// Generate a keypair
unsigned char pk[crypto_sign_PUBLICKEYBYTES];
unsigned char sk[crypto_sign_SECRETKEYBYTES];
crypto_sign_keypair(pk, sk);

// Sign a message
std::string message = "authenticate this";
std::vector<unsigned char> sig(crypto_sign_BYTES);
unsigned long long sig_len;

crypto_sign_detached(
    sig.data(), &sig_len,
    reinterpret_cast<const unsigned char*>(message.data()),
    message.size(), sk);

// Verify
bool ok = crypto_sign_verify_detached(
    sig.data(),
    reinterpret_cast<const unsigned char*>(message.data()),
    message.size(), pk) == 0;

libsodium: Password Hashing (Argon2id)

cpp
char hash[crypto_pwhash_STRBYTES];
const char* password = "hunter2";

// Hash (for storage)
if (crypto_pwhash_str(
        hash, password, strlen(password),
        crypto_pwhash_OPSLIMIT_INTERACTIVE,
        crypto_pwhash_MEMLIMIT_INTERACTIVE) != 0)
    throw std::runtime_error("out of memory for hashing");

// Verify
bool matches = crypto_pwhash_str_verify(
    hash, password, strlen(password)) == 0;

Secure Memory Handling

cpp
// Allocate memory that won't be swapped or appear in core dumps
void* secret = sodium_malloc(32);   // guarded allocation
// or:
unsigned char key[32];
sodium_mlock(key, sizeof key);  // prevent swapping

// ALWAYS zero sensitive data before freeing
sodium_memzero(key, sizeof key);         // libsodium — compiler can't optimize away
OPENSSL_cleanse(key, sizeof key);        // OpenSSL equivalent
// Don't use memset() — optimizer may remove it if buffer is "dead"

// volatile trick (if no library available)
volatile unsigned char* p = key;
for (size_t i = 0; i < sizeof key; ++i) p[i] = 0;

Constant-Time Comparison

cpp
// WRONG: early exit leaks timing info about secret
bool bad_compare(const char* a, const char* b, size_t len) {
    for (size_t i = 0; i < len; ++i)
        if (a[i] != b[i]) return false;  // attacker can measure response time
    return true;
}

// CORRECT: constant-time comparison
bool ct_compare(const unsigned char* a, const unsigned char* b, size_t len) {
    return CRYPTO_memcmp(a, b, len) == 0;     // OpenSSL
    // or: sodium_memcmp(a, b, len) == 0;    // libsodium
}

TLS with Asio SSL

cpp
#include <asio/ssl.hpp>

asio::io_context io;
asio::ssl::context ssl_ctx(asio::ssl::context::tls_client);

// Configure
ssl_ctx.set_verify_mode(asio::ssl::verify_peer);
ssl_ctx.set_default_verify_paths();
ssl_ctx.load_verify_file("ca.crt");

// Create SSL stream
using SSLStream = asio::ssl::stream<asio::ip::tcp::socket>;
SSLStream stream(io, ssl_ctx);

// Set SNI hostname (required for many servers)
SSL_set_tlsext_host_name(stream.native_handle(), "example.com");

// Connect and handshake
asio::ip::tcp::resolver resolver(io);
auto endpoints = resolver.resolve("example.com", "443");
asio::connect(stream.next_layer(), endpoints);
stream.handshake(asio::ssl::stream_base::client);

// Read/write like a normal socket
asio::write(stream, asio::buffer("GET / HTTP/1.0\r\n\r\n"));

std::string response;
asio::error_code ec;
asio::read(stream, asio::dynamic_buffer(response), ec);

Common Pitfalls

cpp
1. Nonce/IV reuse — catastrophic for GCM/CTR modes (breaks confidentiality)
   Fix: use a counter or random nonce, never reuse

2. Not checking return values — OpenSSL returns 1 for success
   Fix: always check EVP_* return codes

3. Weak RNG — using rand() or time-based seeds for crypto
   Fix: use randombytes_buf() (libsodium) or RAND_bytes() (OpenSSL)

4. Key material in core dumps / swap
   Fix: mlock(), guarded allocations, zero on free

5. Timing attacks on secret comparison
   Fix: use constant-time compare (CRYPTO_memcmp, sodium_memcmp)

6. Using ECB mode — reveals plaintext patterns
   Fix: use GCM (authenticated) or CBC with HMAC

7. Short keys — AES-128 is fine, never go below 128 bits

Library Comparison

OpenSSLlibsodiumBotanmbedTLS
API complexityHigh (EVP is verbose)Low (opinionated)MediumMedium
Algorithm choiceUnlimitedCurated safe setWideWide
TLS/DTLSYesNo (use libsodium + hand-roll)YesYes (embedded focus)
FootprintLargeSmallMediumSmall
LicenseApache 2.0ISCSimplified BSDApache 2.0
Best forGeneral use, TLSModern crypto, easy APIWide algorithm supportEmbedded/IoT

Fuzzing with libFuzzer

Coverage-guided fuzzing finds vulnerabilities that unit tests miss. libFuzzer is built into Clang and requires no external harness — you write a single entry point and the fuzzer drives execution automatically, mutating inputs to maximize branch coverage.

cpp
// fuzz_target.cpp — compile with: clang++ -fsanitize=fuzzer,address fuzz_target.cpp
#include <cstdint>
#include <cstddef>
#include <string>

// Your production function under test
bool parse_packet(const uint8_t* data, size_t size);

// libFuzzer entry point — called repeatedly with mutated input
extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
    // Wrap in try/catch if you want to suppress expected exceptions
    parse_packet(data, size);
    return 0;  // non-zero means "crash" and causes libFuzzer to report a bug
}
bash
# Build fuzz target with coverage instrumentation + ASan
clang++ -std=c++20 -fsanitize=fuzzer,address,undefined \
        -fprofile-instr-generate -fcoverage-mapping \
        fuzz_target.cpp libmyparser.a -o fuzz_parser

# Run with a corpus directory (libFuzzer mutates existing inputs)
mkdir -p corpus seed_corpus
cp tests/inputs/*.bin seed_corpus/
./fuzz_parser corpus/ seed_corpus/ -max_len=4096 -jobs=4

# Minimize a crashing input
./fuzz_parser -minimize_crash=1 crash-abc123 -exact_artifact_path=min_crash

Integrate sanitizers for maximum bug-finding power: -fsanitize=fuzzer,address catches heap overflows, while adding undefined catches UB like signed integer overflow. Keep a seed corpus of valid inputs to help the fuzzer reach deeper code paths quickly.


Cryptographic Primitives: libsodium vs OpenSSL EVP

Choosing between libsodium and OpenSSL EVP depends on your constraints. libsodium hides algorithm choice behind safe defaults (XSalsa20-Poly1305, Ed25519, Argon2id), while OpenSSL EVP exposes the full algorithm zoo — necessary when interoperability with legacy systems demands specific ciphers.

OperationlibsodiumOpenSSL EVP
Symmetric encryptioncrypto_secretbox_easy (XSalsa20-Poly1305)EVP_EncryptInit_ex + cipher constant
AEADcrypto_aead_chacha20poly1305_ietf_encryptAES-256-GCM via EVP_aes_256_gcm()
Digital signaturescrypto_sign_detached (Ed25519)EVP_DigestSign* with EC/RSA key
Password hashingcrypto_pwhash_str (Argon2id)No built-in KDF; use EVP_PBE_scrypt
Key exchangecrypto_kx_* (X25519)EVP_PKEY_derive with X25519
Constant-time comparesodium_memcmpCRYPTO_memcmp
Secure zerosodium_memzeroOPENSSL_cleanse

Prefer libsodium for new code; use OpenSSL EVP when you need TLS, PKCS#11 hardware tokens, or FIPS 140-2 validated modules.


Secure Memory Handling: mlock and Zeroing Secrets

Key material left in memory can be recovered from swap files, core dumps, or cold-boot attacks. The defense is a combination of mlock/sodium_mlock to prevent swapping and a guaranteed zero-on-release discipline.

cpp
#include <sodium.h>
#include <sys/mman.h>

// Option 1: libsodium guarded heap allocation
// Surrounds the allocation with guard pages; access-after-free is caught immediately
void* sensitive = sodium_malloc(32);
if (!sensitive) throw std::bad_alloc{};
// ... use it ...
sodium_memzero(sensitive, 32);   // zero first, then free
sodium_free(sensitive);

// Option 2: stack buffer locked into RAM
alignas(64) unsigned char key[32];
sodium_mlock(key, sizeof key);           // tells OS: do not swap this page
// ... fill and use key ...
sodium_memzero(key, sizeof key);         // zero before scope exit
sodium_munlock(key, sizeof key);

// Option 3: raw mlock (POSIX) when libsodium not available
unsigned char raw_key[32];
mlock(raw_key, sizeof raw_key);
// volatile pointer trick — optimizer cannot elide this loop
volatile unsigned char* vp = raw_key;
for (size_t i = 0; i < sizeof raw_key; ++i) vp[i] = 0;
munlock(raw_key, sizeof raw_key);

memset on a dead buffer is routinely removed by optimizing compilers — always use sodium_memzero, OPENSSL_cleanse, or explicit_bzero (POSIX.1-2024). Set the PR_SET_DUMPABLE prctl flag on Linux to suppress core dumps for sensitive processes.


TLS/DTLS Library Comparison

Three libraries dominate C++ TLS integration: OpenSSL, BoringSSL, and mbedTLS. The right choice depends on your deployment target, licensing requirements, and tolerance for API surface area.

CriterionOpenSSL 3.xBoringSSLmbedTLS 3.x
TLS versions1.0–1.31.2–1.3 (removes legacy)1.2–1.3
DTLSYesYesYes
QUIC supportYes (3.x)Yes (Chromium/curl)Partial
API styleCallback-heavy, verboseOpenSSL-derived, prunedStruct-based, clean
FIPS modeYes (FIPS provider)NoNo
Footprint~3 MB~2 MB~300 KB
LicenseApache 2.0OpenSSL + ISCApache 2.0
Best forGeneral servers, FIPSChrome, gRPC, curlEmbedded, IoT, RTOS
cpp
// mbedTLS minimal TLS client skeleton
#include <mbedtls/net_sockets.h>
#include <mbedtls/ssl.h>
#include <mbedtls/entropy.h>
#include <mbedtls/ctr_drbg.h>

mbedtls_ssl_context ssl;
mbedtls_ssl_config conf;
mbedtls_entropy_context entropy;
mbedtls_ctr_drbg_context ctr_drbg;
mbedtls_net_context server_fd;

mbedtls_ssl_init(&ssl);
mbedtls_ssl_config_init(&conf);
mbedtls_entropy_init(&entropy);
mbedtls_ctr_drbg_init(&ctr_drbg);

mbedtls_ctr_drbg_seed(&ctr_drbg, mbedtls_entropy_func,
                       &entropy, nullptr, 0);
mbedtls_ssl_config_defaults(&conf, MBEDTLS_SSL_IS_CLIENT,
                            MBEDTLS_SSL_TRANSPORT_STREAM,
                            MBEDTLS_SSL_PRESET_DEFAULT);
mbedtls_ssl_conf_rng(&conf, mbedtls_ctr_drbg_random, &ctr_drbg);
mbedtls_ssl_conf_authmode(&conf, MBEDTLS_SSL_VERIFY_REQUIRED);
mbedtls_ssl_setup(&ssl, &conf);
mbedtls_ssl_set_hostname(&ssl, "example.com");
// ... connect net_fd, set bio, handshake ...

Code Hardening Compile Flags

Compiler and linker hardening flags provide defense-in-depth against exploitation. These flags do not fix bugs, but they raise the cost of exploiting memory-safety vulnerabilities like buffer overflows and use-after-free.

cmake
# CMakeLists.txt — hardening preset for Linux release builds
if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
    target_compile_options(myapp PRIVATE
        -fstack-protector-strong          # guard against stack smashing
        -D_FORTIFY_SOURCE=2               # GLIBC bounds-checking for string/mem funcs
        -fPIE                             # position-independent code (needed for ASLR)
        -fcf-protection=full              # Intel CET: shadow stack + indirect branch tracking
        -fvisibility=hidden               # reduce exported symbol surface
        -Wformat -Wformat-security        # warn on printf format string bugs
    )
    target_link_options(myapp PRIVATE
        -pie                              # position-independent executable
        -Wl,-z,relro                      # mark GOT read-only after relocation
        -Wl,-z,now                        # resolve all symbols at load (full RELRO)
        -Wl,-z,noexecstack                # non-executable stack segment
        -Wl,-z,separate-code             # separate code from data pages
    )
endif()
bash
# Verify hardening on a built binary
checksec --file=./myapp
# or:
hardening-check ./myapp

# Typical secure output:
# RELRO:    Full RELRO
# STACK CANARY: Canary found
# NX:       NX enabled
# PIE:      PIE enabled
# FORTIFY:  Fortified
FlagProtectionPerformance cost
-fstack-protector-strongStack canary on functions with arrays/alloca~1%
-D_FORTIFY_SOURCE=2Runtime bounds checks in libc~0%
-fPIE + -pieASLR for the executable~0%
-Wl,-z,relro,-z,nowFull RELRO: read-only GOT~0% startup overhead
-fcf-protection=fullControl-flow integrity (Intel CET)~1–3%

Threat Modeling for C++ Applications

Threat modeling is the practice of identifying what can go wrong before writing a line of code. The STRIDE framework (Spoofing, Tampering, Repudiation, Information Disclosure, Denial of Service, Elevation of Privilege) gives a structured vocabulary for enumerating threats.

cpp
STRIDE threat enumeration for a typical C++ network service:

┌────────────────────────────────────────────────────────┐
│  External clients       │  Trust boundary              │
│  (untrusted input)      │  (TLS termination)           │
│                         ▼                              │
│            ┌─────────────────────┐                     │
│            │  Request parser     │ ← Tampering, DoS    │
│            └────────┬────────────┘                     │
│                     │                                  │
│            ┌────────▼────────────┐                     │
│            │  Business logic     │ ← EoP, Info Disc.   │
│            └────────┬────────────┘                     │
│                     │  Trust boundary (DB credentials) │
│            ┌────────▼────────────┐                     │
│            │  Database / secrets │ ← Spoofing, Tamper  │
│            └─────────────────────┘                     │
└────────────────────────────────────────────────────────┘

Attack surface checklist:
- All network-facing parsers (binary protocols, JSON, XML, protobuf)
- File paths constructed from user input (path traversal)
- Shell commands constructed from user input (command injection)
- Shared memory / IPC channels between processes
- Signal handlers that call async-signal-unsafe functions
- Third-party plugins or shared libraries loaded at runtime

Prioritize threats by likelihood × impact. Document mitigations inline in code comments referencing threat IDs so reviewers can trace security decisions back to the model.


Supply Chain Security

Modern C++ projects pull in dozens of transitive dependencies — each is a potential attack vector. Supply chain hardening covers dependency auditing, reproducible builds, and Software Bill of Materials (SBOM) generation.

cmake
# Pin dependencies to exact commit SHAs in FetchContent, not branch names
FetchContent_Declare(fmt
    GIT_REPOSITORY https://github.com/fmtlib/fmt.git
    GIT_TAG        e69e5f977d458f2650bb346dadf2ad30c5320281  # v10.2.1
    GIT_SHALLOW    FALSE   # FALSE ensures full history for verification
)

# Verify source archive checksums when using URL downloads
FetchContent_Declare(zlib
    URL      https://zlib.net/zlib-1.3.1.tar.gz
    URL_HASH SHA256=9a93b2b7dfdac77ceba5a558a580e74667dd6fede4585b91eefb60f03b72df23
)
bash
# Generate SBOM in CycloneDX format (requires cmake-sbom or syft)
syft dir:. -o cyclonedx-json > sbom.cdx.json

# Audit known CVEs against your dependency list
grype sbom:sbom.cdx.json

# Reproducible builds: record build environment
cmake -DCMAKE_BUILD_TYPE=Release \
      -DCMAKE_C_COMPILER_LAUNCHER=ccache \
      -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
      -DSOURCE_DATE_EPOCH=$(git log -1 --format=%ct) \
      ..

# Lock vcpkg dependencies to a specific baseline commit
# vcpkg.json
json
{
  "name": "myapp",
  "version": "1.0.0",
  "builtin-baseline": "7adc2e4d7ea7a90a4b0bf6e4eb9e3b7d8bfd1e63",
  "dependencies": [
    { "name": "openssl", "version>=": "3.2.0" },
    { "name": "libsodium", "version>=": "1.0.19" }
  ]
}
PracticeToolWhat it prevents
Pin to commit SHACMake FetchContent, vcpkg baselineMutable tag hijacking
Checksum verificationURL_HASH in CMakeTampered tarballs
SBOM generationsyft, cmake-sbomUnknown dependencies
CVE scanninggrype, OSV-ScannerKnown vulnerable versions
Reproducible buildsSOURCE_DATE_EPOCH, ccacheBinary tampering after build