Kernel Bypass Networking in C++ (DPDK & RDMA)

Standard TCP path:
  NIC receives packet
  → DMA into kernel buffer            ~200ns
  → NIC interrupt fires               ~1–2µs  (interrupt latency)
  → Kernel TCP/IP stack               ~5–10µs
  → Copy to userspace buffer          ~500ns
  → recv() syscall overhead           ~200ns
  Total: ~7–15µs per message

DPDK path:
  NIC receives packet
  → DMA into DPDK-managed buffer      ~200ns
  → Userspace poll detects packet     ~100–300ns (no interrupt)
  → Application processes buffer      ~200ns
  Total: ~500ns–1µs per message

#include <rte_eal.h>
#include <rte_ethdev.h>
#include <rte_mbuf.h>

int main(int argc, char* argv[]) {
    // Initialize EAL — sets up hugepages, CPU affinity, memory
    int ret = rte_eal_init(argc, argv);
    if (ret < 0) rte_exit(EXIT_FAILURE, "EAL init failed\n");
    argc -= ret;
    argv += ret;

    uint16_t port_id = 0;  // first NIC port

    // Create memory pool for packet buffers (hugepage-backed)
    rte_mempool* mp = rte_pktmbuf_pool_create(
        "MBUF_POOL",
        8192,            // number of elements
        256,             // cache size per core
        0,               // private data size
        RTE_MBUF_DEFAULT_BUF_SIZE,
        rte_socket_id()  // NUMA node
    );
    if (!mp) rte_exit(EXIT_FAILURE, "Cannot create mbuf pool\n");

    // Configure the port
    rte_eth_conf port_conf{};
    port_conf.rxmode.max_lro_pkt_size = RTE_ETHER_MAX_LEN;
    rte_eth_dev_configure(port_id, 1, 1, &port_conf);

    // Setup RX and TX queues
    rte_eth_rx_queue_setup(port_id, 0, 128, rte_eth_dev_socket_id(port_id),
                           nullptr, mp);
    rte_eth_tx_queue_setup(port_id, 0, 128, rte_eth_dev_socket_id(port_id),
                           nullptr);

    rte_eth_dev_start(port_id);

    return runApp(port_id, mp);
}

constexpr uint16_t BURST_SIZE = 32;

void rxLoop(uint16_t port_id) {
    rte_mbuf* pkts[BURST_SIZE];

    while (running) {
        // Poll RX queue — no syscall, no interrupt, no context switch
        uint16_t nb_rx = rte_eth_rx_burst(port_id, 0, pkts, BURST_SIZE);

        for (uint16_t i = 0; i < nb_rx; ++i) {
            processPacket(pkts[i]);
            rte_pktmbuf_free(pkts[i]);  // return to pool
        }
        // nb_rx == 0 is normal — keep polling
    }
}

void processPacket(rte_mbuf* pkt) {
    // Direct pointer into DMA buffer — zero copy
    auto* eth = rte_pktmbuf_mtod(pkt, rte_ether_hdr*);
    auto* ip  = reinterpret_cast<rte_ipv4_hdr*>(eth + 1);
    auto* udp = reinterpret_cast<rte_udp_hdr*>(ip + 1);
    auto* payload = reinterpret_cast<uint8_t*>(udp + 1);
    uint16_t len = rte_be_to_cpu_16(udp->dgram_len) - sizeof(rte_udp_hdr);

    decodeMarketData(payload, len);
}

void sendOrder(uint16_t port_id, rte_mempool* mp,
               const OrderMessage& order) {
    rte_mbuf* pkt = rte_pktmbuf_alloc(mp);
    if (!pkt) { /* handle pool exhaustion */ return; }

    // Build packet in-place — no copy from application buffer
    uint8_t* data = reinterpret_cast<uint8_t*>(
        rte_pktmbuf_append(pkt, sizeof(rte_ether_hdr) +
                                sizeof(rte_ipv4_hdr)  +
                                sizeof(rte_udp_hdr)   +
                                sizeof(OrderMessage)));

    buildEthHeader(data, /* dst/src MAC */ );
    buildIPHeader(data + sizeof(rte_ether_hdr), sizeof(OrderMessage));
    buildUDPHeader(data + sizeof(rte_ether_hdr) + sizeof(rte_ipv4_hdr));
    memcpy(data + sizeof(rte_ether_hdr) + sizeof(rte_ipv4_hdr)
                + sizeof(rte_udp_hdr), &order, sizeof(order));

    rte_eth_tx_burst(port_id, 0, &pkt, 1);
}

Standard TCP send:       ~5–15 µs
DPDK UDP send:           ~1–2 µs
RDMA WRITE (one-way):    ~600–1000 ns (wire latency dominates)
InfiniBand QDR:          ~1.5 µs RTT
100GbE RoCE:             ~2 µs RTT

Verbs API (libibverbs):
  - Protection Domain (PD): isolates resources
  - Queue Pair (QP): bidirectional channel, has Send Queue + Receive Queue
  - Completion Queue (CQ): signals completed operations
  - Memory Region (MR): registers memory with the NIC (pinned, DMA-ready)
  - Work Request (WR): describes an operation (SEND, RECV, WRITE, READ)

#include <infiniband/verbs.h>

struct RDMAContext {
    ibv_context*     ctx;    // device context
    ibv_pd*          pd;     // protection domain
    ibv_cq*          cq;     // completion queue
    ibv_qp*          qp;     // queue pair
    ibv_mr*          mr;     // memory region
    void*            buf;    // application buffer (pinned)
    size_t           buf_size;
};

RDMAContext setup(const char* device_name, size_t buf_size) {
    RDMAContext ctx{};
    ctx.buf_size = buf_size;

    // Open device
    ibv_device** dev_list = ibv_get_device_list(nullptr);
    ctx.ctx = ibv_open_device(dev_list[0]);  // first device
    ibv_free_device_list(dev_list);

    ctx.pd = ibv_alloc_pd(ctx.ctx);
    ctx.cq = ibv_create_cq(ctx.ctx, 128, nullptr, nullptr, 0);

    // Create Queue Pair
    ibv_qp_init_attr qp_attr{};
    qp_attr.send_cq = ctx.cq;
    qp_attr.recv_cq = ctx.cq;
    qp_attr.qp_type = IBV_QPT_RC;  // Reliable Connected
    qp_attr.cap.max_send_wr  = 64;
    qp_attr.cap.max_recv_wr  = 64;
    qp_attr.cap.max_send_sge = 1;
    qp_attr.cap.max_recv_sge = 1;
    ctx.qp = ibv_create_qp(ctx.pd, &qp_attr);

    // Register memory — pins pages, enables DMA
    ctx.buf = aligned_alloc(4096, buf_size);
    ctx.mr = ibv_reg_mr(ctx.pd, ctx.buf, buf_size,
                        IBV_ACCESS_LOCAL_WRITE |
                        IBV_ACCESS_REMOTE_WRITE |
                        IBV_ACCESS_REMOTE_READ);
    return ctx;
}

// After QP is connected to remote peer (via out-of-band exchange of QP info)
void rdmaWrite(RDMAContext& ctx,
               uint64_t remote_addr, uint32_t remote_key,
               const void* data, size_t len) {
    // Copy data into registered buffer
    memcpy(ctx.buf, data, len);

    ibv_sge sge{};
    sge.addr   = reinterpret_cast<uint64_t>(ctx.buf);
    sge.length = static_cast<uint32_t>(len);
    sge.lkey   = ctx.mr->lkey;

    ibv_send_wr wr{};
    wr.opcode     = IBV_WR_RDMA_WRITE;
    wr.sg_list    = &sge;
    wr.num_sge    = 1;
    wr.wr.rdma.remote_addr = remote_addr;
    wr.wr.rdma.rkey        = remote_key;
    wr.send_flags = IBV_SEND_SIGNALED;  // generate completion event

    ibv_send_wr* bad_wr = nullptr;
    ibv_post_send(ctx.qp, &wr, &bad_wr);

    // Poll for completion
    ibv_wc wc{};
    while (ibv_poll_cq(ctx.cq, 1, &wc) == 0) {}  // spin — no syscall
    if (wc.status != IBV_WC_SUCCESS)
        throw std::runtime_error("RDMA write failed");
}

# Install OpenOnload, then preload the library
LD_PRELOAD=/usr/lib/libciul1.so ./trading_app

# Or link directly
g++ -o app app.cpp -lonload

// No code changes needed — existing socket code benefits automatically
int sock = socket(AF_INET, SOCK_STREAM, 0);
connect(sock, ...);  // OpenOnload intercepts this — no kernel involvement
send(sock, data, len, 0);  // kernel-bypass path

# Disable power management — frequency scaling kills latency consistency
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor

# Isolate cores from Linux scheduler (add to kernel cmdline)
isolcpus=2,3,4,5 nohz_full=2,3,4,5 rcu_nocbs=2,3,4,5

# Pin DPDK app to isolated cores
taskset -c 2,3 ./dpdk_app -- --lcores 0@2,1@3

# Move network IRQs off isolated cores
set_irq_affinity.sh eth0 1  # IRQs on core 1 only