Systems Programming in C++

TL;DR

Systems programming in C++ means wrapping POSIX/Linux C APIs with RAII. Use std::system_error for error reporting. Always check return values; use errno via std::errc.

RAII File Descriptor

cpp

Godbolt

#include <unistd.h>
#include <fcntl.h>
#include <cerrno>
#include <system_error>

class FileDescriptor {
    int fd_ = -1;

public:
    FileDescriptor() = default;
    explicit FileDescriptor(int fd) : fd_{fd} {}

    // Non-copyable, movable
    FileDescriptor(const FileDescriptor&) = delete;
    FileDescriptor& operator=(const FileDescriptor&) = delete;

    FileDescriptor(FileDescriptor&& o) noexcept : fd_{std::exchange(o.fd_, -1)} {}
    FileDescriptor& operator=(FileDescriptor&& o) noexcept {
        if (this != &o) { close(); fd_ = std::exchange(o.fd_, -1); }
        return *this;
    }

    ~FileDescriptor() { close(); }

    int get() const { return fd_; }
    bool valid() const { return fd_ >= 0; }
    int release() { return std::exchange(fd_, -1); }

private:
    void close() { if (fd_ >= 0) ::close(fd_); }
};

// Usage
FileDescriptor fd{::open("/etc/hosts", O_RDONLY)};
if (!fd.valid()) throw std::system_error(errno, std::generic_category());

char buf[1024];
ssize_t n = ::read(fd.get(), buf, sizeof(buf));
// fd closed automatically at end of scope

Process Management

cpp

Godbolt

#include <unistd.h>
#include <sys/wait.h>

// Fork a child process
pid_t child = fork();
if (child < 0) {
    throw std::system_error(errno, std::generic_category(), "fork");
} else if (child == 0) {
    // Child process
    execl("/usr/bin/ls", "ls", "-la", nullptr);
    _exit(1);  // exec failed
} else {
    // Parent process: wait for child
    int status;
    waitpid(child, &status, 0);
    if (WIFEXITED(status))
        std::println("child exited: {}", WEXITSTATUS(status));
}

// Run a command and capture output (popen)
FILE* pipe = popen("date", "r");
if (!pipe) throw std::runtime_error("popen failed");
char line[256];
while (fgets(line, sizeof(line), pipe))
    std::print("{}", line);
pclose(pipe);

Memory-Mapped Files

cpp

Godbolt

#include <sys/mman.h>
#include <sys/stat.h>

class MappedFile {
    void*  addr_ = MAP_FAILED;
    size_t size_ = 0;

public:
    MappedFile(const char* path, bool writable = false) {
        int flags = writable ? O_RDWR : O_RDONLY;
        FileDescriptor fd{::open(path, flags)};
        if (!fd.valid()) throw std::system_error(errno, std::generic_category());

        struct stat st;
        if (fstat(fd.get(), &st) < 0)
            throw std::system_error(errno, std::generic_category());
        size_ = st.st_size;

        int prot = PROT_READ | (writable ? PROT_WRITE : 0);
        addr_ = mmap(nullptr, size_, prot, MAP_SHARED, fd.get(), 0);
        if (addr_ == MAP_FAILED)
            throw std::system_error(errno, std::generic_category());
    }

    ~MappedFile() { if (addr_ != MAP_FAILED) munmap(addr_, size_); }

    MappedFile(const MappedFile&) = delete;
    MappedFile& operator=(const MappedFile&) = delete;

    std::span<const std::byte> bytes() const {
        return {static_cast<const std::byte*>(addr_), size_};
    }
    std::string_view str() const {
        return {static_cast<const char*>(addr_), size_};
    }
    size_t size() const { return size_; }
};

// Parse a large file without reading it into memory
MappedFile f{"/var/log/app.log"};
auto content = f.str();
// Process content as string_view — no copy

POSIX Sockets

cpp

Godbolt

#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>

// TCP server
int setup_server(uint16_t port) {
    int sock = socket(AF_INET, SOCK_STREAM, 0);
    if (sock < 0) throw std::system_error(errno, std::generic_category());

    // Allow reuse of port immediately after close
    int yes = 1;
    setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes));

    sockaddr_in addr{};
    addr.sin_family      = AF_INET;
    addr.sin_addr.s_addr = INADDR_ANY;
    addr.sin_port        = htons(port);

    if (bind(sock, (sockaddr*)&addr, sizeof(addr)) < 0)
        throw std::system_error(errno, std::generic_category());
    if (listen(sock, SOMAXCONN) < 0)
        throw std::system_error(errno, std::generic_category());

    return sock;
}

// Accept loop
FileDescriptor server{setup_server(8080)};
while (true) {
    sockaddr_in client_addr{};
    socklen_t   client_len = sizeof(client_addr);
    int client_fd = accept(server.get(), (sockaddr*)&client_addr, &client_len);
    if (client_fd < 0) continue;

    char ip[INET_ADDRSTRLEN];
    inet_ntop(AF_INET, &client_addr.sin_addr, ip, sizeof(ip));
    std::println("connection from {}", ip);

    FileDescriptor client{client_fd};
    // handle client in a thread
}

Signal Handling

cpp

Godbolt

#include <signal.h>
#include <atomic>

// Use atomic for signal-handler communication — signal-safe
std::atomic<bool> g_running{true};

// Simple signal handler
extern "C" void on_sigint(int) {
    g_running.store(false, std::memory_order_relaxed);
}

int main() {
    struct sigaction sa{};
    sa.sa_handler = on_sigint;
    sigemptyset(&sa.sa_mask);
    sa.sa_flags = SA_RESTART;  // restart interrupted syscalls
    sigaction(SIGINT, &sa, nullptr);
    sigaction(SIGTERM, &sa, nullptr);

    while (g_running.load(std::memory_order_relaxed)) {
        // main loop
    }
    std::println("shutting down");
}

// Block signals in threads, handle in a dedicated signal thread
void signal_thread_setup() {
    sigset_t mask;
    sigfillset(&mask);
    pthread_sigmask(SIG_BLOCK, &mask, nullptr);  // block all in this thread

    // Then create a thread that calls sigwait() to handle signals
}

Pipes and IPC

cpp

Godbolt

#include <unistd.h>

// Anonymous pipe (parent-child communication)
int pipefd[2];
if (pipe(pipefd) < 0) throw std::system_error(errno, std::generic_category());

FileDescriptor read_end{pipefd[0]};
FileDescriptor write_end{pipefd[1]};

pid_t child = fork();
if (child == 0) {
    // Child: close write end, read from pipe
    write_end = {};  // closes write_end
    char buf[256];
    ssize_t n = read(read_end.get(), buf, sizeof(buf));
    // process buf[0..n]
    _exit(0);
}
// Parent: close read end, write to pipe
read_end = {};  // closes read_end
const char msg[] = "hello from parent";
write(write_end.get(), msg, sizeof(msg));
write_end = {};  // close to send EOF
waitpid(child, nullptr, 0);

epoll — Scalable I/O Multiplexing

cpp

Godbolt

#include <sys/epoll.h>

class EventLoop {
    int epfd_;
    std::unordered_map<int, std::function<void(uint32_t)>> handlers_;

public:
    EventLoop() {
        epfd_ = epoll_create1(EPOLL_CLOEXEC);
        if (epfd_ < 0) throw std::system_error(errno, std::generic_category());
    }
    ~EventLoop() { ::close(epfd_); }

    void add(int fd, uint32_t events, std::function<void(uint32_t)> cb) {
        handlers_[fd] = std::move(cb);
        epoll_event ev{};
        ev.events  = events;
        ev.data.fd = fd;
        epoll_ctl(epfd_, EPOLL_CTL_ADD, fd, &ev);
    }

    void run() {
        std::array<epoll_event, 64> events;
        while (true) {
            int n = epoll_wait(epfd_, events.data(), events.size(), -1);
            for (int i = 0; i < n; ++i) {
                int fd = events[i].data.fd;
                if (auto it = handlers_.find(fd); it != handlers_.end())
                    it->second(events[i].events);
            }
        }
    }
};

EventLoop loop;
loop.add(server_fd, EPOLLIN, [&](uint32_t events) {
    int client = accept(server_fd, nullptr, nullptr);
    loop.add(client, EPOLLIN | EPOLLET, [client](uint32_t ev) {
        char buf[4096];
        ssize_t n = read(client, buf, sizeof(buf));
        if (n <= 0) { ::close(client); return; }
        write(client, buf, n);  // echo
    });
});
loop.run();

Memory-Mapped I/O and Device Files

Hardware registers on Linux are accessed through /dev device files mapped into process memory with mmap. Unlike normal file I/O, reads and writes go directly to physical hardware addresses, so volatile qualifiers and memory barriers are essential. This pattern is common in embedded Linux, GPU drivers, and custom PCIe peripherals.

cpp

Godbolt

#include <sys/mman.h>
#include <fcntl.h>
#include <cstdint>

// Map a hardware register block (e.g., GPIO on a SoC)
class HardwareRegisterMap {
    volatile uint32_t* regs_ = nullptr;
    size_t size_;

public:
    HardwareRegisterMap(off_t phys_addr, size_t size)
        : size_{size}
    {
        // /dev/mem gives userspace access to physical memory
        int fd = open("/dev/mem", O_RDWR | O_SYNC);
        if (fd < 0) throw std::system_error(errno, std::generic_category(), "/dev/mem");

        void* p = mmap(nullptr, size, PROT_READ | PROT_WRITE,
                       MAP_SHARED, fd, phys_addr);
        ::close(fd);
        if (p == MAP_FAILED)
            throw std::system_error(errno, std::generic_category(), "mmap");
        regs_ = static_cast<volatile uint32_t*>(p);
    }

    ~HardwareRegisterMap() {
        if (regs_) munmap(const_cast<uint32_t*>(regs_), size_);
    }

    uint32_t read(size_t reg_index) const {
        return regs_[reg_index];   // volatile read — no caching
    }
    void write(size_t reg_index, uint32_t val) {
        regs_[reg_index] = val;    // volatile write — hits hardware immediately
        __sync_synchronize();      // memory barrier before next operation
    }
};

// Example: toggle GPIO pin 17 on a Raspberry Pi (BCM2837)
// GPIO base at 0x3F200000, each register is 4 bytes
HardwareRegisterMap gpio{0x3F200000, 4096};
// Set pin 17 output (GPFSEL1 register)
uint32_t sel = gpio.read(1);
sel &= ~(7u << 21);   // clear FSEL17
sel |=  (1u << 21);   // set to output
gpio.write(1, sel);
// Set pin high (GPSET0 register at offset 7)
gpio.write(7, 1u << 17);

Mechanism	Physical address	Requires root	Use case
`/dev/mem` + `mmap`	Yes	Yes	Raw SoC register access
UIO (Userspace I/O)	No (driver handles)	No (with udev rules)	Safe userspace driver
`ioctl` on char dev	No	Depends	Structured kernel interface

Netlink Sockets — Kernel/Userspace Communication

Netlink is the standard Linux mechanism for bidirectional communication between the kernel and userspace without syscall overhead. It underlies ip, ss, iproute2, and SELinux policy loading. The struct nlmsghdr header precedes every message, followed by family-specific payloads.

cpp

Godbolt

#include <sys/socket.h>
#include <linux/netlink.h>
#include <linux/rtnetlink.h>
#include <net/if.h>

// Query network interfaces using RTNETLINK
int query_interfaces() {
    int fd = socket(AF_NETLINK, SOCK_RAW, NETLINK_ROUTE);
    if (fd < 0) throw std::system_error(errno, std::generic_category());

    // Bind to our PID
    sockaddr_nl local{};
    local.nl_family = AF_NETLINK;
    local.nl_pid    = getpid();
    if (bind(fd, (sockaddr*)&local, sizeof(local)) < 0) {
        ::close(fd); throw std::system_error(errno, std::generic_category());
    }

    // Build RTM_GETLINK request
    struct {
        nlmsghdr  hdr;
        ifinfomsg ifi;
    } req{};

    req.hdr.nlmsg_len   = NLMSG_LENGTH(sizeof(ifinfomsg));
    req.hdr.nlmsg_type  = RTM_GETLINK;
    req.hdr.nlmsg_flags = NLM_F_REQUEST | NLM_F_DUMP;
    req.hdr.nlmsg_seq   = 1;
    req.hdr.nlmsg_pid   = getpid();
    req.ifi.ifi_family  = AF_UNSPEC;

    sockaddr_nl kernel{};
    kernel.nl_family = AF_NETLINK;
    send(fd, &req, req.hdr.nlmsg_len, 0);

    // Parse response
    char buf[8192];
    ssize_t n;
    while ((n = recv(fd, buf, sizeof(buf), 0)) > 0) {
        for (auto* h = (nlmsghdr*)buf;
             NLMSG_OK(h, (unsigned)n);
             h = NLMSG_NEXT(h, n))
        {
            if (h->nlmsg_type == NLMSG_DONE) goto done;
            if (h->nlmsg_type != RTM_NEWLINK)  continue;

            auto* ifi = (ifinfomsg*)NLMSG_DATA(h);
            int rta_len = IFLA_PAYLOAD(h);
            for (auto* rta = IFLA_RTA(ifi); RTA_OK(rta, rta_len);
                 rta = RTA_NEXT(rta, rta_len))
            {
                if (rta->rta_type == IFLA_IFNAME)
                    std::println("interface: {}", (char*)RTA_DATA(rta));
            }
        }
    }
done:
    ::close(fd);
    return 0;
}

Common Netlink families: NETLINK_ROUTE (network config), NETLINK_NETFILTER (iptables), NETLINK_AUDIT, NETLINK_KOBJECT_UEVENT (udev events).

eBPF from C++ with libbpf and CO-RE

eBPF lets you load verified bytecode programs into the Linux kernel at runtime to trace syscalls, instrument network packets, and profile CPU hotspots — without kernel modules. CO-RE (Compile Once, Run Everywhere) uses BTF type information so the same binary runs across kernel versions.

bash

# Install dependencies (Ubuntu/Debian)
sudo apt install libbpf-dev linux-headers-$(uname -r) clang llvm

cpp

Godbolt

// bpf_program.bpf.c — compiled to BPF bytecode
#include <linux/bpf.h>
#include <bpf/bpf_helpers.h>

struct {
    __uint(type,        BPF_MAP_TYPE_HASH);
    __uint(max_entries, 1024);
    __type(key,   __u32);   // PID
    __type(value, __u64);   // syscall count
} syscall_count SEC(".maps");

SEC("tracepoint/syscalls/sys_enter_openat")
int count_opens(void* ctx) {
    __u32 pid = bpf_get_current_pid_tgid() >> 32;
    __u64 *cnt = bpf_map_lookup_elem(&syscall_count, &pid);
    if (cnt) {
        __sync_fetch_and_add(cnt, 1);
    } else {
        __u64 one = 1;
        bpf_map_update_elem(&syscall_count, &pid, &one, BPF_ANY);
    }
    return 0;
}
char LICENSE[] SEC("license") = "GPL";

cpp

Godbolt

// userspace loader — C++ with libbpf
#include <bpf/libbpf.h>
#include <bpf/bpf.h>

struct bpf_object* obj = bpf_object__open_file("bpf_program.bpf.o", nullptr);
bpf_object__load(obj);

struct bpf_program* prog = bpf_object__find_program_by_name(obj, "count_opens");
struct bpf_link*    link = bpf_program__attach(prog);

// Read map values from userspace
int map_fd = bpf_object__find_map_fd_by_name(obj, "syscall_count");
uint32_t pid  = target_pid;
uint64_t count = 0;
bpf_map_lookup_elem(map_fd, &pid, &count);
std::println("PID {} opened {} files", pid, count);

bpf_link__destroy(link);
bpf_object__close(obj);

Process Isolation — Namespaces, cgroups, and seccomp

Linux containers are built from three kernel primitives: namespaces (what a process can see), cgroups (how much resource it can use), and seccomp (which syscalls it can make). You can apply all three from a C++ daemon without Docker.

cpp

Godbolt

#include <sched.h>
#include <sys/prctl.h>
#include <linux/seccomp.h>
#include <linux/filter.h>
#include <sys/resource.h>

// 1. Unshare namespaces — child gets its own view of PIDs, network, mounts
void isolate_namespaces() {
    if (unshare(CLONE_NEWPID | CLONE_NEWNET | CLONE_NEWUTS | CLONE_NEWNS) < 0)
        throw std::system_error(errno, std::generic_category(), "unshare");
}

// 2. Apply a simple seccomp filter using BPF — allow only specific syscalls
void apply_seccomp_allowlist() {
    // sock_filter programs: approve read/write/exit, reject everything else
    struct sock_filter filter[] = {
        BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, nr)),
        BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_read,   3, 0),
        BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_write,  2, 0),
        BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_exit,   1, 0),
        BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_KILL),   // deny
        BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ALLOW),  // allow
    };
    struct sock_fprog prog{ .len = 6, .filter = filter };

    prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0);  // required before seccomp
    if (syscall(SYS_seccomp, SECCOMP_SET_MODE_FILTER, 0, &prog) < 0)
        throw std::system_error(errno, std::generic_category(), "seccomp");
}

// 3. Limit memory via cgroup v2 (write to /sys/fs/cgroup)
void set_memory_limit_bytes(pid_t pid, size_t bytes) {
    std::string cg = "/sys/fs/cgroup/myapp_" + std::to_string(pid);
    std::filesystem::create_directories(cg);

    auto write_file = [](const std::string& path, const std::string& val) {
        std::ofstream f(path);
        f << val;
    };
    write_file(cg + "/memory.max",  std::to_string(bytes));
    write_file(cg + "/cgroup.procs", std::to_string(pid));
}

Shared Memory IPC — POSIX shm_open and Semaphores

POSIX shared memory lets unrelated processes share a region of RAM with zero copy overhead; sem_open or in-memory semaphores serialize access. This is faster than pipes or sockets for large data transfers between cooperating processes.

cpp

Godbolt

#include <sys/mman.h>
#include <fcntl.h>
#include <semaphore.h>

// Layout in shared memory
struct SharedBuffer {
    sem_t   mutex;          // in-memory POSIX semaphore
    uint32_t write_idx;
    uint32_t read_idx;
    char     data[65536];   // ring buffer
};

// Producer: create and map shared memory
class SharedProducer {
    int           fd_;
    SharedBuffer* buf_;
    static constexpr size_t SIZE = sizeof(SharedBuffer);

public:
    SharedProducer(const char* name) {
        fd_ = shm_open(name, O_CREAT | O_RDWR, 0600);
        if (fd_ < 0) throw std::system_error(errno, std::generic_category());
        ftruncate(fd_, SIZE);

        buf_ = static_cast<SharedBuffer*>(
            mmap(nullptr, SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, fd_, 0));
        if (buf_ == MAP_FAILED)
            throw std::system_error(errno, std::generic_category());

        // Initialize semaphore (pshared=1 means cross-process)
        sem_init(&buf_->mutex, /*pshared=*/1, /*value=*/1);
        buf_->write_idx = buf_->read_idx = 0;
    }

    ~SharedProducer() { munmap(buf_, SIZE); ::close(fd_); }

    void write(std::string_view msg) {
        sem_wait(&buf_->mutex);
        size_t len = std::min(msg.size(), sizeof(buf_->data) - 1);
        std::memcpy(buf_->data + buf_->write_idx, msg.data(), len);
        buf_->write_idx = (buf_->write_idx + len) % sizeof(buf_->data);
        sem_post(&buf_->mutex);
    }
};

// Consumer: attach to existing shared memory
SharedBuffer* attach_shm(const char* name) {
    int fd = shm_open(name, O_RDWR, 0);
    return static_cast<SharedBuffer*>(
        mmap(nullptr, sizeof(SharedBuffer),
             PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0));
}

Cleanup: call shm_unlink("/myshm") once when the last process exits to remove the name from /dev/shm.

Signal Handling Best Practices — signalfd and async-signal-safe

Classic signal handlers are hazardous because most C and C++ library functions are not async-signal-safe. The safe rule: only use write(), _exit(), and atomic operations inside a SA_SIGACTION handler. A better design uses signalfd to demultiplex signals as file descriptors in an event loop, keeping all logic outside handlers.

cpp

Godbolt

#include <sys/signalfd.h>
#include <signal.h>

// Convert signals to readable file descriptor events
int make_signal_fd(std::initializer_list<int> sigs) {
    sigset_t mask;
    sigemptyset(&mask);
    for (int s : sigs) sigaddset(&mask, s);

    // Block the signals so they don't invoke handlers — read them via fd instead
    pthread_sigmask(SIG_BLOCK, &mask, nullptr);

    int sfd = signalfd(-1, &mask, SFD_NONBLOCK | SFD_CLOEXEC);
    if (sfd < 0) throw std::system_error(errno, std::generic_category());
    return sfd;
}

// In the event loop (e.g., alongside epoll):
int sfd = make_signal_fd({SIGINT, SIGTERM, SIGHUP});

// When sfd becomes readable:
signalfd_siginfo si{};
read(sfd, &si, sizeof(si));
switch (si.ssi_signo) {
    case SIGINT:  std::println("SIGINT — shutting down"); break;
    case SIGTERM: std::println("SIGTERM — shutting down"); break;
    case SIGHUP:  std::println("SIGHUP — reloading config"); reload_config(); break;
}

Approach	Async-signal-safe	Can call stdlib	Use with epoll
`SA_SIGACTION` handler	Required	No	No
Atomic flag + `signalfd`	N/A	Yes	Yes
`sigwaitinfo` in dedicated thread	N/A	Yes	No

Kernel Module Interface — ioctl, Character Devices, and /proc

Kernel modules expose functionality to userspace via character device files in /dev (for ioctl/read/write) and via /proc or /sys for text-based configuration. From C++, you open the device like a regular file and call ioctl with driver-defined command codes.

cpp

Godbolt

#include <sys/ioctl.h>
#include <linux/i2c-dev.h>   // example: I2C bus driver

// Communicate with an I2C device through the kernel i2c-dev driver
class I2CBus {
    int fd_;

public:
    explicit I2CBus(int bus_num) {
        std::string path = "/dev/i2c-" + std::to_string(bus_num);
        fd_ = open(path.c_str(), O_RDWR);
        if (fd_ < 0) throw std::system_error(errno, std::generic_category(), path);
    }
    ~I2CBus() { ::close(fd_); }

    void select_device(uint8_t addr) {
        // I2C_SLAVE is a driver-defined ioctl command
        if (ioctl(fd_, I2C_SLAVE, addr) < 0)
            throw std::system_error(errno, std::generic_category(), "I2C_SLAVE");
    }

    void write_register(uint8_t reg, uint8_t val) {
        uint8_t buf[2] = {reg, val};
        if (::write(fd_, buf, 2) != 2)
            throw std::system_error(errno, std::generic_category(), "i2c write");
    }

    uint8_t read_register(uint8_t reg) {
        ::write(fd_, &reg, 1);
        uint8_t val = 0;
        if (::read(fd_, &val, 1) != 1)
            throw std::system_error(errno, std::generic_category(), "i2c read");
        return val;
    }
};

// Reading /proc files (plain text interface to kernel data)
std::string read_proc_file(const char* path) {
    std::ifstream f(path);
    return {std::istreambuf_iterator<char>(f), {}};
}

std::string meminfo = read_proc_file("/proc/meminfo");
std::string cpustat  = read_proc_file("/proc/stat");

// Enumerate loaded kernel modules
std::string modules = read_proc_file("/proc/modules");
// Each line: name size refcount ... state

The canonical pattern for a custom character device: implement file_operations in the kernel module (open, read, write, unlocked_ioctl), then interact from C++ userspace using the same POSIX calls.