Build Time Optimization

# CMake Ninja: show build time per file
cmake -G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo ..
ninja -j4 -v 2>&1 | ts '%.s' | head -100

# Clang: time each translation unit
clang++ -ftime-trace main.cpp  # generates main.cpp.json
# Open in Chrome DevTools (chrome://tracing) or speedscope.app

# CMake 3.16+ native PCH
target_precompile_headers(myapp PRIVATE
    <vector>
    <string>
    <unordered_map>
    <memory>
    <algorithm>
    "include/common.hpp"
)

# Share PCH across targets (avoids recompiling it for each)
target_precompile_headers(myapp REUSE_FROM mylib)

// pch.hpp — include expensive headers here
#pragma once
#include <vector>
#include <string>
#include <unordered_map>
#include <memory>
#include <algorithm>
#include <functional>
// Project-wide types
#include "types.hpp"
#include "logging.hpp"

# CMake 3.16+ unity builds
set_target_properties(myapp PROPERTIES UNITY_BUILD ON)

# Fine-grained control
set_target_properties(myapp PROPERTIES
    UNITY_BUILD ON
    UNITY_BUILD_BATCH_SIZE 16   # combine up to 16 .cpp files
)

# Exclude files that have ODR or static variable issues
set_source_files_properties(src/globals.cpp PROPERTIES SKIP_UNITY_BUILD_INCLUSION ON)

// BAD: widget.hpp including everything
#include <vector>
#include <string>
#include "canvas.hpp"    // heavyweight

// GOOD: forward declare, include in .cpp only
class Canvas;           // forward declaration — no #include needed in header

class Widget {
    Canvas* canvas_;    // pointer: ok with forward declaration
    void draw(Canvas&); // reference param: ok with forward declaration
    // Canvas canvas_; // value member: needs full definition
};

// widget.cpp
#include "widget.hpp"
#include "canvas.hpp"  // full include only where needed

# Install: sudo apt install iwyu
# Run:
include-what-you-use -std=c++17 file.cpp -- -I include/

# With CMake:
set(CMAKE_CXX_INCLUDE_WHAT_YOU_USE "include-what-you-use;-Xiwyu;--no_comments")

// Explicit instantiation: instantiate once, use everywhere
// In template.hpp — declare, don't instantiate:
extern template class std::vector<MyType>;

// In template.cpp — force instantiation here only:
template class std::vector<MyType>;

// All other TUs link against this, skip generating the code

# Install
sudo apt install ccache

# CMake integration
cmake -DCMAKE_CXX_COMPILER_LAUNCHER=ccache ..

# Check stats
ccache --show-stats

# Reset
ccache --clear

// math.cppm — module interface
export module math;

export double add(double a, double b) { return a + b; }
export double square(double x) { return x * x; }

// main.cpp — import module (no header, no repeated parsing)
import math;
import std;  // std library as module (C++23)

int main() {
    std::println("{}", add(1.0, 2.0));
}

# CMake 3.28+ modules support
target_sources(myapp
    PUBLIC FILE_SET CXX_MODULES FILES
        src/math.cppm
)

# CMake: use mold linker (requires mold in PATH)
if (CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang")
    target_link_options(myapp PRIVATE -fuse-ld=mold)
endif()

# Or set globally in CMakePresets.json:
# "CMAKE_EXE_LINKER_FLAGS": "-fuse-ld=mold"

# Install on Ubuntu 22.04+:
sudo apt install mold

# Check it's being used:
ldd myapp   # should show no unusual dynamic deps

# Install (Cargo needed)
cargo install sccache

# CMake
cmake -DCMAKE_CXX_COMPILER_LAUNCHER=sccache ..

# Show stats
sccache --show-stats

# Cloud backend (CI caching across machines)
export SCCACHE_BUCKET=my-build-cache
export SCCACHE_REGION=us-east-1
cmake -DCMAKE_CXX_COMPILER_LAUNCHER=sccache ..

# Install: sudo apt install distcc
# Set up distcc daemon on build machines, then:
export DISTCC_HOSTS="buildbox1/4 buildbox2/4"
cmake -DCMAKE_CXX_COMPILER_LAUNCHER=distcc ..

# CMake: separate debug info into .dwo files (GCC/Clang)
target_compile_options(myapp PRIVATE -gsplit-dwarf)

# Or strip debug into a .debug file post-build:
add_custom_command(TARGET myapp POST_BUILD
    COMMAND ${CMAKE_OBJCOPY} --only-keep-debug $<TARGET_FILE:myapp>
            $<TARGET_FILE:myapp>.debug
    COMMAND ${CMAKE_OBJCOPY} --strip-debug $<TARGET_FILE:myapp>
    COMMAND ${CMAKE_OBJCOPY}
        --add-gnu-debuglink=$<TARGET_FILE:myapp>.debug
        $<TARGET_FILE:myapp>
)

# Ninja: show per-file elapsed time after a build
cmake -G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo -B build
cmake --build build -- -j$(nproc) 2>&1 | grep -E '^\[.*\] [0-9]+\.[0-9]+s'

# Ninja -d stats: aggregated pool/rule statistics
ninja -C build -d stats 2>&1 | tail -30

# Clang -ftime-trace: per-TU flame graph (one .json per .cpp)
cmake -DCMAKE_CXX_FLAGS="-ftime-trace" -B build-trace
cmake --build build-trace -- -j1   # single-threaded so traces don't overlap

# ClangBuildAnalyzer: merge all trace files, find the worst offenders
ClangBuildAnalyzer --start build-trace
cmake --build build-trace
ClangBuildAnalyzer --stop build-trace trace.out
ClangBuildAnalyzer --analyze trace.out
# Output: slowest files, slowest #includes, slowest template instantiations

# --- distcc (open source, simple) ---
# On each build agent:
sudo apt install distcc
distccd --daemon --allow 192.168.1.0/24 --jobs 8

# On the dev machine:
export DISTCC_HOSTS="localhost/2 agent1/8 agent2/8"
cmake -DCMAKE_CXX_COMPILER_LAUNCHER=distcc -B build
cmake --build build -- -j18   # sum of all --jobs slots + local

# --- icecc (smarter scheduling, cross-compile support) ---
# iceccd daemon on each machine; icemon GUI for cluster view
export ICECC_VERSION=gcc-14    # pre-built toolchain tarball
cmake -DCMAKE_CXX_COMPILER_LAUNCHER=icecc -B build
cmake --build build -- -j64    # icecc schedules automatically

# --- sccache with distributed mode ---
# sccache-dist server on a central host:
sccache-dist server --config /etc/sccache-dist.toml
# Client:
export SCCACHE_NO_DAEMON=0
export SCCACHE_DIST_SERVER=http://buildserver:10501
cmake -DCMAKE_CXX_COMPILER_LAUNCHER=sccache -B build

# Symbol visibility: hide everything by default (GCC/Clang)
# Reduces the linker's symbol table work significantly on shared libs
target_compile_options(mylib PRIVATE -fvisibility=hidden -fvisibility-inlines-hidden)

# Mark only the public API as visible
# In headers:
# #define MYLIB_API __attribute__((visibility("default")))
# MYLIB_API int my_function();

# Linker comparison (approximate, 500-KLOC project, debug build)
# GNU ld:   ~18 s
# gold:      ~8 s   (parallel, part of binutils)
# lld:       ~3 s   (LLVM, best for Clang projects)
# mold:      ~1 s   (fastest, parallel, modern)

# Select linker in CMake (prefer mold, fall back gracefully)
include(CheckLinkerFlag)
foreach(_ld mold lld gold)
    check_linker_flag(CXX "-fuse-ld=${_ld}" HAVE_LD_${_ld})
    if(HAVE_LD_${_ld})
        add_link_options("-fuse-ld=${_ld}")
        break()
    endif()
endforeach()

# Full LTO: slow link, best code (release builds only)
target_compile_options(myapp PRIVATE -flto=full)
target_link_options(myapp PRIVATE -flto=full)

# Thin LTO: parallel, nearly as fast as no-LTO link, ~5-10% runtime gain
target_compile_options(myapp PRIVATE -flto=thin)
target_link_options(myapp PRIVATE -flto=thin)

# Cache Thin LTO summaries to speed up incremental links
target_link_options(myapp PRIVATE -Wl,--thinlto-cache-dir=${CMAKE_BINARY_DIR}/lto-cache)

# CMakeLists.txt: enable IWYU for all targets
find_program(IWYU_PATH NAMES include-what-you-use iwyu)
if(IWYU_PATH)
    set(CMAKE_CXX_INCLUDE_WHAT_YOU_USE
        "${IWYU_PATH};-Xiwyu;--mapping_file=${CMAKE_SOURCE_DIR}/.iwyu.imp;-Xiwyu;--no_comments")
endif()

# .iwyu.imp — teach IWYU about project-specific mappings
[
  { "include": ["<bits/std_abs.h>", "private", "<cmath>", "public"] },
  { "include": ["\"detail/fwd.hpp\"",  "private", "\"mything.hpp\"", "public"] }
]

# Apply IWYU fixes automatically
cmake --build build 2>&1 | fix_include --comments < /dev/stdin

# CI enforcement: fail if any file needs changes
cmake --build build 2>&1 | grep -q "should add\|should remove" && exit 1 || exit 0

Phase 1 — Audit headers
  • Run IWYU; remove unused includes from each header
  • Add `#pragma once` everywhere; eliminate indirect-include reliance
  • Identify "leaf" headers (no project includes) — these become first modules

Phase 2 — Convert leaves to module interface units
  • Rename leaf.hpp → leaf.cppm; wrap with `export module mylib.leaf;`
  • Keep a compatibility shim leaf.hpp that `#include`s the generated header
    (CMake can generate these with configure_file)

Phase 3 — Convert mid-level headers
  • Replace `#include "leaf.hpp"` with `import mylib.leaf;` inside .cppm files
  • The shim headers keep non-module consumers working

Phase 4 — Remove shims once all consumers are converted

// leaf.cppm — converted module interface unit
module;                       // global module fragment: C headers only
#include <cstdint>            // C headers must stay in global fragment

export module mylib.leaf;
import <string>;              // importable standard headers (Clang/MSVC)

export struct LeafConfig {
    std::string name;
    uint32_t    flags = 0;
};

export LeafConfig default_config();

# CMake 3.28+: declare module sources
target_sources(mylib PUBLIC
    FILE_SET CXX_MODULES FILES
        src/leaf.cppm
        src/core.cppm
)
# CMake handles BMI dependency ordering automatically

# syntax=docker/dockerfile:1.7
FROM ubuntu:24.04 AS builder

# Layer 1: OS packages (changes rarely)
RUN apt-get update && apt-get install -y --no-install-recommends \
    build-essential cmake ninja-build ccache git \
    && rm -rf /var/lib/apt/lists/*

# Layer 2: ccache directory as a BuildKit cache mount (never busts)
RUN mkdir -p /ccache

# Layer 3: third-party dependencies (changes on conanfile/vcpkg lock change)
COPY conanfile.txt /deps/
RUN --mount=type=cache,target=/ccache \
    cd /deps && conan install . --build=missing

# Layer 4: CMakeLists only (changes on build system changes)
COPY CMakeLists.txt cmake/ /src/
RUN --mount=type=cache,target=/ccache \
    cmake -G Ninja -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
          -DCMAKE_BUILD_TYPE=Release -S /src -B /build-deps \
    && cmake --build /build-deps --target install_deps

# Layer 5: source code (changes on every commit — intentionally last)
COPY src/ /src/src/
RUN --mount=type=cache,target=/ccache \
    CCACHE_DIR=/ccache cmake --build /build-deps

# Build with BuildKit (enables --mount=type=cache)
DOCKER_BUILDKIT=1 docker build -t myapp:latest .

# Inspect ccache hit rate after a build
docker run --rm myapp:latest ccache --show-stats | grep "cache hit"

# Cold build benchmark (drop FS cache, clear ccache)
ccache --clear
sync && echo 3 | sudo tee /proc/sys/vm/drop_caches
time cmake --build build --clean-first -- -j$(nproc)

# Warm rebuild benchmark (only changed files, warm ccache)
touch src/main.cpp
time cmake --build build -- -j$(nproc)

# Null rebuild benchmark (nothing changed — measures cmake overhead)
time cmake --build build -- -j$(nproc)

# Per-TU timing via Ninja
ninja -C build -j1 2>&1 | awk '/\.cpp\.o$/{print $0}' | sort -t' ' -k2 -rn | head -20