(07) bench

.github/workflows/dev.yml

@@ -549,6 +549,8 @@ jobs:
         profile: "${{ fromJSON(needs.plan.outputs.profiles) }}"
         exclude:
           - profile: release
+          # Temporarily disabled: coverage/fuzz exhausts runner memory.
+          - profile: fuzz
     steps:
       - *checkout
       - *nix-setup

Cargo.toml

@@ -146,7 +146,12 @@ cloud-hypervisor-client = { version = "0.3.3", default-features = false, feature
 color-eyre = { version = "0.6.5", default-features = false, features = [] }
 colored = { version = "3.1.1", default-features = false, features = [] }
 command-fds = { version = "0.3.2", default-features = false, features = [] }
-criterion = { version = "0.8.2", default-features = false, features = [] }
+criterion = { version = "0.8.2", default-features = false, features = [
+  "cargo_bench_support",
+  "html_reports",
+  "plotters",
+] }
+iai-callgrind = { version = "0.16.1" }
 crossbeam-utils = { version = "0.8.23", default-features = false, features = [] }
 dashmap = { version = "6.2.1", default-features = false, features = [] }
 derive_builder = { version = "0.20.2", default-features = false, features = [] }

acl/Cargo.toml

@@ -27,7 +27,7 @@ clock = { workspace = true, features = ["virtual"] }
 # differential-test against it, so make it available whenever test/bench targets are built.
 dataplane-acl = { path = ".", features = ["reference"] }
 bolero = { workspace = true, features = ["std"] }
-criterion = { workspace = true, features = ["cargo_bench_support"] }
+criterion = { workspace = true }
 dpdk = { workspace = true, features = ["test"] }
 match-action = { workspace = true, features = ["derive", "bolero"] }
 net = { workspace = true, features = ["test_buffer", "builder"] }

default.nix

@@ -201,6 +201,7 @@ let
       duvet
       gateway-crd
       gettext
+      iai-callgrind-runner
       jq
       just
       kopium
@@ -219,6 +220,8 @@ let
       rust-toolchain
       shellcheck
       skopeo
+      static-web-server
+      valgrind
       wasmtime
       wget
       yq

deny.toml

@@ -34,6 +34,9 @@ ignore = [
     # vIOMMU -- so that bump is a port, not a version bump.  Ignore until it is
     # done.
     "RUSTSEC-2023-0081",
+    # bincode is unmaintained and is used only by the routing benchmark harness.
+    # Revisit this exception if it becomes a runtime dependency.
+    "RUSTSEC-2025-0141",
 ]
 
 [licenses]

development/code/README.md

@@ -9,7 +9,7 @@
 - Enforce invariants at compile time when the language or framework supports it.
 - Require runtime validation and for invariants which cannot be compile-time enforced
 - Performance is important, but it is less important than correctness; it does not matter how quickly you can do the
-  wrong thing.
+  wrong thing. See [benchmarking] for performance measurements.
 - If you are writing code that corresponds to an RFC, use [Duvet][duvet] to document compliance
 
 If you need to write a test, prefer [property-based tests] over simple unit tests.
@@ -28,4 +28,5 @@ See [testing instructions](./running-tests.md)
 [avoid-global-reasoning]: ./avoid-global-reasoning.md
 [property-based tests]: ./property-testing.md
 [duvet]: ./spec-compliance.md
+[benchmarking]: ./benchmarking.md
 [error]: ./error-handling.md

development/code/benchmarking.md

@@ -0,0 +1,37 @@
+# Benchmarking
+
+`routing/benches/fib_lookup.rs` (Criterion) and `fib_lookup_callgrind.rs` (iai-callgrind)
+use the same fixtures and lookup code.
+
+```sh
+just profile=release bench  # wall-clock time
+just bench-callgrind        # instructions and modelled cache traffic
+```
+
+The dev shell provides `valgrind` and `iai-callgrind-runner`.
+The runner version must match the `iai-callgrind` crate;
+`nix/overlays/dataplane-dev.nix` reads it from the workspace `Cargo.toml`.
+
+## Interpreting results
+
+Criterion measures elapsed time on the host; run it on a quiet machine.
+Callgrind counts instructions and modelled cache events, which help compare changes in work
+but do not predict elapsed time.
+Fewer instructions can still mean slower execution after a data-layout change.
+"Estimated Cycles" is derived from counters, not a measured cycle count.
+
+Callgrind scopes measurements to the benchmark function.
+Cachegrind counts the whole process, so its totals are not directly comparable.
+The cache geometry in `fib_lookup_callgrind.rs` is a fixed reference configuration,
+not a description of the current host.
+
+Valgrind can change DPDK's CPU dispatch.
+Confirm which implementation ran before comparing its results with native measurements.
+
+## Fixtures
+
+The shared fixture asserts that the packet matches the installed prefix.
+This prevents accidentally benchmarking a miss or the default route.
+
+Callgrind counts argument destruction inside the measured function.
+The fixture is leaked during setup so its teardown is excluded from the measurement.

justfile

@@ -41,6 +41,9 @@ kernel := if platform == "wasm32-wasip1" { "wasip1" } else { "linux" }
 # cargo build profile (debug/release/checked)
 profile := "debug"
 
+export callgrind_package := "dataplane-routing"
+export callgrind_bench := "fib_lookup_callgrind"
+
 # sanitizers to use, comma-separated (address/cfi/leak/safe-stack/shadow-stack/thread/"")
 sanitize := ""
 
@@ -305,14 +308,69 @@ fuzz target time="60s" *args="":
         {{ if sanitize == "thread" { "--build-std" } else { "" } }} \
         {{ _cargo_feature_flags }} {{ args }}
 
-# Build and run the criterion benches. The rte_acl benches are gated behind the
-# `dpdk` feature, so run `just features=dpdk bench` to exercise them; a plain
-# `just bench` builds them as empty `main()` and only runs the reference benches.
+[private]
 [script]
-bench: (build "benches")
+_bench-release-only:
+    {{ _just_debuggable_ }}
+    if [ '{{ profile }}' != "release" ]; then
+      echo "error: benchmarks want profile=release, not '{{ profile }}'" >&2
+      echo "       run: just profile=release bench" >&2
+      exit 1
+    fi
+
+[doc("Wall-clock time, via criterion")]
+[script]
+bench *args: _bench-release-only (build "benches")
     {{ _just_debuggable_ }}
     shopt -s nullglob
-    for bench in ./results/benches/bin/*; do "$bench" --bench; done
+    for bench in ./results/benches/bin/*; do
+      case "${bench}" in
+        *_callgrind) continue ;;
+      esac
+      "${bench}" --bench {{ args }}
+    done
+    if [ -f target/criterion/report/index.html ]; then
+      echo
+      echo "html report: target/criterion/report/index.html"
+    fi
+
+[doc("Instructions and cache traffic, via iai-callgrind")]
+[script]
+bench-callgrind *args:
+    {{ _just_debuggable_ }}
+    cargo bench -p "${callgrind_package}" --bench "${callgrind_bench}" {{ args }}
+
+[doc("Compare against a baseline and print a markdown report")]
+[script]
+bench-compare baseline="base" *args:
+    {{ _just_debuggable_ }}
+    mkdir -p results/bench
+    # Scope baseline detection to this suite, allowing an optional target triple.
+    # Keep benchmark failures out of the save-baseline path.
+    scope="*/${callgrind_package}/${callgrind_bench}/*"
+    if find target/iai -type f -path "${scope}" -name '*base@{{ baseline }}*' -print -quit 2>/dev/null | grep -q .; then
+      cargo bench -p "${callgrind_package}" --bench "${callgrind_bench}" -- \
+        --baseline='{{ baseline }}' --output-format=json > results/bench/run.jsonl
+    else
+      echo "no baseline '{{ baseline }}' for ${callgrind_package}/${callgrind_bench}; recording one" >&2
+      cargo bench -p "${callgrind_package}" --bench "${callgrind_bench}" -- \
+        --save-baseline='{{ baseline }}' --output-format=json > results/bench/run.jsonl
+    fi
+    ./scripts/bench-report.ts results/bench/run.jsonl {{ args }}
+
+[doc("Record a baseline for `bench-compare`, without reporting")]
+[script]
+bench-baseline name="base":
+    {{ _just_debuggable_ }}
+    cargo bench -p "${callgrind_package}" --bench "${callgrind_bench}" -- \
+      --save-baseline='{{ name }}' > /dev/null
+    echo "recorded baseline '{{ name }}'"
+
+[doc("Serve the criterion html report over http")]
+[script]
+bench-serve port="8080":
+    {{ _just_debuggable_ }}
+    just serve ./target/criterion '{{ port }}' report/index.html
 
 [script]
 build-each *args: (build "workspace" args)
@@ -1054,6 +1112,31 @@ coverage *args: (setup-roots)
     cargo llvm-cov report --branch --lcov --output-path="${out}/lcov.info"
     cargo llvm-cov report --branch --codecov --output-path="${out}/codecov.json"
     cargo llvm-cov report --branch --summary-only
+    echo
+    echo "html report: ${out}/html/index.html  (\`just serve-coverage\` to browse it)"
+
+serve_host := "127.0.0.1"
+
+[doc("Serve a directory of generated html over http")]
+[script]
+serve dir port="8080" index="index.html":
+    {{ _just_debuggable_ }}
+    if [ ! -d '{{ dir }}' ]; then
+      echo "error: no such directory: {{ dir }}" >&2
+      exit 1
+    fi
+    server="$(command -v static-web-server || true)"
+    if [ -z "${server}" ] && [ -x ./devroot/bin/static-web-server ]; then
+      server="$(pwd)/devroot/bin/static-web-server"
+    fi
+    if [ -z "${server}" ]; then
+      echo "error: static-web-server not found; re-enter the dev shell, or \`just setup-roots\`" >&2
+      exit 1
+    fi
+    echo "serving {{ dir }} at http://{{ serve_host }}:{{ port }}/{{ index }} (ctrl-c to stop)"
+    "${server}" --root '{{ dir }}' --host '{{ serve_host }}' --port '{{ port }}' --log-level warn
+
+serve-coverage port="8080": (serve "./target/nextest/coverage/html" port)
 
 [script]
 duvet *args:

k8s-intf/src/bolero/support.rs

@@ -184,6 +184,7 @@ mod test {
     const UNIQUE_COUNTS: [u16; 4] = [0, 1, 10, 16];
 
     #[test]
+    #[cfg_attr(miri, ignore = "the uniqueness sweep is too slow under miri")]
     fn test_unique_v4_interface_address_generator() {
         for count in UNIQUE_COUNTS {
             let generator = crate::bolero::support::UniqueV4InterfaceAddressGenerator::new(count);
@@ -219,6 +220,7 @@ mod test {
     }
 
     #[test]
+    #[cfg_attr(miri, ignore = "the uniqueness sweep is 124s under miri")]
     fn test_unique_v6_interface_address_generator() {
         for count in UNIQUE_COUNTS {
             let generator = crate::bolero::support::UniqueV6InterfaceAddressGenerator::new(count);

miri.just

@@ -27,7 +27,7 @@ export schedule_seed := choose('5', "0123456789")
 export seeds := "1"
 export stacked_borrow_check := env("STACKED_BORROW_CHECK", "disabled")
 
-export bolero_test_time_ms := env("BOLERO_TEST_TIME_MS", "30000")
+export bolero_test_time_ms := env("BOLERO_TEST_TIME_MS", "10000")
 export preemption_rate := "0.10"
 export weak_failure_rate := "0.05"
 export randomize_struct_layout := "enabled"

nat/src/masquerade/apalloc/pool_fuzz.rs

@@ -268,7 +268,7 @@ fn re_reservation_after_a_config_change_is_honoured() {
 //# REQ-3:  A NAT MUST NOT have a "Port assignment" behavior of "Port
 //# overloading".
 #[test]
-#[cfg_attr(miri, ignore = "exhaustive allocator walk is too slow under miri")]
+#[cfg_attr(emulated, ignore = "exhaustive allocator walk is too slow to emulate")]
 fn a_region_can_be_allocated_dry() {
     const PORTS_PER_ADDRESS: usize = 65536 - 1024;
     const ADDRESSES: usize = 2;

nat/src/masquerade/test_state_machine.rs

@@ -94,6 +94,7 @@ fn expected_tcp(action: NatAction, status: NatFlowStatus, f: Flags) -> NatFlowSt
 }
 
 #[test]
+#[cfg_attr(miri, ignore = "the full close sequence is 69s under miri")]
 fn the_tcp_state_machine_follows_the_close_sequence() {
     for action in [NatAction::SrcNat, NatAction::DstNat] {
         for status in STATUSES {