diff --git a/Cargo.lock b/Cargo.lock index 2f68de835a..1bf2765d72 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -202,6 +202,12 @@ dependencies = [ "serde", ] +[[package]] +name = "assert_matches" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b34d609dfbaf33d6889b2b7106d3ca345eacad44200913df5ba02bfd31d2ba9" + [[package]] name = "assoc" version = "0.1.3" @@ -357,6 +363,36 @@ dependencies = [ "tower-service", ] +[[package]] +name = "aya" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66e644424fada9fff4fdc63848db1732fb69b626e8328202ef55c03df1f4d939" +dependencies = [ + "assert_matches", + "aya-obj", + "bitflags 2.13.1", + "hashbrown 0.17.1", + "libc", + "log", + "object 0.39.1", + "once_cell", + "scopeguard", + "thiserror", +] + +[[package]] +name = "aya-obj" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c76b9c75d9cdc155ff8f6a06d61e873f67bf47be8cfa92a3b5aaea43f4b4077" +dependencies = [ + "bytes", + "log", + "object 0.39.1", + "thiserror", +] + [[package]] name = "backon" version = "1.6.0" @@ -409,6 +445,8 @@ dependencies = [ "cexpr", "clang-sys", "itertools 0.13.0", + "log", + "prettyplease", "proc-macro2", "quote", "regex", @@ -892,7 +930,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -1193,6 +1231,7 @@ dependencies = [ "dataplane-stats", "dataplane-tracectl", "dataplane-vpcmap", + "dataplane-xdp", "dyn-iter", "futures", "hyper", @@ -1893,6 +1932,19 @@ dependencies = [ "thiserror", ] +[[package]] +name = "dataplane-xdp" +version = "0.27.0" +dependencies = [ + "aya", + "dataplane-concurrency", + "dataplane-net", + "nix 0.31.3", + "thiserror", + "tracing", + "xsk-rs", +] + [[package]] name = "debugid" version = "0.8.0" @@ -2202,7 +2254,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -2641,6 +2693,7 @@ version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" dependencies = [ + "equivalent", "foldhash", ] @@ -3397,6 +3450,17 @@ dependencies = [ "slab", ] +[[package]] +name = "libbpf-sys" +version = "1.5.1+v1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "912fae30b08bcbdb861d4b85bd09c05352c0ac9d7b93765ced5ca23709e7e590" +dependencies = [ + "cc", + "nix 0.30.1", + "pkg-config", +] + [[package]] name = "libc" version = "0.2.189" @@ -3437,6 +3501,17 @@ dependencies = [ "windows-link", ] +[[package]] +name = "libxdp-sys" +version = "0.2.4+1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6098c8281e42ed6f46240af889297dae1e37f70ee505dd26fe5c7199563e4d86" +dependencies = [ + "bindgen", + "cc", + "libbpf-sys", +] + [[package]] name = "linkme" version = "0.3.37" @@ -3939,6 +4014,18 @@ dependencies = [ "memoffset", ] +[[package]] +name = "nix" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" +dependencies = [ + "bitflags 2.13.1", + "cfg-if", + "cfg_aliases", + "libc", +] + [[package]] name = "nix" version = "0.31.3" @@ -3983,7 +4070,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4075,7 +4162,10 @@ version = "0.39.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e5a6c098c7a3b6547378093f5cc30bc54fd361ce711e05293a5cc589562739b" dependencies = [ + "crc32fast", "flate2", + "hashbrown 0.17.1", + "indexmap", "memchr", "ruzstd", ] @@ -4413,6 +4503,16 @@ dependencies = [ "yansi", ] +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.119", +] + [[package]] name = "proc-macro-crate" version = "2.0.2" @@ -4953,7 +5053,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -5010,7 +5110,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -5673,10 +5773,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand 2.5.0", - "getrandom 0.3.4", + "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -5686,7 +5786,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" dependencies = [ "rustix", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -6389,7 +6489,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -6593,6 +6693,19 @@ dependencies = [ "tap", ] +[[package]] +name = "xsk-rs" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0527b5447bc96b6379383108a355100a1b3b65f3699838141932cd741684a71a" +dependencies = [ + "bitflags 2.13.1", + "cfg-if", + "libc", + "libxdp-sys", + "log", +] + [[package]] name = "yansi" version = "1.0.1" diff --git a/Cargo.toml b/Cargo.toml index 1ddc29434d..456e37f7c6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,6 +42,7 @@ members = [ "tracectl", "validator", "vpcmap", + "xdp", ] resolver = "3" @@ -107,9 +108,11 @@ test-utils = { path = "./test-utils", package = "dataplane-test-utils", features tracectl = { path = "./tracectl", package = "dataplane-tracectl", features = [] } validator = { path = "./validator", package = "dataplane-validator", features = [] } vpcmap = { path = "./vpcmap", package = "dataplane-vpcmap", features = [] } +xdp = { path = "./xdp", package = "dataplane-xdp", default-features = false, features = [] } # External afpacket = { version = "0.2.3", default-features = false, features = [] } +aya = { version = "0.14.0", default-features = false, features = [] } ahash = { version = "0.8.12", default-features = false, features = [] } anyhow = { version = "1.0.104", default-features = false, features = [] } arc-swap = { version = "1.9.2", default-features = false, features = [] } @@ -223,6 +226,7 @@ tracing-test = { version = "0.2.6", default-features = false, features = [] } ureq = { version = "3.4.0", default-features = false, features = [] } url = { version = "2.5.8", default-features = false, features = [] } uuid = { version = "1.26.0", default-features = false, features = [] } +xsk-rs = { version = "0.11.0", default-features = false, features = [] } # NOTE: panic strategy is intentionally left unset here. # @@ -435,3 +439,8 @@ wasm = false # hopeless + pointless package = "dataplane-vpcmap" miri = true wasm = false # miss + +[workspace.metadata.package.xdp] +package = "dataplane-xdp" +miri = false # hopeless + pointless +wasm = false # hopeless + pointless diff --git a/README.md b/README.md index 8130a8005c..8852f68eda 100644 --- a/README.md +++ b/README.md @@ -153,6 +153,77 @@ You can override this with the `oci_repo` argument just oci_repo=my-registry.example.com:5000 push-container dataplane ``` +## Packet drivers + +The dataplane can move packets in more than one way, selected with `--driver`: + +| Driver | Notes | +| ----------- | --------------------------------------------------------------------- | +| `af-xdp` | The default. `AF_XDP` sockets; no copy where the NIC driver allows it | +| `af-packet` | `AF_PACKET` sockets; works anywhere, copies every frame | +| `dpdk` | Not wired up yet | + +`--driver kernel` is the old name for `af-packet`, and is **temporarily served +by the `AF_XDP` driver instead**: the gateway controller in the fabric repo +passes it unconditionally for interfaces named the way the kernel names them, +so it is the only name the VLAB and CI ever ask for. Pass `af-packet` to get +the `AF_PACKET` driver. This goes away once fabric can ask for a driver by +name. + +### AF_XDP + +`AF_XDP` is what the dataplane uses when it is not told otherwise, so nothing +has to be passed to get it: + +```bash +dataplane --interface eth0=kernel@eth0 --interface eth1=kernel@eth1 +``` + +The process needs `CAP_NET_RAW` to open the sockets and `CAP_BPF` to load the +XDP program that decides where packets go. Where it cannot be given those, pass +`--driver kernel` instead. + +The driver runs one worker per RX queue, each with a socket on every interface. +Zero-copy is tried first on every one of them and copy mode used where the NIC +driver will not do it, which the log says at startup. + +The kernel will not register a UMEM whose chunks are larger than a page, so a +packet above about 3.8KB does not fit in one and is carried in several. Those +are gathered into one buffer on the way in and split again on the way out, +which costs a copy each way; everything smaller is untouched. Without it a +jumbo MTU would not work at all -- the kernel refuses to attach an XDP program +that does not declare it to an interface whose MTU exceeds a buffer, and drops +what will not fit. + +### What the host still receives + +Redirecting a packet to an `AF_XDP` socket is final: the network stack never +sees it. If everything on an interface were redirected, anything running on the +host -- routing sessions, neighbour discovery -- would stop receiving, and +nothing in userspace can put a packet back on an interface's receive path. + +So the XDP program in `xdp-ebpf/` decides. It passes to the kernel what is not +IP, what is addressed to one of the host's own addresses -- which the driver +keeps it told about as they are configured and removed -- and link-local +multicast, which is how neighbours address each other: `ND`, router +advertisements, OSPF, VRRP. VXLAN is the exception: it arrives addressed to the +gateway too, and it is what the dataplane is for. Everything else goes to the +dataplane. + +Packets the pipeline asks to be delivered locally are counted as `to-kernel` in +`show driver status`. That number should stay at zero; anything else is traffic +the XDP program redirected to us that the host was expecting. + +### Building + +The build compiles libxdp from source, along with the libbpf it bundles, and +builds the XDP program for the BPF target with bpf-linker. The nix shell and +the packaged build provide both; `just build-ebpf` rebuilds the program alone +when iterating on it. + +`--no-default-features` leaves the driver out altogether, for a build without +the toolchain, and such a build has to be told to use the kernel driver. + ## Common build arguments Most just recipes accept the following arguments, which can be combined freely: diff --git a/args/src/lib.rs b/args/src/lib.rs index f11ae22070..b8ed54984f 100644 --- a/args/src/lib.rs +++ b/args/src/lib.rs @@ -375,9 +375,20 @@ pub struct GeneralConfigSection { name: Option, } +/// The packet driver used when the command line does not name one. +/// +/// `AF_XDP` needs no more of a NIC than the kernel driver does, and moves packets +/// without the copy through a socket buffer that the kernel driver pays for on +/// every frame, so it is what a dataplane that is not told otherwise should run. +pub const DEFAULT_DRIVER: &str = "af-xdp"; + /// Configuration for the packet processing driver used by the dataplane. /// -/// The dataplane supports two packet processing backends: +/// The dataplane supports three packet processing backends: +/// +/// - **`AF_XDP`**: The default. Binds `AF_XDP` sockets to kernel interfaces, which lets +/// the kernel put received packets straight into memory shared with the dataplane. +/// Where the NIC driver supports it, that happens without a copy. /// /// - **DPDK (Data Plane Development Kit)**: High-performance userspace driver for /// specialized network hardware. Provides kernel-bypass networking with direct access @@ -388,10 +399,11 @@ pub struct GeneralConfigSection { /// /// # Choosing a Driver /// +/// - Use **`AF_XDP`** unless there is a reason not to; it needs nothing of the NIC that +/// the kernel driver does not. /// - Use **DPDK** for maximum performance on supported hardware, typically in production /// environments with dedicated NICs. -/// - Use **Kernel** for development, testing, or environments without DPDK-compatible -/// hardware. +/// - Use **Kernel** where the process cannot be given the capabilities `AF_XDP` needs. #[derive( Debug, PartialEq, Eq, serde::Serialize, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, )] @@ -399,6 +411,8 @@ pub struct GeneralConfigSection { #[serde(rename_all = "snake_case")] #[rkyv(attr(derive(PartialEq, Eq, Debug)))] pub enum DriverConfigSection { + /// `AF_XDP` driver configuration + AfXdp(AfXdpDriverConfigSection), /// DPDK userspace driver configuration Dpdk(DpdkDriverConfigSection), /// Linux kernel driver configuration @@ -428,6 +442,27 @@ pub struct DpdkDriverConfigSection { pub eal_args: Vec, } +/// Configuration for the `AF_XDP` driver. +/// +/// Interfaces are named the way the kernel names them, as they are for the kernel +/// driver; the difference is in how packets are moved, not in how the devices are +/// found. +#[derive( + Debug, + PartialEq, + Eq, + serde::Serialize, + rkyv::Serialize, + rkyv::Deserialize, + rkyv::Archive, + CheckBytes, +)] +#[rkyv(attr(derive(PartialEq, Eq, Debug)))] +pub struct AfXdpDriverConfigSection { + /// Kernel network interfaces to bind sockets to + pub interfaces: Vec, +} + /// Configuration for the Linux kernel networking driver. /// /// Uses the standard Linux kernel network stack for packet processing. @@ -1105,10 +1140,8 @@ pub enum InvalidCmdArguments { /// (e.g., `eth0`, `ens3`) #[error(transparent)] InvalidInterfaceName(#[from] IllegalInterfaceName), - #[error("\"{0}\" is not a valid driver. Must be dpdk or kernel")] + #[error("\"{0}\" is not a valid driver. Must be af-xdp, af-packet or dpdk")] InvalidDriver(String), - #[error("Must specify driver as dpdk or kernel")] - NoDriverSpecified, #[error("No network interfaces specified")] NoInterfacesSpecified, #[error(transparent)] @@ -1164,13 +1197,22 @@ impl TryFrom for LaunchConfiguration { eal_args, }) } - Some(driver) if driver == "kernel" => { + Some(driver) if driver == "af-packet" || driver == "kernel" => { DriverConfigSection::Kernel(KernelDriverConfigSection { interfaces: value.interfaces().collect(), }) } + // No --driver is not an omission to complain about: it names + // the default, which is what `driver_name` resolves it to. + Some(driver) if driver == DEFAULT_DRIVER => { + DriverConfigSection::AfXdp(AfXdpDriverConfigSection { + interfaces: value.interfaces().collect(), + }) + } Some(other) => Err(InvalidCmdArguments::InvalidDriver(other.clone()))?, - None => Err(InvalidCmdArguments::NoDriverSpecified)?, + None => DriverConfigSection::AfXdp(AfXdpDriverConfigSection { + interfaces: value.interfaces().collect(), + }), }, cli: CliConfigSection { cli_sock_path: value.cli_sock_path(), @@ -1220,7 +1262,7 @@ impl TryFrom for LaunchConfiguration { #[command(about = "A dataplane for hedgehog's fabric gateway", long_about = None)] #[allow(clippy::struct_excessive_bools)] pub struct CmdArgs { - #[arg(long, value_name = "packet driver to use: kernel or dpdk")] + #[arg(long, value_name = "packet driver to use: af-xdp, af-packet or dpdk")] driver: Option, #[arg( long, @@ -1361,12 +1403,16 @@ elsewhere and copy it in the configuration directory. This mode is meant mostly impl CmdArgs { /// Get the configured driver name. /// - /// Returns `"dpdk"` if no driver was explicitly specified (the default), - /// otherwise returns the specified driver name (`"dpdk"` or `"kernel"`). + /// Returns [`DEFAULT_DRIVER`] if no driver was explicitly specified, + /// otherwise the name that was (`"af-xdp"`, `"af-packet"` or `"dpdk"`). + /// + /// `"kernel"` is the old name for the `AF_PACKET` driver and is still + /// accepted, though the dataplane currently serves it with `AF_XDP`; see + /// the note in `runtime.rs`. #[must_use] pub fn driver_name(&self) -> &str { match &self.driver { - None => "dpdk", + None => DEFAULT_DRIVER, Some(name) => name, } } @@ -1453,7 +1499,8 @@ impl CmdArgs { /// /// # Note /// - /// This is only used with the kernel driver. + /// This is used by the drivers that name interfaces the way the kernel + /// does: the kernel driver and the `AF_XDP` one. #[must_use] pub fn kernel_interfaces(&self) -> Vec { self.interface diff --git a/dataplane/Cargo.toml b/dataplane/Cargo.toml index c258c38e95..70b114daf9 100644 --- a/dataplane/Cargo.toml +++ b/dataplane/Cargo.toml @@ -6,7 +6,11 @@ publish.workspace = true version.workspace = true [features] -default = [] +default = ["af-xdp"] +# AF_XDP packet driver, and the default one. Building it needs libxdp, which +# the `dataplane-xdp` build compiles from source; turn it off for a build +# without that toolchain, which then has only the kernel driver. +af-xdp = ["dep:xdp", "xdp/runtime", "nix/fs", "nix/mount"] # Bin is gated out under loom (see `src/main.rs`); the feature exists so # `--features loom` resolves at workspace level and propagates to libs. loom = ["concurrency/loom"] @@ -53,6 +57,7 @@ tracectl = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true, default-features = true } vpcmap = { workspace = true } +xdp = { workspace = true, optional = true } [dev-dependencies] # internal diff --git a/dataplane/src/drivers/af_xdp/bpffs.rs b/dataplane/src/drivers/af_xdp/bpffs.rs new file mode 100644 index 0000000000..d69a3a3dc4 --- /dev/null +++ b/dataplane/src/drivers/af_xdp/bpffs.rs @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright Open Network Fabric Authors + +//! The BPF filesystem libxdp keeps its programs in. +//! +//! libxdp pins the dispatcher program it loads, and the count of sockets using +//! it, under `/sys/fs/bpf`. That is how it knows, when a socket goes away, +//! whether it was the last one on that interface and the program can be +//! detached. Without the filesystem it says so and carries on with a program +//! it cannot later detach, which leaves one attached to every interface the +//! dataplane ran on -- and the next run then has to displace it. +//! +//! Most systems have it mounted already. A container often does not, and the +//! dataplane runs in one, so mount it rather than asking whoever launches us +//! to. + +use std::path::Path; + +use nix::mount::{MsFlags, mount}; +use nix::sys::statfs::{BPF_FS_MAGIC, statfs}; +use tracing::{debug, info, warn}; + +/// Where the BPF filesystem is mounted, by convention and by what libxdp looks +/// for. +const MOUNT_POINT: &str = "/sys/fs/bpf"; + +/// Mount the BPF filesystem if it is not mounted already. +/// +/// Does not fail: a dataplane that cannot mount it still runs, so this reports +/// what it could not do and leaves the decision to carry on to the caller. +pub(super) fn ensure_mounted() { + let path = Path::new(MOUNT_POINT); + + match statfs(path) { + Ok(stats) if stats.filesystem_type() == BPF_FS_MAGIC => { + debug!("BPF filesystem already mounted at {MOUNT_POINT}"); + return; + } + Ok(_) => debug!("{MOUNT_POINT} exists but is not a BPF filesystem; mounting one"), + Err(e) => { + // No mount point at all. Creating one only works where /sys is + // writable, which is the same condition as the mount itself, so + // let the mount report the failure rather than pre-empting it. + debug!("Cannot stat {MOUNT_POINT} ({e}); trying to create and mount it"); + if let Err(e) = std::fs::create_dir_all(path) { + warn!("Could not create {MOUNT_POINT}: {e}"); + } + } + } + + match mount( + Some("bpf"), + path, + Some("bpf"), + MsFlags::empty(), + None::<&str>, + ) { + Ok(()) => info!("Mounted the BPF filesystem at {MOUNT_POINT}"), + Err(e) => warn!( + "Could not mount the BPF filesystem at {MOUNT_POINT}: {e}. \ + XDP programs will be left attached to the interfaces on exit, \ + and the next run will have to displace them." + ), + } +} diff --git a/dataplane/src/drivers/af_xdp/local_addrs.rs b/dataplane/src/drivers/af_xdp/local_addrs.rs new file mode 100644 index 0000000000..5d78f7bfac --- /dev/null +++ b/dataplane/src/drivers/af_xdp/local_addrs.rs @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright Open Network Fabric Authors + +//! Keeping the XDP program's idea of "our own addresses" current. +//! +//! Redirecting a packet to an `AF_XDP` socket takes it away from the kernel for +//! good, so the XDP program has to know which traffic the host is expecting +//! before it decides. What it needs is the set of addresses the host answers +//! to, and that set is not fixed: the gateway's addresses arrive with its +//! configuration, well after the driver starts, and change with it afterwards. +//! +//! So the set is read from the kernel and handed to the program repeatedly. +//! Polling rather than following netlink is a deliberate trade: the cost is a +//! bounded delay before a new address stops being redirected away from the +//! host, and what it buys is that the program's view is rebuilt from what the +//! kernel actually holds every time, rather than accumulated from a stream of +//! events that a missed message would leave permanently wrong. + +use std::net::IpAddr; + +use concurrency::sync::Arc; +use concurrency::thread; +#[allow(unused_imports)] // used under loom/shuttle backends +use concurrency::thread::BuilderExt; +use lifecycle::{CancellationToken, Subsystem}; +use tracing::{error, info}; +use xdp::program::Redirect; + +/// How often the host's addresses are read back and handed to the program. +/// +/// Also the worst case for how long traffic to a newly configured address goes +/// to the dataplane instead of the host. A session that address is for will +/// retry well past this. +const REFRESH_PERIOD: std::time::Duration = std::time::Duration::from_secs(1); + +/// Every address the host answers to, on any interface. +/// +/// Not just the interfaces the driver serves: an address on a loopback or a +/// VRF device is the host's too, and a packet carrying it arrives on a real +/// interface all the same. +fn host_addresses() -> Vec { + let mut addresses: Vec = netdev::get_interfaces() + .iter() + .flat_map(|interface| { + interface + .ipv4 + .iter() + .map(|net| IpAddr::V4(net.addr())) + .chain(interface.ipv6.iter().map(|net| IpAddr::V6(net.addr()))) + }) + .collect(); + addresses.sort_unstable(); + addresses.dedup(); + addresses +} + +/// Hand the host's addresses to `redirect`, and keep doing so until the +/// subsystem is cancelled. +/// +/// The first set is installed before this returns, so that no socket is bound +/// while the program still believes the host has no addresses of its own. +/// +/// # Errors +/// +/// Returns an error if the first set cannot be installed, or if the thread +/// that keeps them current cannot be spawned. A later refresh that fails is +/// reported and retried. +pub(super) fn track<'scope>( + scope: &'scope thread::Scope<'scope, '_>, + subsystem: &Subsystem, + redirect: &Arc, +) -> Result<(), std::io::Error> { + let mut installed = host_addresses(); + redirect.set_local_addresses(&installed)?; + info!( + "The host answers to {} address(es); traffic to them stays with the kernel", + installed.len() + ); + + let cancel = subsystem.cancel_token(); + let redirect = redirect.clone(); + let builder = thread::Builder::new().name("af-xdp-local-addrs".to_string()); + + builder.spawn_scoped(scope, move || { + refresh_until_cancelled(&redirect, &mut installed, &cancel); + })?; + + Ok(()) +} + +/// Re-read the host's addresses until cancelled, installing them when they +/// have changed. +fn refresh_until_cancelled( + redirect: &Redirect, + installed: &mut Vec, + cancel: &CancellationToken, +) { + while !cancel.is_cancelled() { + thread::sleep(REFRESH_PERIOD); + + let current = host_addresses(); + if current == *installed { + continue; + } + + match redirect.set_local_addresses(¤t) { + Ok(()) => { + info!( + "The host answers to {} address(es) now, was {}", + current.len(), + installed.len() + ); + *installed = current; + } + // Left unchanged so the next pass tries again rather than + // believing something we failed to install. + Err(e) => error!("Could not update the host's addresses: {e}"), + } + } +} diff --git a/dataplane/src/drivers/af_xdp/mod.rs b/dataplane/src/drivers/af_xdp/mod.rs new file mode 100644 index 0000000000..0b658e5c20 --- /dev/null +++ b/dataplane/src/drivers/af_xdp/mod.rs @@ -0,0 +1,204 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright Open Network Fabric Authors + +//! `AF_XDP` dataplane driver +//! +//! The kernel driver reads packets with `AF_PACKET`, which copies every frame +//! through a socket buffer. This one binds `AF_XDP` sockets instead, so the +//! kernel puts received packets straight into memory shared with the worker +//! that will process them, and takes transmitted ones from the same place. +//! +//! An `AF_XDP` socket is bound to one RX queue of one interface, so the workers +//! are organised by queue: worker `q` owns a socket on queue `q` of every +//! interface, and the queue count of the interfaces decides how many workers +//! there are. Each worker has one UMEM behind all of its sockets, which is +//! what lets it forward a packet from one interface to another without +//! copying it out of the mapping. + +#![deny( + unsafe_code, + clippy::all, + clippy::pedantic, + clippy::unwrap_used, + clippy::expect_used, + clippy::panic +)] + +mod bpffs; +mod local_addrs; +mod worker; + +use concurrency::sync::Arc; +use concurrency::thread; +#[allow(unused_imports)] // used under loom/shuttle backends +use concurrency::thread::BuilderExt; +use lifecycle::Subsystem; +use pipeline::DynPipeline; +use tracectl::trace_target; +#[allow(unused)] +use tracing::{debug, error, info, trace, warn}; +use xdp::buffer::XdpBuffer; +use xdp::program::Redirect; +use xdp::socket::BATCH_SIZE; + +use super::DriverError; +use super::kif::{self, Kif}; +use super::status::{DriverParams, DriverStatusWriter}; +use super::supervisor::{WorkerMonitor, supervise}; +use worker::Worker; + +trace_target!("af-xdp-driver", LevelFilter::INFO, &["driver"]); + +/// `AF_XDP` driver. Spawns one worker per RX queue, each with a socket on every +/// interface and a pipeline of its own. +pub struct DriverAfXdp; + +impl DriverAfXdp { + /// How often, in seconds, a worker pats its watchdog even if it sees no traffic + pub(crate) const TASK_PAT_PERIOD: u16 = 2; + + /// Slack, in seconds, on top of the pat period before a missed pat is treated as a deadline miss. + pub(crate) const TASK_GRACE_PERIOD: u16 = 4; + + /// Interval, in seconds, at which the supervisor will check worker watchdogs + pub(crate) const TASK_CHECK_PERIOD: u16 = Self::TASK_PAT_PERIOD + Self::TASK_GRACE_PERIOD; + + /// Interval, in seconds, at which the supervisor checks worker activity, ignoring watchdogs + pub(crate) const TASK_POLL_PERIOD: u16 = 1; + + /// What this driver reports about itself alongside its workers' status. + fn params() -> DriverParams { + DriverParams { + name: "af-xdp", + rx_batch: BATCH_SIZE, + poll_period: Self::TASK_POLL_PERIOD, + pat_period: Self::TASK_PAT_PERIOD, + check_period: Self::TASK_CHECK_PERIOD, + } + } + + /// How many queues every one of `interfaces` has. + /// + /// A worker serves the same queue index on all of them, so it is the + /// smallest queue count that decides how many there can be. Traffic on the + /// queues above it is left to the kernel stack rather than dropped: the + /// XDP program only redirects the queues it finds a socket for. + fn queues_to_serve(interfaces: &[Kif]) -> u32 { + let smallest = interfaces + .iter() + .map(|kif| { + kif.num_rx_queues().unwrap_or_else(|e| { + warn!( + "Could not read the queue count of '{}' ({e}); assuming one", + kif.name + ); + 1 + }) + }) + .min() + // Not reachable: an empty interface list is refused before this. A + // worker with nothing to serve would spin, so do not spawn one. + .unwrap_or(0); + + let smallest = smallest.min(xdp::program::MAX_QUEUES); + info!( + "Serving {smallest} RX queue(s) on each of {} interfaces", + interfaces.len() + ); + smallest + } + + /// Spawn one worker per queue into `scope`, each serving that queue on + /// every interface. Bails on the first spawn failure; workers that did + /// spawn drain via the scope join. + fn spawn_workers_scoped<'scope>( + scope: &'scope thread::Scope<'scope, '_>, + workers_subsystem: &Subsystem, + queues: u32, + interfaces: &[Kif], + redirect: &Arc, + setup_pipeline: &Arc DynPipeline>, + ) -> Result>, std::io::Error> { + let mut monitors: Vec = Vec::with_capacity(queues as usize); + + info!("Spawning {queues} workers"); + + for queue_id in 0..queues { + let worker = Worker::new( + queue_id as usize, + queue_id, + interfaces, + redirect, + setup_pipeline, + workers_subsystem.clone(), + ); + monitors.push(worker.start(scope)?); + } + Ok(monitors) + } + + /// Spawn worker threads + supervisor into `scope`. The scope joins + /// all driver threads on closure return. + /// + /// # Errors + /// Returns [`DriverError`] on interface setup, XDP program load, or thread + /// spawn failure. + pub fn start<'scope>( + scope: &'scope thread::Scope<'scope, '_>, + workers_subsystem: &Subsystem, + args: impl IntoIterator + Clone>, + setup_pipeline: &Arc DynPipeline>, + status_writer: DriverStatusWriter, + ) -> Result<(), DriverError> { + // libxdp pins what it loads under the BPF filesystem, and uses those + // pins to know when the last socket on an interface has gone and its + // program can be detached. Without them a program is left behind on + // every interface we ran on. + bpffs::ensure_mounted(); + + let interfaces = kif::prepare(args)?; + + // Attach whatever is going to redirect packets to our sockets before + // any of them is bound, so that no worker binds a socket the redirect + // does not know about. + let redirect = Arc::new(Redirect::attach( + interfaces.iter().map(|kif| kif.name.as_str()), + )?); + + // Before any socket is bound, so that no packet is redirected away + // from the host while the program still knows of no local address. + local_addrs::track(scope, workers_subsystem, &redirect)?; + + let queues = Self::queues_to_serve(interfaces.as_slice()); + let mut worker_monitors = Self::spawn_workers_scoped( + scope, + workers_subsystem, + queues, + interfaces.as_slice(), + &redirect, + setup_pipeline, + )?; + + let supervisor_builder = + thread::Builder::new().name("af-xdp-worker-supervisor".to_string()); + + let subsystem = workers_subsystem.clone(); + + supervisor_builder.spawn_scoped(scope, move || { + info!("Worker supervisor started"); + supervise( + Self::params(), + &subsystem, + &mut worker_monitors, + &status_writer, + ); + // The XDP programs stay attached for as long as the workers may + // still be receiving, which is until the supervisor has joined + // them all. + drop(redirect); + info!("Worker supervisor thread terminated"); + })?; + info!("AF_XDP driver started successfully"); + Ok(()) + } +} diff --git a/dataplane/src/drivers/af_xdp/worker.rs b/dataplane/src/drivers/af_xdp/worker.rs new file mode 100644 index 0000000000..e2e61a87be --- /dev/null +++ b/dataplane/src/drivers/af_xdp/worker.rs @@ -0,0 +1,424 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright Open Network Fabric Authors + +//! One worker thread, serving one RX queue of every interface. +//! +//! The worker owns a UMEM, a socket per interface bound to its queue, and a +//! pipeline of its own. Its loop is: +//! +//! 1. wait for any of its sockets to have packets, or for the poll to time out +//! 2. take what has arrived off each RX ring as `XdpBuffer`s +//! 3. run them through the pipeline +//! 4. put what comes out on the TX ring of the interface it is destined for +//! 5. hand the frames of dropped packets, and of sent ones, back to the pool, +//! and top the fill rings back up +//! +//! Nothing here is async: the poll is what the worker blocks on, and its +//! timeout is also what bounds how long the worker takes to notice that it has +//! been cancelled. + +use std::io; +use std::time::{Duration, Instant}; + +use concurrency::sync::Arc; +use concurrency::thread; +#[allow(unused_imports)] // used under loom/shuttle backends +use concurrency::thread::BuilderExt; +use lifecycle::{CancellationToken, Subsystem}; +use net::interface::InterfaceIndex; +use net::packet::{DoneReason, Packet}; +use pipeline::{DynPipeline, NetworkFunction}; +use tracing::{debug, error, info, trace, warn}; +use xdp::buffer::XdpBuffer; +use xdp::program::Redirect; +use xdp::socket::{XskConfig, XskSocket, XskUmem}; + +use crate::drivers::kif::Kif; +use crate::drivers::status::WorkerId; +use crate::drivers::supervisor::{WorkerIfaceMonitor, WorkerMonitor}; +use crate::drivers::watchdog::{RxCounters, Watchdog}; + +/// How long a pass of the loop waits for packets before going round again. +/// +/// This is also the worst-case delay between cancellation and the worker +/// noticing it, so it is short enough not to hold up shutdown. +const POLL_TIMEOUT_MS: u16 = 100; + +/// How often the kernel's own drop counts are read back. +/// +/// Reading them is a syscall, and under load a pass is one batch of packets, +/// so reading every pass would cost one syscall per batch for a number the +/// supervisor only looks at once a second. +const STATISTICS_PERIOD: Duration = Duration::from_millis(500); + +/// One interface as the worker sees it: the socket it reads and writes, and +/// the watchdog it reports that interface's traffic on. +struct WorkerInterface { + /// Interface index, which every packet received here is tagged with. + ifindex: InterfaceIndex, + /// The socket bound to this worker's queue of that interface. + socket: XskSocket, + /// Watchdog the worker pats and reports this interface's counters on. + watchdog: Watchdog, + /// Counters for the pass in progress, reported at the end of it. + counters: RxCounters, + /// The kernel's drop count as of the last read. It counts up for the life + /// of the socket rather than being cleared when it is read, so only the + /// difference belongs in this pass's counters. + kernel_drops: u64, + /// When to read that count again. + next_statistics: Instant, +} + +/// A worker, before it has a thread. +pub(super) struct Worker { + /// Which worker this is. There is one per queue, so this is the queue too, + /// but the supervisor names workers by their own index. + id: WorkerId, + /// The RX queue this worker serves on every interface. + queue_id: u32, + /// The interfaces to serve. + interfaces: Vec, + /// What redirects packets to this worker's sockets, and which has to be + /// told about each one as it is bound. + redirect: Arc, + /// Builds this worker's pipeline, on the worker's own thread. + setup_pipeline: Arc DynPipeline>, + /// The workers subsystem, for cancellation and fatal reporting. + subsystem: Subsystem, +} + +impl Worker { + pub(super) fn new( + id: WorkerId, + queue_id: u32, + interfaces: &[Kif], + redirect: &Arc, + setup_pipeline: &Arc DynPipeline>, + subsystem: Subsystem, + ) -> Self { + Self { + id, + queue_id, + interfaces: interfaces.to_vec(), + redirect: redirect.clone(), + setup_pipeline: setup_pipeline.clone(), + subsystem, + } + } + + /// Put the worker on a thread of `scope` and hand back what the supervisor + /// watches it through. + /// + /// # Errors + /// + /// Returns an error if the thread cannot be spawned. Everything the worker + /// itself sets up is reported from the worker's own thread. + pub(super) fn start<'scope>( + self, + scope: &'scope thread::Scope<'scope, '_>, + ) -> Result, io::Error> { + let Self { + id, + queue_id, + interfaces, + redirect, + setup_pipeline, + subsystem, + } = self; + + // The supervisor needs the watchdogs now, and the worker needs the + // same ones once it is running, so they are made here and shared. + let ifmonitors: Vec<_> = interfaces + .iter() + .map(|kif| WorkerIfaceMonitor::new(&kif.name)) + .collect(); + let worker_ifmonitors = ifmonitors.clone(); + + let cancel = subsystem.cancel_token(); + let thread_builder = thread::Builder::new().name(format!("dp-xdp-worker-{id}")); + + let handle = thread_builder.spawn_scoped(scope, move || { + info!(worker = id, queue = queue_id, "Worker started"); + + let mut guard = subsystem.new_exit_guard(format!("worker {id}"), true); + + let result = run( + id, + queue_id, + &interfaces, + &worker_ifmonitors, + &redirect, + &setup_pipeline, + &cancel, + ); + + if subsystem.is_cancelled() { + guard.disarm(); + } + info!(worker = id, "worker exited"); + result + })?; + + Ok(WorkerMonitor::new(id, handle, ifmonitors)) + } +} + +/// Bind this worker's sockets and run its loop until it is cancelled. +fn run( + id: WorkerId, + queue_id: u32, + interfaces: &[Kif], + ifmonitors: &[WorkerIfaceMonitor], + redirect: &Redirect, + setup_pipeline: &Arc DynPipeline>, + cancel: &CancellationToken, +) -> Result<(), io::Error> { + let config = + XskConfig::default().for_sockets(u32::try_from(interfaces.len()).unwrap_or(u32::MAX)); + + let mut umem = XskUmem::new(config).map_err(io::Error::other)?; + + // Zipped, so the table is in the order of the monitors the supervisor + // holds and a worker's counters land on the right interface. + let mut ports: Vec = Vec::with_capacity(interfaces.len()); + for (kif, monitor) in interfaces.iter().zip(ifmonitors) { + let socket = umem + .bind(&kif.name, queue_id, redirect) + .map_err(|e| io::Error::other(format!("{e}")))?; + ports.push(WorkerInterface { + ifindex: kif.ifindex, + socket, + watchdog: monitor.watchdog.clone(), + counters: RxCounters::default(), + kernel_drops: 0, + next_statistics: Instant::now(), + }); + } + + let mut pipeline = setup_pipeline(); + // Everything the loop needs room for, allocated once rather than per pass. + let mut received: Vec = Vec::with_capacity(xdp::socket::BATCH_SIZE); + let mut processed: Vec> = Vec::with_capacity(xdp::socket::BATCH_SIZE); + let mut ready: Vec = Vec::with_capacity(ports.len()); + + info!( + worker = id, + queue = queue_id, + interfaces = ports.len(), + frames = umem.total_frames(), + "Worker ready" + ); + + while !cancel.is_cancelled() { + let any = xdp::socket::wait_for_packets( + ports.iter().map(|port| &port.socket), + POLL_TIMEOUT_MS, + &mut ready, + )?; + + // Even with nothing to read the rings need servicing: the kernel may + // have finished transmitting, and the fill rings run down. + if any { + for index in 0..ports.len() { + // Skip the sockets the poll said have nothing; asking anyway + // reads a cache line the kernel owns. + if ready.get(index).copied().unwrap_or(true) { + let counters = forward_from( + id, + index, + &mut umem, + &mut ports, + &mut pipeline, + &mut received, + &mut processed, + ); + if let Some(port) = ports.get_mut(index) { + port.counters.rx += counters.rx; + port.counters.tx += counters.tx; + port.counters.ppline_drops += counters.ppline_drops; + port.counters.local += counters.local; + port.counters.tx_drops += counters.tx_drops; + port.counters.parse_errors += counters.parse_errors; + } + } + } + } + + pat_and_service(&mut umem, &mut ports); + } + + info!(worker = id, queue = queue_id, "Worker cancelled"); + Ok(()) +} + +/// Take what has arrived on one interface, run it through the pipeline, and +/// send what comes out on whichever interface it is destined for. +fn forward_from( + id: WorkerId, + index: usize, + umem: &mut XskUmem, + ports: &mut [WorkerInterface], + pipeline: &mut DynPipeline, + received: &mut Vec, + processed: &mut Vec>, +) -> RxCounters { + let mut counters = RxCounters::default(); + + let Some(port) = ports.get_mut(index) else { + return counters; + }; + + received.clear(); + let frames = port.socket.recv(umem, received) as u64; + if frames == 0 { + return counters; + } + let ifindex = port.ifindex; + + let mut parse_errors = 0u64; + let packets = received + .drain(..) + .filter_map(|buffer| match Packet::new(buffer) { + Ok(mut packet) => { + // The pipeline routes on where a packet came from, so this has + // to be set before it reaches the ingress stage. + packet.meta_mut().iif = Some(ifindex); + Some(packet) + } + Err(e) => { + parse_errors += 1; + debug!(worker = id, interface = %ifindex, "Failed to parse packet: {e}"); + None + } + }); + + // The pipeline borrows the buffers it is given, so what comes out has to + // be collected before any socket is touched again. + processed.clear(); + processed.extend(pipeline.process(packets)); + + // A frame we could not parse never became a packet, so it is counted as a + // parse error rather than as something the pipeline dropped. + let parsed = frames.saturating_sub(parse_errors); + counters.rx = parsed; + counters.parse_errors = parse_errors; + counters.ppline_drops = parsed.saturating_sub(processed.len() as u64); + + for packet in processed.drain(..) { + match packet.get_done() { + Some(DoneReason::Delivered) => { + if transmit(id, umem, ports, packet) { + counters.tx += 1; + } else { + counters.tx_drops += 1; + } + } + // The pipeline wants the host stack to have this one, and we + // cannot give it: nothing in userspace can inject into an + // interface's receive path. It has to be the XDP program that + // passes such traffic to the kernel rather than redirecting it + // here, so counting these counts what the host is missing. + Some(DoneReason::Local) => counters.local += 1, + // Anything else the pipeline is done with goes no further. + _ => {} + } + } + + counters +} + +/// Send one packet out of the interface its metadata names. Returns whether it +/// went. +fn transmit( + id: WorkerId, + umem: &mut XskUmem, + ports: &mut [WorkerInterface], + packet: Packet, +) -> bool { + let Some(oif) = packet.meta().oif else { + error!( + worker = id, + "Missing oif in packet metadata. Will drop packet (pipeline bug)" + ); + return false; + }; + + // A worker serves a handful of interfaces, so a scan beats hashing. + let Some(port) = ports.iter_mut().find(|port| port.ifindex == oif) else { + warn!(worker = id, "TX drop: unknown oif {oif} (driver bug)"); + return false; + }; + + let buffer = match packet.serialize() { + Ok(buffer) => buffer, + Err(e) => { + warn!(worker = id, "Serialize failed: {e:?}"); + return false; + } + }; + + // The serialized packet is in a UMEM frame of its own, which goes back to + // the pool when it is dropped at the end of this function; send copies it + // into the frame it puts on the ring. + match port.socket.send(umem, buffer.as_ref()) { + Ok(()) => { + trace!( + worker = id, + "TX {} bytes on interface {}", + buffer.as_ref().len(), + port.socket.if_name() + ); + true + } + Err(e) => { + debug!( + worker = id, + "TX failed on interface {}: {e}", + port.socket.if_name() + ); + false + } + } +} + +/// Report the pass that just ended, and give the kernel back what it needs: +/// the frames it has finished with, and free ones to receive into. +fn pat_and_service(umem: &mut XskUmem, ports: &mut [WorkerInterface]) { + // Frames the pipeline dropped come back over a channel, from wherever they + // were dropped, and are of no use until they are back on the free list. + umem.reclaim_dropped(); + + for port in ports.iter_mut() { + if let Err(e) = port.socket.flush_tx() { + debug!( + "Could not wake the kernel for TX on {}: {e}", + port.socket.if_name() + ); + } + port.socket.service(umem); + + // The kernel counts what it had to drop because the fill ring was + // empty; that is ours to know about, not something we ever see. It is + // a syscall, so it is read on a timer rather than every pass. + let now = Instant::now(); + if now >= port.next_statistics { + port.next_statistics = now + STATISTICS_PERIOD; + match port.socket.statistics() { + Ok(stats) => { + let dropped = stats.rx_dropped(); + port.counters.kernel_drops += dropped.saturating_sub(port.kernel_drops); + port.kernel_drops = dropped; + } + Err(e) => debug!( + "Could not read socket statistics on {}: {e}", + port.socket.if_name() + ), + } + } + + port.watchdog.pat(); + port.watchdog.record(&port.counters); + port.counters = RxCounters::default(); + } +} diff --git a/dataplane/src/drivers/kernel/mod.rs b/dataplane/src/drivers/kernel/mod.rs index 51a0072b32..9ba7ad9555 100644 --- a/dataplane/src/drivers/kernel/mod.rs +++ b/dataplane/src/drivers/kernel/mod.rs @@ -13,18 +13,13 @@ )] mod fanout; -mod kif; mod sockstats; mod worker; -use std::ops::Add; -use std::time::{Duration, Instant}; - use concurrency::sync::Arc; use concurrency::thread; #[allow(unused_imports)] // used under loom/shuttle backends use concurrency::thread::BuilderExt; -use concurrency::thread::ScopedJoinHandle; use lifecycle::Subsystem; use net::buffer::test_buffer::TestBuffer; use pipeline::DynPipeline; @@ -33,12 +28,9 @@ use tracectl::trace_target; use tracing::{debug, error, info, trace, warn}; use super::DriverError; -use super::status::{ - DriverStatus, DriverStatusWriter, RxTaskStatus, WorkerEndResult, WorkerId, WorkerState, - WorkerStatus, -}; -use super::watchdog::{Activity, Watchdog}; -use kif::{Kif, bring_kifs_up}; +use super::kif::{self, Kif}; +use super::status::{DriverParams, DriverStatusWriter}; +use super::supervisor::{WorkerMonitor, supervise}; use worker::Worker; trace_target!("kernel-driver", LevelFilter::INFO, &["driver"]); @@ -47,8 +39,6 @@ trace_target!("kernel-driver", LevelFilter::INFO, &["driver"]); /// fanout and per-worker pipelines. pub struct DriverKernel; -#[allow(clippy::cast_possible_truncation)] -#[allow(clippy::cast_precision_loss)] impl DriverKernel { /// How often, in seconds, a worker interface pats its watchdog even if no activity (worst case) pub(crate) const TASK_PAT_PERIOD: u16 = 2; @@ -65,6 +55,17 @@ impl DriverKernel { /// Max number of packets that a RX task will attempt to read in one go pub(crate) const MAX_RX_PKT_BATCH: usize = 128; + /// What this driver reports about itself alongside its workers' status. + fn params() -> DriverParams { + DriverParams { + name: "kernel", + rx_batch: Self::MAX_RX_PKT_BATCH, + poll_period: Self::TASK_POLL_PERIOD, + pat_period: Self::TASK_PAT_PERIOD, + check_period: Self::TASK_CHECK_PERIOD, + } + } + /// Spawn `num_workers` worker threads into `scope`, each with its own /// pipeline. Bails on the first spawn failure; workers that did spawn /// drain via the scope join. @@ -97,98 +98,11 @@ impl DriverKernel { Ok(monitors) } - /// Join a worker given its handle, log how the thread terminated and report the outcome in a `WorkerEndResult`. - /// This method should only be called when we know that a worker has ended (or is about to do so due to cancellation). - /// Otherwise it would block the supervisor. - fn join_worker( - id: WorkerId, - handle: ScopedJoinHandle<'_, Result<(), std::io::Error>>, - ) -> WorkerEndResult { - match handle.join() { - Ok(Ok(())) => { - info!("Worker {id} exited successfully"); - WorkerEndResult::Ok - } - Ok(Err(e)) => { - error!("Worker {id} exited with error: {e}"); - WorkerEndResult::Failed(e.to_string()) - } - Err(panic_payload) => { - let msg = format!("Worker {id} panicked {panic_payload:?}"); - error!("Worker {id} panicked: {msg}"); - WorkerEndResult::Panicked(msg) - } - } - } - - /// Check if the `Subsystem` got cancelled (and we must shutdown). If so, - /// join all of the workers. This will wait for all workers to finish. - /// Returns `true` if the subsystem was cancelled and `false` otherwise. - fn must_run( - subsystem: &Subsystem, - wk_monitors: &mut Vec>, - wk_status: &mut [WorkerStatus], - ) -> bool { - // we must run if were not cancelled - if !subsystem.is_cancelled() { - return true; - } - info!("Got cancelled. Will join worker(s)"); - for (pos, monitor) in wk_monitors.iter_mut().enumerate() { - if let Some(handle) = monitor.handle.take() { - let status = &mut wk_status[pos]; - status.state = WorkerState::Terminated(Self::join_worker(monitor.id, handle)); - } else { - info!("Not joining worker {} (ended before shutdown)", monitor.id); - } - } - info!("All workers joined. Supervisor should terminate soon..."); - false - } - - /// Check the activity of the rx tasks for a worker (watchdog, if `check_watchdog` is true) - /// from its `WorkerMonitor` and update the corresponding `WorkerStatus` - fn check_worker_rx_tasks( - monitor: &mut WorkerMonitor, - wk_status: &mut WorkerStatus, - check_watchdog: bool, - ) { - for (idx, ifm) in monitor.intf.iter().enumerate() { - let rx_task_status = &mut wk_status.rx_tasks[idx]; - debug_assert_eq!(rx_task_status.ifname, ifm.ifname); - - // check the rx task activity, and watchdog, if we've been told to do so - let (counters, activity) = ifm.watchdog.check_and_clear(check_watchdog); - - // The counters have been cleared by the read above, so accumulate them whatever - // the activity: dropping them here would lose them for good. - rx_task_status.accumulate(&counters); - - // update the rx task status - rx_task_status.activity = activity; - match rx_task_status.activity { - Activity::Stuck => { - rx_task_status.misses += 1; - rx_task_status.pps = 0.0; - error!( - "RX task for interface {} in worker {} did not pat the watchdog", - ifm.ifname, monitor.id - ); - } - Activity::Active => { - rx_task_status.pps = counters.rx as f64 / f64::from(Self::TASK_POLL_PERIOD); - } - Activity::Idle => rx_task_status.pps = 0.0, - } - } - } - /// Spawn worker threads + supervisor into `scope`. The scope joins /// all driver threads on closure return. /// /// # Errors /// Returns [`DriverError`] on interface setup or thread spawn failure. - #[allow(clippy::too_many_lines)] pub fn start<'scope>( scope: &'scope thread::Scope<'scope, '_>, workers_subsystem: &Subsystem, @@ -197,20 +111,7 @@ impl DriverKernel { setup_pipeline: &Arc DynPipeline>, status_writer: DriverStatusWriter, ) -> Result<(), DriverError> { - // A current_thread runtime built inside another tokio runtime - // panics; catch nesting in debug. - debug_assert!( - tokio::runtime::Handle::try_current().is_err(), - "DriverKernel::start must not be invoked from within a tokio runtime context" - ); - - info!("Collecting interfaces from config"); - let interfaces = kif::get_interfaces(args)?; - - tokio::runtime::Builder::new_current_thread() - .enable_all() - .build()? - .block_on(bring_kifs_up(interfaces.as_slice()))?; + let interfaces = kif::prepare(args)?; let mut worker_monitors = Self::spawn_workers_scoped( scope, @@ -227,139 +128,19 @@ impl DriverKernel { let supervisor_builder = thread::Builder::new().name("kernel-worker-supervisor".to_string()); - // the two time intervals that matter for liveness detection - let check_period = Duration::from_secs(u64::from(Self::TASK_CHECK_PERIOD)); - let poll_period = Duration::from_secs(u64::from(Self::TASK_POLL_PERIOD)); - let subsystem = workers_subsystem.clone(); supervisor_builder.spawn_scoped(scope, move || { info!("Worker supervisor started"); - - // build a vector of worker status from their monitors to expose their state outside of this thread - // each WorkerStatus contains a list of RxTaskStatus - let mut workers_status: Vec = worker_monitors - .iter() - .map(|monitor| { - let mut ws = WorkerStatus::new(monitor.id); - ws.rx_tasks = monitor - .intf - .iter() - .map(|i| RxTaskStatus::new(i.ifname.clone())) - .collect(); - ws - }) - .collect(); - - // the next instant when the rx tasks watchdogs should be checked. - let mut next_watchdog_check = Instant::now().add(check_period); - - loop { - // check if we must run. Otherwise (got cancelled) join all workers - if !Self::must_run(&subsystem, &mut worker_monitors, &mut workers_status) { - break; - } - - // check the current time and decide if we should check whether the rx tasks patted the watchdogs. - // If so, compute the next time we should check them again in the future. - let now = Instant::now(); - let check_watchdog = now >= next_watchdog_check; - if check_watchdog { - while next_watchdog_check <= now { - next_watchdog_check = next_watchdog_check.add(check_period); - } - } - - // check each worker using its monitor - let mut any_running = false; - for (pos, monitor) in worker_monitors.iter_mut().enumerate() { - // get status object for the worker/monitor - let wk_status = &mut workers_status[pos]; - - if let Some(handle) = monitor.handle.take() { - if handle.is_finished() { - // join the worker - let result = Self::join_worker(monitor.id, handle); - wk_status.state = WorkerState::Terminated(result); - wk_status.rx_tasks.iter_mut().for_each(|r| { - r.activity = Activity::Idle; - r.pps = 0.0; - }); - } else { - // worker is running, update its `WorkerStatus` and restore its handle - wk_status.state = WorkerState::Running; - monitor.handle = Some(handle); - any_running = true; - - // check the worker's rx tasks and update the corresponding status - Self::check_worker_rx_tasks(monitor, wk_status, check_watchdog); - } - } else { - // A worker monitor without a handle means that the worker was joined already - // We do nothing in this case. The monitor is kept in the list. - } - } - - if !any_running { - error!("No more workers are running!!. Stopping..."); - break; - } - - // publish the status of the driver - status_writer.publish(DriverStatus { - workers: workers_status.clone(), - }); - - // sleep for the poll period - thread::sleep(poll_period); - } - - // update status on termination. This is in case we - // want to log the last state - let last = DriverStatus { - workers: workers_status.clone(), - }; - status_writer.publish(last); - + supervise( + Self::params(), + &subsystem, + &mut worker_monitors, + &status_writer, + ); info!("Worker supervisor thread terminated"); })?; info!("Kernel driver started successfully"); Ok(()) } } - -#[derive(Clone)] -struct WorkerIfaceMonitor { - ifname: Arc, - watchdog: Watchdog, -} - -impl WorkerIfaceMonitor { - #[must_use] - fn new(ifname: &str) -> Self { - Self { - ifname: Arc::from(ifname), - watchdog: Watchdog::new(), - } - } -} - -struct WorkerMonitor<'scope> { - id: WorkerId, - handle: Option>>, - intf: Vec, -} -impl<'scope> WorkerMonitor<'scope> { - #[must_use] - fn new( - id: WorkerId, - handle: ScopedJoinHandle<'scope, Result<(), std::io::Error>>, - intf: Vec, - ) -> Self { - Self { - id, - handle: Some(handle), - intf, - } - } -} diff --git a/dataplane/src/drivers/kernel/worker.rs b/dataplane/src/drivers/kernel/worker.rs index 5e3f41b095..e992e0ac68 100644 --- a/dataplane/src/drivers/kernel/worker.rs +++ b/dataplane/src/drivers/kernel/worker.rs @@ -26,10 +26,10 @@ use pipeline::{DynPipeline, NetworkFunction}; use crate::drivers::kernel::DriverKernel; use crate::drivers::kernel::fanout::{PacketFanoutType, set_packet_fanout}; -use crate::drivers::kernel::kif::Kif; use crate::drivers::kernel::sockstats; -use crate::drivers::kernel::{WorkerIfaceMonitor, WorkerMonitor}; +use crate::drivers::kif::Kif; use crate::drivers::status::WorkerId; +use crate::drivers::supervisor::{WorkerIfaceMonitor, WorkerMonitor}; use crate::drivers::watchdog::{RxCounters, Watchdog}; use tracing::{debug, error, info, trace, warn}; @@ -219,6 +219,7 @@ impl Worker { let mut to_tx: u64 = 0; // number of packets to send let mut tx_pkts: u64 = 0; // number of packets successfully sent let mut tx_drops: u64 = 0; // number of packets dropped on tx + let mut local: u64 = 0; // number of packets left to the kernel let rx_pkts = packets_vec.len() as u64; // number of packets received counters.rx = rx_pkts; if rx_pkts == 0 { @@ -247,6 +248,11 @@ impl Worker { } else { tx_drops += 1; } + } else if done == Some(DoneReason::Local) { + // Nothing to do: the kernel has its own copy of every + // frame this driver reads. Counted so that the number + // means the same thing for every driver. + local += 1; } } @@ -258,6 +264,7 @@ impl Worker { // update rx task stats counters.ppline_drops = rx_pkts.saturating_sub(num_out_pkts); + counters.local = local; counters.tx = tx_pkts; counters.tx_drops = tx_drops; intf.watchdog.record(&counters); diff --git a/dataplane/src/drivers/kernel/kif.rs b/dataplane/src/drivers/kif.rs similarity index 69% rename from dataplane/src/drivers/kernel/kif.rs rename to dataplane/src/drivers/kif.rs index 82a5edff8a..7f38df06ba 100644 --- a/dataplane/src/drivers/kernel/kif.rs +++ b/dataplane/src/drivers/kif.rs @@ -31,6 +31,45 @@ impl Kif { debug!("Successfully created interface '{name}'"); Ok(iface) } + + /// Number of RX queues the interface exposes, as counted from the `rx-*` + /// entries under its sysfs `queues` directory. + /// + /// A driver that binds one socket per queue needs this to size its worker + /// pool. Interfaces without a `queues` directory (veth pairs on older + /// kernels, some virtual devices) are reported as having a single queue, + /// which is what the kernel gives them anyway. + /// + /// # Errors + /// + /// Returns an error if the `queues` directory exists but cannot be read. + #[cfg(feature = "af-xdp")] + pub fn num_rx_queues(&self) -> io::Result { + let queues_path = format!("/sys/class/net/{}/queues", self.name); + let path = std::path::Path::new(&queues_path); + + if !path.exists() { + tracing::warn!( + "No sysfs queues directory for '{}'; assuming a single RX queue", + self.name + ); + return Ok(1); + } + + let count = std::fs::read_dir(path)? + .filter_map(Result::ok) + .filter(|entry| entry.file_name().as_encoded_bytes().starts_with(b"rx-")) + .count(); + + // The kernel gives every netdev at least one RX queue, so a count of + // zero means we failed to see them, not that there are none. + let count = u32::try_from(count.max(1)) + .map_err(|_| io::Error::other(format!("Too many RX queues for '{}'", self.name)))?; + + debug!("Interface '{}' has {count} RX queues", self.name); + Ok(count) + } + /// Bring the kernel interface represented by a [`Kif`] up and double check it went up. async fn bring_up(&self, handle: &Handle) -> io::Result<()> { info!("Bringing interface {} up ...", self.name); @@ -164,8 +203,37 @@ pub fn get_interfaces(args: impl IntoIterator>) -> io::Re Ok(kifs) } +/// Collect the interfaces named on the command line and bring them up. +/// +/// Every driver needs both, and the second half needs a tokio runtime that the +/// drivers themselves do not otherwise have, so the runtime lives here rather +/// than being built the same way in each of them. +/// +/// # Errors +/// +/// Returns an error if an interface is not one this host has, or if it cannot +/// be brought up. +pub fn prepare(args: impl IntoIterator + Clone>) -> io::Result> { + // A current_thread runtime built inside another tokio runtime panics; + // catch nesting in debug. + debug_assert!( + tokio::runtime::Handle::try_current().is_err(), + "interfaces must not be prepared from within a tokio runtime context" + ); + + info!("Collecting interfaces from config"); + let interfaces = get_interfaces(args)?; + + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()? + .block_on(bring_kifs_up(interfaces.as_slice()))?; + + Ok(interfaces) +} + /// Bring all of the interfaces in the slice of `Kif`s up -pub async fn bring_kifs_up(kifs: &[Kif]) -> io::Result<()> { +async fn bring_kifs_up(kifs: &[Kif]) -> io::Result<()> { let (connection, handle, _) = rtnetlink::new_connection()?; let h = tokio::spawn(connection); diff --git a/dataplane/src/drivers/mod.rs b/dataplane/src/drivers/mod.rs index 4437f9060e..123e8ec80f 100644 --- a/dataplane/src/drivers/mod.rs +++ b/dataplane/src/drivers/mod.rs @@ -3,8 +3,12 @@ use thiserror::Error; +#[cfg(feature = "af-xdp")] +pub mod af_xdp; pub mod kernel; +pub(crate) mod kif; pub mod status; +pub(crate) mod supervisor; pub mod watchdog; #[derive(Error, Debug)] diff --git a/dataplane/src/drivers/status.rs b/dataplane/src/drivers/status.rs index 3e5d89d567..856cd1f314 100644 --- a/dataplane/src/drivers/status.rs +++ b/dataplane/src/drivers/status.rs @@ -13,7 +13,6 @@ use concurrency::sync::Arc; use std::fmt::Display; -use crate::drivers::kernel::DriverKernel; use crate::drivers::watchdog::{Activity, RxCounters}; // The unique Id of a worker @@ -29,6 +28,7 @@ pub struct RxTaskStatus { pub total_rx: u64, pub total_tx: u64, pub total_ppline_drops: u64, + pub total_local: u64, pub total_tx_drops: u64, pub total_parse_errors: u64, pub total_truncated: u64, @@ -46,6 +46,7 @@ impl RxTaskStatus { total_rx: 0, total_tx: 0, total_ppline_drops: 0, + total_local: 0, total_tx_drops: 0, total_parse_errors: 0, total_truncated: 0, @@ -61,6 +62,7 @@ impl RxTaskStatus { self.total_rx += counters.rx; self.total_tx += counters.tx; self.total_ppline_drops += counters.ppline_drops; + self.total_local += counters.local; self.total_tx_drops += counters.tx_drops; self.total_parse_errors += counters.parse_errors; self.total_truncated += counters.truncated; @@ -102,8 +104,27 @@ impl WorkerStatus { } } +/// What a driver is, and the periods its supervisor works to. Fixed for the +/// life of the process, but reported with the status so that a reader knows +/// which driver, and which cadence, the numbers below came from. +#[derive(Clone, Copy, Default)] +pub struct DriverParams { + /// Name of the driver, as it is given on the command line. + pub name: &'static str, + /// Most packets a worker will take from an interface in one go. + pub rx_batch: usize, + /// How often, in seconds, the supervisor samples worker activity. + pub poll_period: u16, + /// How often, in seconds, a worker pats its watchdog when it is idle. + pub pat_period: u16, + /// How long, in seconds, the supervisor waits for a pat before calling a + /// worker stuck. + pub check_period: u16, +} + #[derive(Clone, Default)] pub struct DriverStatus { + pub params: DriverParams, pub workers: Vec, } @@ -167,7 +188,7 @@ macro_rules! RX_TASK_TBL_FMT { macro_rules! RX_DROP_TBL_FMT { () => { - " {:<16} {:>12} {:>10} {:>10} {:>10} {:>10} {:>12}" + " {:<16} {:>12} {:>10} {:>10} {:>10} {:>10} {:>10} {:>12}" }; } @@ -202,6 +223,7 @@ fn fmt_rx_drop_heading(f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { RX_DROP_TBL_FMT!(), "iface", "ppline-drops", + "to-kernel", "tx-drops", "parse-err", "truncated", @@ -219,6 +241,7 @@ fn fmt_rx_drop(f: &mut std::fmt::Formatter<'_>, rx: &RxTaskStatus) -> std::fmt:: RX_DROP_TBL_FMT!(), rx.ifname, rx.total_ppline_drops, + rx.total_local, rx.total_tx_drops, rx.total_parse_errors, rx.total_truncated, @@ -231,10 +254,11 @@ fn fmt_rx_drop(f: &mut std::fmt::Formatter<'_>, rx: &RxTaskStatus) -> std::fmt:: impl Display for DriverStatus { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { Heading("Packet driver status").fmt(f)?; - writeln!(f, " max rx batch: {} pkts", DriverKernel::MAX_RX_PKT_BATCH)?; - write!(f, " activity poll: {} s", DriverKernel::TASK_POLL_PERIOD)?; - write!(f, " watchdog pat: {} s", DriverKernel::TASK_PAT_PERIOD)?; - writeln!(f, " watchdog check: {} s", DriverKernel::TASK_CHECK_PERIOD)?; + writeln!(f, " driver: {}", self.params.name)?; + writeln!(f, " max rx batch: {} pkts", self.params.rx_batch)?; + write!(f, " activity poll: {} s", self.params.poll_period)?; + write!(f, " watchdog pat: {} s", self.params.pat_period)?; + writeln!(f, " watchdog check: {} s", self.params.check_period)?; writeln!(f)?; if self.workers.is_empty() { @@ -275,6 +299,7 @@ mod test { rx: 1, tx: 2, ppline_drops: 3, + local: 9, tx_drops: 4, parse_errors: 5, truncated: 6, @@ -288,6 +313,7 @@ mod test { assert_eq!(status.total_rx, 2); assert_eq!(status.total_tx, 4); assert_eq!(status.total_ppline_drops, 6); + assert_eq!(status.total_local, 18); assert_eq!(status.total_tx_drops, 8); assert_eq!(status.total_parse_errors, 10); assert_eq!(status.total_truncated, 12); diff --git a/dataplane/src/drivers/supervisor.rs b/dataplane/src/drivers/supervisor.rs new file mode 100644 index 0000000000..ce7deec4b2 --- /dev/null +++ b/dataplane/src/drivers/supervisor.rs @@ -0,0 +1,264 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright Open Network Fabric Authors + +//! Supervision of a driver's worker threads. +//! +//! Whatever a driver does with packets, its workers are watched the same way: +//! a worker that ends is joined and reported, one whose rx tasks stop patting +//! their watchdogs is called stuck, and the counters they report are published +//! for the CLI to show. [`supervise`] is that loop, and runs on a thread of +//! the driver's own until the workers subsystem is cancelled or every worker +//! has ended. + +#![deny( + unsafe_code, + clippy::all, + clippy::pedantic, + clippy::unwrap_used, + clippy::expect_used, + clippy::panic +)] + +use std::ops::Add; +use std::time::{Duration, Instant}; + +use concurrency::sync::Arc; +use concurrency::thread; +use concurrency::thread::ScopedJoinHandle; +use lifecycle::Subsystem; +use tracing::{error, info}; + +use super::status::{ + DriverParams, DriverStatus, DriverStatusWriter, RxTaskStatus, WorkerEndResult, WorkerId, + WorkerState, WorkerStatus, +}; +use super::watchdog::{Activity, Watchdog}; + +/// What the supervisor watches one of a worker's rx tasks through. +#[derive(Clone)] +pub(crate) struct WorkerIfaceMonitor { + /// Interface the rx task reads from. + pub(crate) ifname: Arc, + /// Watchdog the rx task pats and reports its counters on. + pub(crate) watchdog: Watchdog, +} + +impl WorkerIfaceMonitor { + #[must_use] + pub(crate) fn new(ifname: &str) -> Self { + Self { + ifname: Arc::from(ifname), + watchdog: Watchdog::new(), + } + } +} + +/// What the supervisor watches one worker through: its thread, and a monitor +/// for each of the rx tasks it runs. +pub(crate) struct WorkerMonitor<'scope> { + /// Which worker this is. + pub(crate) id: WorkerId, + /// The worker's thread, taken when it is joined. + pub(crate) handle: Option>>, + /// One monitor per interface the worker reads from. + pub(crate) intf: Vec, +} + +impl<'scope> WorkerMonitor<'scope> { + #[must_use] + pub(crate) fn new( + id: WorkerId, + handle: ScopedJoinHandle<'scope, Result<(), std::io::Error>>, + intf: Vec, + ) -> Self { + Self { + id, + handle: Some(handle), + intf, + } + } +} + +/// Join a worker, log how its thread ended, and report the outcome. +/// +/// Only call this once the worker has ended, or is about to because it was +/// cancelled: otherwise it blocks the supervisor. +fn join_worker( + id: WorkerId, + handle: ScopedJoinHandle<'_, Result<(), std::io::Error>>, +) -> WorkerEndResult { + match handle.join() { + Ok(Ok(())) => { + info!("Worker {id} exited successfully"); + WorkerEndResult::Ok + } + Ok(Err(e)) => { + error!("Worker {id} exited with error: {e}"); + WorkerEndResult::Failed(e.to_string()) + } + Err(panic_payload) => { + let msg = format!("Worker {id} panicked {panic_payload:?}"); + error!("Worker {id} panicked: {msg}"); + WorkerEndResult::Panicked(msg) + } + } +} + +/// Watch `monitors` until the subsystem is cancelled or no worker is left +/// running, publishing the status of each pass through `status_writer`. +/// +/// Worker fatal reporting is not done here: each worker thread holds an +/// `ExitGuard` that reports for it. +pub(crate) fn supervise( + params: DriverParams, + subsystem: &Subsystem, + monitors: &mut [WorkerMonitor<'_>], + status_writer: &DriverStatusWriter, +) { + let check_period = Duration::from_secs(u64::from(params.check_period)); + let poll_period = Duration::from_secs(u64::from(params.poll_period)); + + // One WorkerStatus per monitor, holding one RxTaskStatus per interface. + // These are the running totals: the watchdogs are cleared as they are read. + let mut workers_status: Vec = monitors + .iter() + .map(|monitor| { + let mut status = WorkerStatus::new(monitor.id); + status.rx_tasks = monitor + .intf + .iter() + .map(|i| RxTaskStatus::new(i.ifname.clone())) + .collect(); + status + }) + .collect(); + + // When the watchdogs are next due to be checked. Activity is sampled far + // more often than that, so that the reported rate is a recent one. + let mut next_watchdog_check = Instant::now().add(check_period); + + loop { + if !must_run(subsystem, monitors, &mut workers_status) { + break; + } + + let now = Instant::now(); + let check_watchdog = now >= next_watchdog_check; + if check_watchdog { + while next_watchdog_check <= now { + next_watchdog_check = next_watchdog_check.add(check_period); + } + } + + let mut any_running = false; + for (pos, monitor) in monitors.iter_mut().enumerate() { + let Some(wk_status) = workers_status.get_mut(pos) else { + continue; + }; + + let Some(handle) = monitor.handle.take() else { + // Already joined on an earlier pass. The monitor stays in the + // list so the worker keeps its place in the report. + continue; + }; + + if handle.is_finished() { + wk_status.state = WorkerState::Terminated(join_worker(monitor.id, handle)); + for rx_task in &mut wk_status.rx_tasks { + rx_task.activity = Activity::Idle; + rx_task.pps = 0.0; + } + } else { + wk_status.state = WorkerState::Running; + monitor.handle = Some(handle); + any_running = true; + check_worker_rx_tasks(params, monitor, wk_status, check_watchdog); + } + } + + if !any_running { + error!("No more workers are running!!. Stopping..."); + break; + } + + status_writer.publish(DriverStatus { + params, + workers: workers_status.clone(), + }); + + thread::sleep(poll_period); + } + + // Publish once more on the way out, so the last state is the one a reader + // sees rather than whatever the final pass happened to leave behind. + status_writer.publish(DriverStatus { + params, + workers: workers_status, + }); +} + +/// Whether the supervisor should keep going. If the subsystem was cancelled, +/// join every worker still running first: this waits for all of them. +fn must_run( + subsystem: &Subsystem, + monitors: &mut [WorkerMonitor<'_>], + workers_status: &mut [WorkerStatus], +) -> bool { + if !subsystem.is_cancelled() { + return true; + } + + info!("Got cancelled. Will join worker(s)"); + for (pos, monitor) in monitors.iter_mut().enumerate() { + match (monitor.handle.take(), workers_status.get_mut(pos)) { + (Some(handle), Some(status)) => { + status.state = WorkerState::Terminated(join_worker(monitor.id, handle)); + } + (Some(handle), None) => { + let _ = join_worker(monitor.id, handle); + } + (None, _) => info!("Not joining worker {} (ended before shutdown)", monitor.id), + } + } + info!("All workers joined. Supervisor should terminate soon..."); + false +} + +/// Read the watchdog of each of a worker's rx tasks and fold what it reports +/// into that task's status. +#[allow(clippy::cast_precision_loss)] // packet counts stay well within f64 +fn check_worker_rx_tasks( + params: DriverParams, + monitor: &WorkerMonitor, + wk_status: &mut WorkerStatus, + check_watchdog: bool, +) { + for (idx, ifm) in monitor.intf.iter().enumerate() { + let Some(rx_task_status) = wk_status.rx_tasks.get_mut(idx) else { + continue; + }; + debug_assert_eq!(rx_task_status.ifname, ifm.ifname); + + let (counters, activity) = ifm.watchdog.check_and_clear(check_watchdog); + + // Reading the watchdog cleared it, so the counters have to be + // accumulated whatever the activity: dropping them loses them. + rx_task_status.accumulate(&counters); + + rx_task_status.activity = activity; + match rx_task_status.activity { + Activity::Stuck => { + rx_task_status.misses += 1; + rx_task_status.pps = 0.0; + error!( + "RX task for interface {} in worker {} did not pat the watchdog", + ifm.ifname, monitor.id + ); + } + Activity::Active => { + rx_task_status.pps = counters.rx as f64 / f64::from(params.poll_period); + } + Activity::Idle => rx_task_status.pps = 0.0, + } + } +} diff --git a/dataplane/src/drivers/watchdog.rs b/dataplane/src/drivers/watchdog.rs index b5941378a2..bd02dcc902 100644 --- a/dataplane/src/drivers/watchdog.rs +++ b/dataplane/src/drivers/watchdog.rs @@ -30,6 +30,7 @@ struct WatchdogInner { rx: AtomicU64, // number of packets received by task tx: AtomicU64, // number of packets sent by task ppline_drops: AtomicU64, // number of packets dropped by task pipeline + local: AtomicU64, // number of packets the pipeline sent to the kernel tx_drops: AtomicU64, // number of tx failures parse_errors: AtomicU64, // number of frames we failed to parse truncated: AtomicU64, // number of frames larger than the rx buffer @@ -58,6 +59,7 @@ impl Watchdog { accumulate(&self.0.rx, counters.rx); accumulate(&self.0.tx, counters.tx); accumulate(&self.0.ppline_drops, counters.ppline_drops); + accumulate(&self.0.local, counters.local); accumulate(&self.0.tx_drops, counters.tx_drops); accumulate(&self.0.parse_errors, counters.parse_errors); accumulate(&self.0.truncated, counters.truncated); @@ -76,6 +78,7 @@ impl Watchdog { rx: self.0.rx.swap(0, Ordering::Relaxed), tx: self.0.tx.swap(0, Ordering::Relaxed), ppline_drops: self.0.ppline_drops.swap(0, Ordering::Relaxed), + local: self.0.local.swap(0, Ordering::Relaxed), tx_drops: self.0.tx_drops.swap(0, Ordering::Relaxed), parse_errors: self.0.parse_errors.swap(0, Ordering::Relaxed), truncated: self.0.truncated.swap(0, Ordering::Relaxed), @@ -129,6 +132,14 @@ pub struct RxCounters { pub tx: u64, /// Pkts received that the pipeline dropped pub ppline_drops: u64, + /// Pkts the pipeline asked to be delivered to the kernel, which no driver + /// can do: userspace cannot inject into an interface's receive path. + /// + /// Whether that costs anything depends on how the driver reads packets. A + /// driver reading a copy the kernel also kept loses nothing. One that + /// takes packets away from the kernel loses exactly these, unless it + /// arranges for such traffic never to reach it. + pub local: u64, /// Pkts received dropped on tx pub tx_drops: u64, /// Frames received but that we failed to parse diff --git a/dataplane/src/packet_processor/mod.rs b/dataplane/src/packet_processor/mod.rs index 4e9190c5ce..5a40eab3e4 100644 --- a/dataplane/src/packet_processor/mod.rs +++ b/dataplane/src/packet_processor/mod.rs @@ -13,13 +13,14 @@ use super::packet_processor::ipforward::IpForwarder; use concurrency::sync::Arc; -use acl_filter::{AclFilter, AclFilterContextWriter}; +use acl_filter::{AclFilter, AclFilterContextReaderFactory, AclFilterContextWriter}; use flow_entry::flow_table::{FlowLookup, FlowTable}; -use flow_filter::{FlowFilter, FlowFilterContextWriter}; +use flow_filter::{FlowFilter, FlowFilterContextReaderFactory, FlowFilterContextWriter}; -use nat::masquerade::NatAllocatorWriter; -use nat::portfw::{PortForwarder, PortFwTableWriter}; +use nat::masquerade::{NatAllocatorReaderFactory, NatAllocatorWriter}; +use nat::portfw::{PortForwarder, PortFwTableReaderFactory, PortFwTableWriter}; use nat::static_nat::NatTablesWriter; +use nat::static_nat::natrw::NatTablesReaderFactory; use nat::{IcmpErrorHandler, Masquerade, StaticNat}; use net::packet::PacketStats; @@ -27,18 +28,18 @@ use net::buffer::PacketBufferMut; use pipeline::sample_nfs::{PacketDumper, PacketStatsNF}; use pipeline::{DynPipeline, PipelineData}; -use routing::{CliSources, Router, RouterError, RouterParams}; +use routing::{ + AtableReaderFactory, CliSources, FibTableReaderFactory, IfTableReaderFactory, Router, + RouterError, RouterParams, +}; use vpcmap::map::VpcMapWriter; -use stats::{Stats, StatsCollector, VpcMapName, VpcStatsStore}; +use stats::{PacketStatsWriter, Stats, StatsCollector, VpcMapName, VpcStatsStore}; -pub(crate) struct InternalSetup -where - Buf: PacketBufferMut, -{ +pub(crate) struct InternalSetup { pub router: Router, - pub pipeline: Arc DynPipeline>, + pub pipeline: Arc, pub flow_table: Arc, pub vpcmapw: VpcMapWriter, pub nattablesw: NatTablesWriter, @@ -50,12 +51,103 @@ where pub portfw_w: PortFwTableWriter, } +/// Everything a pipeline is built from, held so that one can be built again. +/// +/// Each worker runs a pipeline of its own, and the buffer type is the driver's +/// to choose: the kernel driver reads into `TestBuffer`, the `AF_XDP` driver +/// into a buffer over a UMEM frame. None of what a stage is built from depends +/// on that choice, so the factory is not generic and [`build`](Self::build) is. +pub(crate) struct PipelineFactory { + pdata: Arc, + pkt_stats: Arc, + stats_w: PacketStatsWriter, + flow_table: Arc, + iftr_factory: IfTableReaderFactory, + fibtr_factory: FibTableReaderFactory, + atabler_factory: AtableReaderFactory, + nattabler_factory: NatTablesReaderFactory, + natallocator_factory: NatAllocatorReaderFactory, + portfw_factory: PortFwTableReaderFactory, + flow_filter_reader_factory: FlowFilterContextReaderFactory, + aclfiltertablesr_factory: AclFilterContextReaderFactory, +} + +impl PipelineFactory { + /// The pipeline data shared by every pipeline built here, which management + /// reads to report on and control the stages. + pub(crate) fn data(&self) -> Arc { + self.pdata.clone() + } + + /// A factory for pipelines over `Buf`, in the shape the drivers take. + /// + /// Each worker calls it to get a pipeline of its own, so it hands out a + /// handle to this factory rather than a pipeline. + pub(crate) fn builder( + self: &Arc, + ) -> Arc DynPipeline> { + let factory = self.clone(); + Arc::new(move || factory.build()) + } + + /// Build a pipeline over `Buf`. + /// + /// The composition of the pipeline is hard-coded. Flow expiration is + /// handled by per-flow tokio timers; no `ExpirationsNF` is needed. + pub(crate) fn build(&self) -> DynPipeline { + // Build network functions + let stage_ingress = Ingress::new("Ingress", self.iftr_factory.handle()); + let stage_egress = Egress::new( + "Egress", + self.iftr_factory.handle(), + self.atabler_factory.handle(), + ); + let iprouter1 = IpForwarder::new("IP-Forward-1", self.fibtr_factory.handle()); + let iprouter2 = IpForwarder::new("IP-Forward-2", self.fibtr_factory.handle()); + let static_nat = StaticNat::with_reader("static-NAT-1", self.nattabler_factory.handle()); + let masquerade = Masquerade::new( + "masquerade", + self.flow_table.clone(), + self.natallocator_factory.handle(), + ); + let pktdump = PacketDumper::new("pipeline-end", true, None); + let stats_stage = Stats::new("stats", self.stats_w.clone()); + let flow_filter = FlowFilter::new("flow-filter", self.flow_filter_reader_factory.handle()); + let acl_filter = AclFilter::new("acl-filter", self.aclfiltertablesr_factory.handle()); + let icmp_error_handler = IcmpErrorHandler::new(self.flow_table.clone()); + let flow_lookup = FlowLookup::new("flow-lookup", self.flow_table.clone()); + let portfw = PortForwarder::new( + "port-forwarder", + self.portfw_factory.handle(), + self.flow_table.clone(), + ); + let pkt_stats_nf = PacketStatsNF::new(self.pkt_stats.clone()); + + DynPipeline::new() + .set_data(self.data()) + .add_stage(stage_ingress) + .add_stage(iprouter1) + .add_stage(icmp_error_handler) + .add_stage(flow_lookup) + .add_stage(flow_filter) + .add_stage(acl_filter) + .add_stage(static_nat) + .add_stage(portfw) + .add_stage(masquerade) + .add_stage(iprouter2) + .add_stage(stage_egress) + .add_stage(pktdump) + .add_stage(pkt_stats_nf) + .add_stage(stats_stage) + } +} + /// Start a router and provide the associated pipeline -pub(crate) fn start_router( +pub(crate) fn start_router( router: &lifecycle::Subsystem, params: RouterParams, driver_status: DriverStatusReader, -) -> Result, RouterError> { +) -> Result { let vpcmapw = VpcMapWriter::::new(); let vpc_stats_store: Arc = VpcStatsStore::new(); @@ -96,58 +188,25 @@ pub(crate) fn start_router( let fibtr_factory = router.get_fibtr_factory(); let atabler_factory = router.get_atabler_factory(); - // create pipeline builder - let flow_table_clone = flow_table.clone(); - let pipeline_builder = move || { - let pdata_clone = pdata.clone(); - - // Build network functions - let stage_ingress = Ingress::new("Ingress", iftr_factory.handle()); - let stage_egress = Egress::new("Egress", iftr_factory.handle(), atabler_factory.handle()); - let iprouter1 = IpForwarder::new("IP-Forward-1", fibtr_factory.handle()); - let iprouter2 = IpForwarder::new("IP-Forward-2", fibtr_factory.handle()); - let static_nat = StaticNat::with_reader("static-NAT-1", nattabler_factory.handle()); - let masquerade = Masquerade::new( - "masquerade", - flow_table_clone.clone(), - natallocator_factory.handle(), - ); - let pktdump = PacketDumper::new("pipeline-end", true, None); - let stats_stage = Stats::new("stats", stats_w.clone()); - let flow_filter = FlowFilter::new("flow-filter", flow_filter_reader_factory.handle()); - let acl_filter = AclFilter::new("acl-filter", aclfiltertablesr_factory.handle()); - let icmp_error_handler = IcmpErrorHandler::new(flow_table_clone.clone()); - let flow_lookup = FlowLookup::new("flow-lookup", flow_table_clone.clone()); - let portfw = PortForwarder::new( - "port-forwarder", - portfw_factory.handle(), - flow_table_clone.clone(), - ); - let pkt_stats_nf = PacketStatsNF::new(pkt_stats.clone()); - - // Build the pipeline for a router. The composition of the pipeline (in stages) is currently - // hard-coded. Flow expiration is handled by per-flow tokio timers; no ExpirationsNF needed. - DynPipeline::new() - .set_data(pdata_clone) - .add_stage(stage_ingress) - .add_stage(iprouter1) - .add_stage(icmp_error_handler) - .add_stage(flow_lookup) - .add_stage(flow_filter) - .add_stage(acl_filter) - .add_stage(static_nat) - .add_stage(portfw) - .add_stage(masquerade) - .add_stage(iprouter2) - .add_stage(stage_egress) - .add_stage(pktdump) - .add_stage(pkt_stats_nf) - .add_stage(stats_stage) + // create the pipeline factory + let pipeline = PipelineFactory { + pdata, + pkt_stats, + stats_w, + flow_table: flow_table.clone(), + iftr_factory, + fibtr_factory, + atabler_factory, + nattabler_factory, + natallocator_factory, + portfw_factory, + flow_filter_reader_factory, + aclfiltertablesr_factory, }; Ok(InternalSetup { router, - pipeline: Arc::new(pipeline_builder), + pipeline: Arc::new(pipeline), flow_table, vpcmapw, nattablesw, diff --git a/dataplane/src/runtime.rs b/dataplane/src/runtime.rs index f132a6895c..90a67c59aa 100644 --- a/dataplane/src/runtime.rs +++ b/dataplane/src/runtime.rs @@ -5,12 +5,15 @@ use crate::packet_processor::start_router; use crate::statistics::spawn_metrics; use args::{CmdArgs, Parser}; +#[cfg(feature = "af-xdp")] +use crate::drivers::af_xdp::DriverAfXdp; use crate::drivers::kernel::DriverKernel; use crate::drivers::status::driver_status_access; use lifecycle::{ CancellationToken, DpSignal, Shutdown, default_deadlines, spawn_shutdown_watchdog, }; use mgmt::{ConfigProcessorParams, LaunchError, MgmtParams, run_mgmt}; +use net::buffer::test_buffer::TestBuffer; use nix::unistd::gethostname; use pyroscope::backend::{BackendConfig, PprofConfig, pprof_backend}; @@ -20,7 +23,7 @@ use tracectl::{ TracingControl, TracingRateLimitConfig, custom_target, get_trace_ctl, trace_target, }; -use tracing::{error, info, level_filters::LevelFilter}; +use tracing::{error, info, level_filters::LevelFilter, warn}; use concurrency::sync::Arc; use config::internal::routing::bmp::BmpOptions; @@ -296,7 +299,7 @@ pub fn main() { setup.stats, ); - let pipeline_factory = setup.pipeline; + let pipeline = setup.pipeline.clone(); concurrency::thread::scope(|scope| { let mgmt_result = run_mgmt( @@ -308,7 +311,7 @@ pub fn main() { interfaces: args.interfaces().map(|i| i.interface).collect(), processor_params: ConfigProcessorParams { router_ctl: setup.router.get_ctl_tx(), - pipeline_data: pipeline_factory().get_data(), + pipeline_data: pipeline.data(), flow_table: setup.flow_table, vpcmapw: setup.vpcmapw, nattablesw: setup.nattablesw, @@ -327,22 +330,68 @@ pub fn main() { Ok(()) => { info!("Management is running now"); - let driver_result = match args.driver_name() { + // TEMPORARY: the gateway controller in the fabric repo works out + // the driver from whether the interfaces are named by PCI + // address or by kernel name, and passes --driver kernel for the + // latter whatever else is true (pkg/ctrl/gateway_ctrl.go). That + // is the only name the VLAB and CI ever ask for, so asking for + // AF_XDP there means changing two other repositories first. + // Serve that name from the AF_XDP driver until they can ask for + // what they want; --driver af-packet still selects the + // AF_PACKET one, and this goes away once fabric stops deciding + // for us. + let requested = args.driver_name(); + let driver = if requested == "kernel" && cfg!(feature = "af-xdp") { + warn!( + "Serving --driver kernel with the AF_XDP driver. Pass --driver \ + af-packet for the AF_PACKET one." + ); + "af-xdp" + } else { + requested + }; + + let driver_result = match driver { "dpdk" => { info!("Using driver DPDK..."); todo!(); } - "kernel" => { - info!("Using driver kernel..."); + // "kernel" only reaches here in a build without the AF_XDP + // driver, where it is the only thing it can mean. + "af-packet" | "kernel" => { + info!("Using driver AF_PACKET..."); + let factory = pipeline.builder::(); Some(DriverKernel::start( scope, &shutdown.workers, args.kernel_interfaces(), args.kernel_num_workers(), - &pipeline_factory, + &factory, driver_status_writer, )) } + #[cfg(feature = "af-xdp")] + "af-xdp" => { + info!("Using driver AF_XDP..."); + let factory = pipeline.builder::(); + Some(DriverAfXdp::start( + scope, + &shutdown.workers, + args.kernel_interfaces(), + &factory, + driver_status_writer, + )) + } + #[cfg(not(feature = "af-xdp"))] + "af-xdp" => { + error!( + "This dataplane was built without the af-xdp feature, so it \ + cannot run the driver it would otherwise default to. Rebuild \ + with it, or pass --driver kernel. Stopping dataplane..." + ); + shutdown.fail(); + None + } other => { error!("Unknown driver '{other}'. Stopping dataplane..."); shutdown.fail(); diff --git a/default.nix b/default.nix index 4f8964ec67..e787bb916d 100644 --- a/default.nix +++ b/default.nix @@ -111,6 +111,14 @@ let fancy.numactl.static fancy.rdma-core.dev fancy.rdma-core.static + # libelf and zlib are what libbpf, which libxdp bundles, links + # against; the AF_XDP driver needs both. Only the static libraries + # go in, as for everything else here: the dataplane ships in an + # image with no shared libraries of its own to find. + fancy.elfutils.dev + fancy.elfutils.static + pkgs.zlib.dev + pkgs.zlib.static ]; } else @@ -175,6 +183,8 @@ let kopium llvmPackages'.clang # you need the host compiler in order to link proc macros llvmPackages'.llvm # needed for coverage + fancy.bpf-linker # links the XDP program of the AF_XDP driver + m4 # libxdp's build preprocesses its dispatcher program with it markdownlint-cli2 nixfmt npins @@ -216,6 +226,15 @@ let # Rust's pkg-config crate refuses cross-target builds by default; opt in # since our PKG_CONFIG_PATH already points at the matching cross sysroot. PKG_CONFIG_ALLOW_CROSS = "1"; + # libxdp compiles BPF programs of its own, and the wrapped compiler adds + # hardening flags that clang rejects for the bpf target. Point that one + # build at the compiler underneath the wrapper; everything else keeps + # using `clang` from the PATH, wrapper and all. + CLANG = "${pkgs.pkgsBuildBuild.llvmPackages'.clang.cc}/bin/clang"; + # So that a build in the shell finds an XDP program without anyone having + # to build one first. Point this at your own to iterate on it; `just + # build-ebpf` says where it leaves the one it builds. + DATAPLANE_XDP_EBPF = "${xdp-ebpf}"; }; }; # Nix escaping made the old regexes match unrelated .sh and .patch files. @@ -309,6 +328,7 @@ let pkgs.stdenv'.targetPlatform.rust.rustcTarget; is-cross-compile = pkgs.stdenv'.buildPlatform.rust.rustcTarget != ctarget; cxx = if is-cross-compile then "${ctarget}-clang++" else "clang++"; + bintools = pkgs.pkgsBuildHost.binutils; strip = if is-cross-compile then "${ctarget}-strip" else "strip"; objcopy = if is-cross-compile then "${ctarget}-objcopy" else "objcopy"; package-list = builtins.fromJSON ( @@ -335,6 +355,66 @@ let ) ); version = (craneLib.crateNameFromCargoToml { inherit src; }).version; + + # The XDP program of the AF_XDP driver, built for the BPF target. + # + # It cannot come from the workspace build: the target has no prebuilt std, so + # core is compiled from source, and the object is linked by bpf-linker rather + # than by the linker everything else uses. It is a derivation of its own, and + # the dataplane build is handed the result through DATAPLANE_XDP_EBPF. + xdp-ebpf-src = pkgs.lib.cleanSourceWith { + name = "xdp-ebpf-source"; + src = lib.cleanSource ./xdp-ebpf; + # `target` is whatever a local `just build-ebpf` left behind, and + # including it would change the derivation every time someone built here. + filter = full-path: _type: baseNameOf full-path != "target"; + }; + xdp-ebpf-vendor = craneLib.vendorMultipleCargoDeps { + cargoLockList = [ + ./xdp-ebpf/Cargo.lock + "${pkgs.rust-toolchain.passthru.availableComponents.rust-src}/lib/rustlib/src/rust/library/Cargo.lock" + ]; + }; + xdp-ebpf = pkgs.stdenvNoCC.mkDerivation { + pname = "dataplane-xdp-ebpf"; + inherit version; + src = xdp-ebpf-src; + nativeBuildInputs = [ + pkgs.pkgsBuildHost.rust-toolchain + pkgs.pkgsBuildHost.fancy.bpf-linker + ]; + configurePhase = '' + runHook preConfigure + export CARGO_HOME="$NIX_BUILD_TOP/cargo-home" + mkdir -p "$CARGO_HOME" + cp ${xdp-ebpf-vendor}/config.toml "$CARGO_HOME/config.toml" + runHook postConfigure + ''; + buildPhase = '' + runHook preBuild + cargo build \ + --offline \ + --release \ + --target bpfel-unknown-none \ + -Zbuild-std=core + runHook postBuild + ''; + installPhase = '' + runHook preInstall + install -Dm444 \ + target/bpfel-unknown-none/release/dataplane-xdp-ebpf \ + "$out" + runHook postInstall + ''; + env = { + # -Zbuild-std is a nightly option, and the toolchain is stable. + RUSTC_BOOTSTRAP = "1"; + }; + # The output is a BPF object, and the fixup phase's tools only understand + # host ELF; left to try, they warn about every one of them. + dontPatchELF = true; + dontStrip = true; + }; # The `loom` and `shuttle` features require `panic = "unwind"` (see # nix/profiles.nix), as do test builds. The sysroot needs the matching # panic runtime and std feature, so we build two cargo command prefixes: @@ -367,6 +447,16 @@ let else [ ] ); + # libxdp wraps each of the BPF programs it compiles in an object of the host + # architecture, with `ld -r -b binary`, and reads LD and OBJCOPY to know which + # tools to do it with. A cross build leaves both naming the build host's, + # which produce an object that the final link, for the target, then refuses. + # Exported here rather than set in `env` because the toolchain's setup hooks + # overwrite both before any of the build's own phases run. + target-binutils = '' + export LD=${bintools}/bin/${bintools.targetPrefix}ld + export OBJCOPY=${bintools}/bin/${bintools.targetPrefix}objcopy + ''; cargo-cmd-prefix = mk-cargo-cmd-prefix needs-unwind; cargo-cmd-prefix-tests = mk-cargo-cmd-prefix needs-unwind-tests; invoke = @@ -406,7 +496,13 @@ let cargo-nextest llvmPackages'.clang llvmPackages'.lld + pkgs.pkgsBuildHost.m4 # libxdp's build preprocesses its dispatcher program with it pkg-config + # As for the dev shell above: under a cross pkgs the one callPackage + # hands us is named for the target, and libbpf's build looks for it + # unprefixed. PKG_CONFIG_PATH below already points at the sysroot of + # the target, so the unprefixed one answers for it. + pkgs.pkgsBuildBuild.pkg-config ]; buildInputs = [ @@ -420,6 +516,17 @@ let CARGO_PROFILE = cargo-profile; DATAPLANE_SYSROOT = "${sysroot}"; LIBCLANG_PATH = "${pkgs.pkgsBuildHost.llvmPackages'.libclang.lib}/lib"; + # libxdp compiles BPF programs of its own, and the wrapped compiler + # adds hardening flags that clang rejects for the bpf target. Point + # that one build at the compiler underneath the wrapper. + CLANG = "${pkgs.pkgsBuildHost.llvmPackages'.clang.cc}/bin/clang"; + # libbpf is compiled by gcc, which on aarch64 calls out to libgcc for + # its atomics, while the link is clang and lld against a sysroot that + # has no libgcc in it. Have it emit the instructions inline instead. + LIBBPF_SYS_EXTRA_CFLAGS = lib.optionalString pkgs.stdenv'.hostPlatform.isAarch64 "-mno-outline-atomics"; + # The XDP program of the AF_XDP driver, built for the BPF target by a + # derivation of its own. + DATAPLANE_XDP_EBPF = "${xdp-ebpf}"; C_INCLUDE_PATH = "${sysroot}/include"; LIBRARY_PATH = "${sysroot}/lib"; PKG_CONFIG_PATH = "${sysroot}/lib/pkgconfig"; @@ -450,6 +557,7 @@ let { # Only dependency builds should retain crane's target archive. doInstallCargoArtifacts = for-deps; + preBuild = target-binutils + (orig.preBuild or ""); postBuild = (orig.postBuild or "") + '' unset RUSTFLAGS; ''; @@ -457,6 +565,7 @@ let else { separateDebugInfo = true; + preBuild = target-binutils + (orig.preBuild or ""); # I'm not 100% sure if I would call it a bug in crane or a bug in cargo, but cross compile is tricky here. # There is no easy way to distinguish RUSTFLAGS intended for the build-time dependencies from the RUSTFLAGS @@ -1213,6 +1322,7 @@ in sysroot tests workspace + xdp-ebpf ; profile = profile'; platform = platform'; diff --git a/justfile b/justfile index b96bf7b534..df29b16c3c 100644 --- a/justfile +++ b/justfile @@ -1045,6 +1045,23 @@ bump_version version: sed -i "s/^version = \".*\"/version = \"${new_version}\"/" Cargo.toml cargo update --workspace +# Build the XDP program of the AF_XDP driver +# +# Only needed to iterate on the program itself: the packaged build compiles it +# in a derivation of its own, and the dev shell points DATAPLANE_XDP_EBPF at +# the result. The BPF target has no prebuilt std, so core is built from source, +# which RUSTC_BOOTSTRAP lets the stable toolchain do. bpf-linker links it. +[script] +build-ebpf *args: + {{ _just_debuggable_ }} + declare profile="debug" + [[ "{{ args }}" == *--release* ]] && profile="release" + cd xdp-ebpf + RUSTC_BOOTSTRAP=1 cargo build --target bpfel-unknown-none -Zbuild-std=core {{ args }} + declare -r object="$(pwd)/target/bpfel-unknown-none/${profile}/dataplane-xdp-ebpf" + echo "Built ${object}" + echo "Build the dataplane against it with DATAPLANE_XDP_EBPF=${object}" + # Enter nix-shell [script] shell: diff --git a/nat/src/masquerade/mod.rs b/nat/src/masquerade/mod.rs index d7b2861dba..93ab694c4b 100644 --- a/nat/src/masquerade/mod.rs +++ b/nat/src/masquerade/mod.rs @@ -15,7 +15,7 @@ mod test; // re exports pub use allocator_writer::MasqueradeConfig; -pub use allocator_writer::NatAllocatorWriter; +pub use allocator_writer::{NatAllocatorReaderFactory, NatAllocatorWriter}; pub use nf::Masquerade; use tracectl::trace_target; diff --git a/nix/overlays/dataplane.nix b/nix/overlays/dataplane.nix index 76a5fca2ce..0869b9144d 100644 --- a/nix/overlays/dataplane.nix +++ b/nix/overlays/dataplane.nix @@ -278,6 +278,49 @@ in ''; }); + # libelf, from elfutils, is what libbpf reads BPF object files with. libxdp + # bundles libbpf and the `dataplane-xdp` build compiles both from source, so + # we need the headers as well as the library. debuginfod would drag in curl + # and is of no use to us here. + fancy.elfutils = (dataplane-dep prev.elfutils).overrideAttrs (orig: { + outputs = (orig.outputs or [ "out" ]) ++ [ "static" ]; + configureFlags = (orig.configureFlags or [ ]) ++ [ + "--enable-static" + "--disable-debuginfod" + # Compressed ELF sections are elfutils' business, not libbpf's, and + # supporting them drags three more libraries into a static link. + "--without-bzlib" + "--without-lzmalib" + "--without-zstd" + ]; + # elfutils' own test suite does not survive our stdenv and flags, and none + # of what it covers is on the path we use. + doCheck = false; + doInstallCheck = false; + # Keep the archives out of the default output, so that a sysroot holding + # only this one leaves the linker nothing but the static library to pick. + # + # elfutils carries a CRC-32 of its own, under the name zlib gives its own, + # and a link that statically takes both is refused for defining `crc32` + # twice. elfutils' copy is reached only through the GNU debuglink helpers + # in libdw, which we do not link, so drop it and leave zlib's standing. + postInstall = (orig.postInstall or "") + '' + mkdir -p "$static/lib" + mv $out/lib/*.a $static/lib/ + # $AR rather than ar: a cross build has only the prefixed one, and it is + # the one that reads the archive we have just built anyway. + "''${AR:-ar}" d "$static/lib/libelf.a" crc32.o + ''; + }); + + # bpf-linker is what turns the XDP program of the AF_XDP driver into a BPF + # object. It reads the bitcode rustc emits, so it has to be built against the + # same LLVM rustc uses -- which is what llvmPackages' is, derived from rustc + # itself. The nixpkgs default is the LLVM of the nixpkgs rustc, and ours comes + # from a rust overlay, so without this it rejects our bitcode as being from a + # newer LLVM than it can read. + fancy.bpf-linker = prev.bpf-linker.override { llvmPackagesForLinker = final.llvmPackages'; }; + # This isn't directly required by dataplane, fancy.perftest = dataplane-dep (final.callPackage ../pkgs/perftest { src = sources.perftest; }); } diff --git a/routing/src/lib.rs b/routing/src/lib.rs index 4c72b937fc..15c300c69f 100644 --- a/routing/src/lib.rs +++ b/routing/src/lib.rs @@ -27,17 +27,17 @@ mod router; mod routingdb; // re-exports -pub use atable::atablerw::AtableReader; +pub use atable::atablerw::{AtableReader, AtableReaderFactory}; pub use config::RouterConfig; pub use errors::RouterError; pub use evpn::Vtep; pub use fib::fibobjects::{EgressObject, FibEntry, PktInstruction}; -pub use fib::fibtable::FibTableReader; +pub use fib::fibtable::{FibTableReader, FibTableReaderFactory}; pub use fib::fibtype::FibKey; pub use frr::frrmi::FrrAppliedConfig; pub use frr::renderer::builder::Render; pub use interfaces::iftable::IfTable; -pub use interfaces::iftablerw::IfTableReader; +pub use interfaces::iftablerw::{IfTableReader, IfTableReaderFactory}; pub use interfaces::interface::{AttachConfig, Attachment, RouterInterfaceConfig}; pub use interfaces::interface::{IfDataEthernet, IfState, IfType, Interface}; pub use rib::encapsulation::{Encapsulation, VxlanEncapsulation}; diff --git a/xdp-ebpf/Cargo.lock b/xdp-ebpf/Cargo.lock new file mode 100644 index 0000000000..48efdbc567 --- /dev/null +++ b/xdp-ebpf/Cargo.lock @@ -0,0 +1,290 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "aya-build" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "609b414caa508f04d2dbbc9481ad6869f43ab9ab1dc9abd237fca27c0818f663" +dependencies = [ + "anyhow", + "cargo_metadata", + "rustc_version", + "which", +] + +[[package]] +name = "aya-ebpf" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eace5970943931fea46a8cbdca208a8f6a9e0413d85a5a6c4e168f0b909570bc" +dependencies = [ + "aya-build", + "aya-ebpf-bindings", + "aya-ebpf-cty", + "aya-ebpf-macros", + "rustversion", +] + +[[package]] +name = "aya-ebpf-bindings" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf1f19286ebbda6673099b68d049034838f08e4939cef66c998a8d3e62825d76" +dependencies = [ + "aya-build", + "aya-ebpf-cty", +] + +[[package]] +name = "aya-ebpf-cty" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0884264bc8a51a49fed03df8feada3a21d703cf501dae33c28bdba5d2313ef33" +dependencies = [ + "aya-build", +] + +[[package]] +name = "aya-ebpf-macros" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23e3a7f86e914df106dd00f9d757bc93f7d5c93380c51eead23d8d2ccd15a87a" +dependencies = [ + "proc-macro2", + "proc-macro2-diagnostics", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "camino" +version = "1.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb1307f12aa967b5a58416e87b3653360e0fd614a016b6e970db08fecbb1b80d" +dependencies = [ + "serde_core", +] + +[[package]] +name = "cargo-platform" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd0061da739915fae12ea00e16397555ed4371a6bb285431aab930f61b0aa4ba" +dependencies = [ + "serde", + "serde_core", +] + +[[package]] +name = "cargo_metadata" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef987d17b0a113becdd19d3d0022d04d7ef41f9efe4f3fb63ac44ba61df3ade9" +dependencies = [ + "camino", + "cargo-platform", + "semver", + "serde", + "serde_json", + "thiserror", +] + +[[package]] +name = "dataplane-xdp-ebpf" +version = "0.27.0" +dependencies = [ + "aya-ebpf", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "proc-macro2-diagnostics" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "version_check", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" +dependencies = [ + "serde", + "serde_core", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "which" +version = "8.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bae2f2b2b816647a1cab1acc91f5bd20812d53cb344382635ec2181940c8034f" +dependencies = [ + "libc", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/xdp-ebpf/Cargo.toml b/xdp-ebpf/Cargo.toml new file mode 100644 index 0000000000..18adcb6157 --- /dev/null +++ b/xdp-ebpf/Cargo.toml @@ -0,0 +1,45 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright Open Network Fabric Authors + +# Deliberately not a workspace member: this crate is built for the BPF target, +# by `just build-ebpf`, with a profile of its own. +[workspace] + +[package] +name = "dataplane-xdp-ebpf" +version = "0.27.0" +edition = "2024" +license = "Apache-2.0" +publish = false + +[dependencies] +aya-ebpf = "0.2.1" + +[[bin]] +name = "dataplane-xdp-ebpf" +path = "src/main.rs" + +# The verifier is stricter than a CPU, and several of the things rustc emits +# for a debug build are things it rejects: the null and alignment checks behind +# debug-assertions land after the program's exit, and it will not load a +# program whose last instruction is a call. Nothing here is a preference; each +# line is something the program will not load without. +[profile.dev] +opt-level = 3 +debug = false +debug-assertions = false +overflow-checks = false +lto = true +panic = "abort" +incremental = false +codegen-units = 1 + +[profile.release] +opt-level = 3 +debug = false +debug-assertions = false +overflow-checks = false +lto = true +panic = "abort" +incremental = false +codegen-units = 1 diff --git a/xdp-ebpf/src/main.rs b/xdp-ebpf/src/main.rs new file mode 100644 index 0000000000..5d6a00f2cf --- /dev/null +++ b/xdp-ebpf/src/main.rs @@ -0,0 +1,239 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright Open Network Fabric Authors + +//! The XDP program the AF_XDP driver attaches to each of its interfaces. +//! +//! It decides, for every packet, whether the dataplane sees it or the kernel +//! does. Redirecting to an AF_XDP socket is final -- the packet never enters +//! the network stack -- so anything the host itself needs has to be recognised +//! here and passed on instead. +//! +//! What the host needs is what the pipeline would have marked `Local`: traffic +//! addressed to the gateway that is not carrying an overlay. Userspace keeps +//! the gateway's own addresses in [`LOCAL_IPV4`] and [`LOCAL_IPV6`], and this +//! program passes anything destined to one of them. VXLAN is the exception: +//! it arrives addressed to the gateway too, and it is exactly what the +//! dataplane exists to handle. +//! +//! Anything that is not IP -- ARP, LLDP -- goes to the kernel as well. The +//! pipeline has nothing to do with those and drops them, and the host needs +//! them to resolve neighbours and be discovered. +//! +//! So does link-local multicast, which is how the protocols that run between +//! neighbours address each other: IPv6 neighbour discovery, router +//! advertisements, OSPF, VRRP. None of it is addressed to the gateway, so +//! nothing above would recognise it, and all of it is the host's. +//! +//! Everything else, which is the traffic the dataplane forwards, is redirected +//! to the socket bound to the queue it arrived on. + +#![no_std] +#![no_main] + +use aya_ebpf::bindings::xdp_action; +use aya_ebpf::macros::{map, xdp}; +use aya_ebpf::maps::{HashMap, XskMap}; +use aya_ebpf::programs::XdpContext; +use core::mem::size_of; + +/// The AF_XDP sockets, indexed by RX queue. +/// +/// Sized for the largest queue count we expect to serve on one interface; +/// queues beyond it are simply not redirected. The userspace side clamps to +/// the same number, which it keeps as `xdp::program::MAX_QUEUES` -- this crate +/// is `no_std` and built for another target, so it cannot share the constant. +#[map] +static XSKMAP: XskMap = XskMap::with_max_entries(64, 0); + +/// The gateway's own IPv4 addresses, as they appear in a packet header. +/// +/// Userspace owns the contents. An empty map means we know of no local +/// address, and everything IP goes to the dataplane. +#[map] +static LOCAL_IPV4: HashMap = HashMap::with_max_entries(MAX_LOCAL_ADDRESSES, 0); + +/// The gateway's own IPv6 addresses, as they appear in a packet header. +#[map] +static LOCAL_IPV6: HashMap<[u8; 16], u8> = HashMap::with_max_entries(MAX_LOCAL_ADDRESSES, 0); + +/// How many addresses of each family the maps hold. +pub const MAX_LOCAL_ADDRESSES: u32 = 1024; + +/// UDP port VXLAN arrives on. Matches `net::vxlan::Vxlan::PORT`. +const VXLAN_PORT: u16 = 4789; + +// Ethernet, in the shapes this program needs. Kept here rather than taken from +// `net`, which is a std crate and cannot be built for this target. +const ETH_HDR_LEN: usize = 14; +const ETH_P_IPV4: u16 = 0x0800; +const ETH_P_IPV6: u16 = 0x86dd; +const ETH_P_VLAN: u16 = 0x8100; +const ETH_P_QINQ: u16 = 0x88a8; +const VLAN_HDR_LEN: usize = 4; +/// How many stacked VLAN tags to look past. Two covers QinQ. +const MAX_VLAN_TAGS: usize = 2; + +const IPPROTO_UDP: u8 = 17; + +/// The IPv4 link-local multicast block, 224.0.0.0/24, as it appears in a +/// header. Everything a host says to its neighbours is in here. +const IPV4_LINK_LOCAL_MULTICAST: [u8; 3] = [224, 0, 0]; +/// The IPv4 limited broadcast address, 255.255.255.255. +const IPV4_BROADCAST: [u8; 4] = [255, 255, 255, 255]; +/// The first byte of any IPv6 multicast address, and the second byte of the +/// link-local scope: ff02::/16. +const IPV6_LINK_LOCAL_MULTICAST: [u8; 2] = [0xff, 0x02]; +const IPV4_MIN_HDR_LEN: usize = 20; +const IPV6_HDR_LEN: usize = 40; + +/// Read a `T` at `offset` from the start of the packet, if the packet is long +/// enough to hold one there. +/// +/// Every read of packet data has to be proven in-bounds against `data_end`, or +/// the verifier rejects the program. +#[inline(always)] +fn read_at(ctx: &XdpContext, offset: usize) -> Option { + let start = ctx.data(); + let end = ctx.data_end(); + // Checked as offset-from-start so that the comparison the verifier sees is + // the one that bounds the read. + if start + offset + size_of::() > end { + return None; + } + // SAFETY: the bounds check above puts the whole of T within the packet, + // and the kernel guarantees the packet is readable for the life of the + // call. The read is unaligned because packet data has no alignment. + Some(unsafe { ((start + offset) as *const T).read_unaligned() }) +} + +/// Decide where a packet goes, and send it there. +/// +/// `frags` says the program copes with a packet that arrives in more than one +/// buffer, which is what the kernel requires before it will attach one to an +/// interface whose MTU is larger than a buffer. It costs nothing here: the +/// headers this reads are all in the first buffer, which is the only part +/// `data`..`data_end` covers either way. +#[xdp(frags)] +pub fn xdp_redirect(ctx: XdpContext) -> u32 { + if is_for_the_host(&ctx) { + return xdp_action::XDP_PASS; + } + + // SAFETY: the kernel hands the program a valid context for the life of + // the call, and `rx_queue_index` is part of it. + let queue_id = unsafe { (*ctx.ctx).rx_queue_index }; + + // A queue with no socket in the map is not an error: the packet belongs + // to the kernel stack. + XSKMAP.redirect(queue_id, 0).unwrap_or(xdp_action::XDP_PASS) +} + +/// Whether the kernel, rather than the dataplane, should have this packet. +#[inline(always)] +fn is_for_the_host(ctx: &XdpContext) -> bool { + let Some((ethertype, l3_offset)) = l3_header(ctx) else { + // Too short to classify. The dataplane would not make sense of it + // either, and the kernel counts what it drops. + return true; + }; + + match ethertype { + ETH_P_IPV4 => ipv4_is_for_the_host(ctx, l3_offset), + ETH_P_IPV6 => ipv6_is_for_the_host(ctx, l3_offset), + // ARP, LLDP, and anything else the pipeline has no use for. + _ => true, + } +} + +/// The ethertype of the packet and the offset of the header it names, looking +/// past any VLAN tags. +#[inline(always)] +fn l3_header(ctx: &XdpContext) -> Option<(u16, usize)> { + let mut ethertype = u16::from_be(read_at::(ctx, 12)?); + let mut offset = ETH_HDR_LEN; + + // Bounded so the verifier can unroll it; QinQ is as deep as we look. + for _ in 0..MAX_VLAN_TAGS { + if ethertype != ETH_P_VLAN && ethertype != ETH_P_QINQ { + break; + } + ethertype = u16::from_be(read_at::(ctx, offset + 2)?); + offset += VLAN_HDR_LEN; + } + + Some((ethertype, offset)) +} + +/// Whether an IPv4 packet is one the host should receive. +#[inline(always)] +fn ipv4_is_for_the_host(ctx: &XdpContext, offset: usize) -> bool { + // The header is read as its parts rather than a struct, so that each read + // carries its own bounds check. + let Some(version_ihl) = read_at::(ctx, offset) else { + return true; + }; + let Some(protocol) = read_at::(ctx, offset + 9) else { + return true; + }; + // The destination address, exactly as it sits in the header. + let Some(destination) = read_at::(ctx, offset + 16) else { + return true; + }; + + let header_len = ((version_ihl & 0x0f) as usize) * 4; + if header_len < IPV4_MIN_HDR_LEN { + return true; + } + + if protocol == IPPROTO_UDP && is_vxlan(ctx, offset + header_len) { + return false; + } + + let octets = destination.to_ne_bytes(); + if octets[..3] == IPV4_LINK_LOCAL_MULTICAST || octets == IPV4_BROADCAST { + return true; + } + + // SAFETY: the map is not created with BPF_F_NO_PREALLOC, and userspace + // only ever replaces whole entries, so a concurrent update cannot alias + // this one into something else. + unsafe { LOCAL_IPV4.get(destination).is_some() } +} + +/// Whether an IPv6 packet is one the host should receive. +#[inline(always)] +fn ipv6_is_for_the_host(ctx: &XdpContext, offset: usize) -> bool { + let Some(next_header) = read_at::(ctx, offset + 6) else { + return true; + }; + let Some(destination) = read_at::<[u8; 16]>(ctx, offset + 24) else { + return true; + }; + + // Only the first header is looked at: VXLAN over IPv6 with extension + // headers in front is not something we produce. + if next_header == IPPROTO_UDP && is_vxlan(ctx, offset + IPV6_HDR_LEN) { + return false; + } + + if destination[..2] == IPV6_LINK_LOCAL_MULTICAST { + return true; + } + + // SAFETY: see `ipv4_is_for_the_host`. + unsafe { LOCAL_IPV6.get(destination).is_some() } +} + +/// Whether the UDP header at `offset` is carrying VXLAN. +#[inline(always)] +fn is_vxlan(ctx: &XdpContext, offset: usize) -> bool { + read_at::(ctx, offset + 2).is_some_and(|port| u16::from_be(port) == VXLAN_PORT) +} + +#[cfg(not(test))] +#[panic_handler] +fn panic(_info: &core::panic::PanicInfo) -> ! { + // Nothing here can panic, and the profile builds without the checks that + // would reach this. It exists because a no_std binary must have one. + loop {} +} diff --git a/xdp/Cargo.toml b/xdp/Cargo.toml new file mode 100644 index 0000000000..43dd5d1cb6 --- /dev/null +++ b/xdp/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "dataplane-xdp" +edition.workspace = true +license.workspace = true +publish.workspace = true +version.workspace = true + +[features] +default = [] +# Sockets, rings, and the XDP program that feeds them. Off by default because +# it pulls in libxdp, which the build compiles from source and which needs +# libelf, zlib and a clang that can target BPF, and the eBPF object built by +# `just build-ebpf`. +runtime = ["dep:aya", "dep:nix", "dep:xsk-rs"] + +[dependencies] +# internal +concurrency = { workspace = true } +net = { workspace = true } + +# external +aya = { workspace = true, optional = true } +nix = { workspace = true, optional = true, features = ["poll"] } +thiserror = { workspace = true } +tracing = { workspace = true } +xsk-rs = { workspace = true, optional = true, features = ["use_cc_build"] } diff --git a/xdp/build.rs b/xdp/build.rs new file mode 100644 index 0000000000..42dfda4c45 --- /dev/null +++ b/xdp/build.rs @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright Open Network Fabric Authors + +//! Puts the compiled XDP program where `src/program.rs` can include it. +//! +//! The eBPF object is not built here: it needs a nightly toolchain and +//! bpf-linker, which the rest of the build does not. Build it beforehand with +//! `just build-ebpf`, or point `DATAPLANE_XDP_EBPF` at one. + +fn main() { + println!("cargo::rerun-if-changed=build.rs"); + #[cfg(feature = "runtime")] + embed_ebpf(); +} + +/// Copy the eBPF object into `OUT_DIR` under the name `program.rs` includes. +#[cfg(feature = "runtime")] +fn embed_ebpf() { + use std::path::PathBuf; + + println!("cargo::rerun-if-env-changed=DATAPLANE_XDP_EBPF"); + + #[allow(clippy::expect_used)] // cargo always sets these + let out_dir = PathBuf::from(std::env::var_os("OUT_DIR").expect("OUT_DIR is set by cargo")); + #[allow(clippy::expect_used)] // cargo always sets these + let manifest_dir = PathBuf::from( + std::env::var_os("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR is set by cargo"), + ); + let workspace_root = manifest_dir.parent().unwrap_or(&manifest_dir).to_path_buf(); + + // `just build-ebpf` leaves the object in the xdp-ebpf crate's own + // target directory -- it is outside the workspace -- under the BPF target + // and the profile it was built with. + let candidates = [ + std::env::var_os("DATAPLANE_XDP_EBPF").map(PathBuf::from), + Some(workspace_root.join("xdp-ebpf/target/bpfel-unknown-none/release/dataplane-xdp-ebpf")), + Some(workspace_root.join("xdp-ebpf/target/bpfel-unknown-none/debug/dataplane-xdp-ebpf")), + ]; + + let destination = out_dir.join("dataplane-xdp-ebpf"); + for candidate in candidates.iter().flatten() { + if !candidate.is_file() { + continue; + } + println!("cargo::rerun-if-changed={}", candidate.display()); + // The object usually comes from the nix store and is read only, and + // `copy` carries the mode across, so a second build would find a + // destination it cannot write. Take it out of the way first. + if let Err(e) = std::fs::remove_file(&destination) + && e.kind() != std::io::ErrorKind::NotFound + { + println!( + "cargo::error=could not replace {}: {e}", + destination.display() + ); + return; + } + if let Err(e) = std::fs::copy(candidate, &destination) { + println!( + "cargo::error=could not copy the XDP program from {}: {e}", + candidate.display() + ); + } + return; + } + + println!( + "cargo::error=no XDP program found. Build one with `just build-ebpf`, or set \ + DATAPLANE_XDP_EBPF to the path of one. Looked in {}", + candidates + .iter() + .flatten() + .map(|c| c.display().to_string()) + .collect::>() + .join(", ") + ); +} diff --git a/xdp/src/buffer.rs b/xdp/src/buffer.rs new file mode 100644 index 0000000000..9ac2fe08bb --- /dev/null +++ b/xdp/src/buffer.rs @@ -0,0 +1,455 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright Open Network Fabric Authors + +//! Packet buffer backed by UMEM frames. +//! +//! [`XdpBuffer`] implements the [`PacketBufferMut`] operations, and for a +//! packet that fits in one frame it does so directly on the memory the socket +//! shares with the kernel: the packet crosses the pipeline without being +//! copied. A buffer owns its storage for as long as it lives and hands it back +//! to the pool it came from when dropped. +//! +//! A packet longer than a frame cannot be held that way. The kernel will not +//! register a UMEM with chunks larger than a page, so such a packet arrives +//! split across frames, and the pipeline needs one contiguous run of bytes to +//! parse and rewrite. Those packets are gathered into a buffer of their own, +//! which costs a copy on the way in and another on the way out. Everything +//! that fits in a frame -- which is everything up to +//! [`MAX_FRAME_PACKET_LEN`](crate::umem::MAX_FRAME_PACKET_LEN) -- does not pay +//! either. +//! +//! [`PacketBufferMut`]: net::buffer::PacketBufferMut + +#![allow(unsafe_code)] // a frame is addressed through the UMEM region pointer + +use concurrency::sync::Arc; +use concurrency::sync::mpsc::Sender; +use net::buffer::{Headroom, Prepend, Tailroom, TrimFromEnd, TrimFromStart}; + +/// A prepend was asked for more room than the buffer has ahead of the packet. +/// +/// `net::buffer` has an error of the same shape, but it is `#[non_exhaustive]` +/// and so cannot be constructed outside that crate. +#[derive(Debug, thiserror::Error)] +#[error("not enough headroom in the packet buffer")] +pub struct NotEnoughHeadRoom; + +/// A trim was asked for more bytes than the packet holds. As above, this +/// cannot be `net::buffer::MemoryBufferNotLongEnough`. +#[derive(Debug, thiserror::Error)] +#[error("packet buffer not long enough")] +pub struct BufferNotLongEnough; +use std::fmt; + +use crate::umem::UmemRegion; + +/// Where a packet's bytes live, and how that storage is given back. +enum Storage { + /// One UMEM frame, read and written where the kernel put it. + Frame { + /// The region the frame is part of. Keeps the mapping alive. + region: Arc, + /// Byte offset of the frame within the region. + frame_addr: usize, + /// Where to hand the frame back on drop. + free_list: Sender, + }, + /// A buffer of its own, for a packet that did not fit in a frame. + Linear { + /// The bytes. Taken on drop, and `Some` until then. + buffer: Option>, + /// Where to hand them back on drop. + free_list: Sender>, + }, +} + +/// A packet, and the storage holding it. +/// +/// ```text +/// base data_offset + data_len + capacity +/// | | | | +/// v v v v +/// +-------------------------+---------------------+----------------+ +/// | headroom | packet data | tailroom | +/// +-------------------------+---------------------+----------------+ +/// ``` +/// +/// Prepending and trimming move `data_offset` and `data_len` within the +/// storage; nothing is ever moved in memory. +pub struct XdpBuffer { + /// Where the packet's bytes are. + storage: Storage, + /// Size of that storage, in bytes. + capacity: u16, + /// Offset of the packet data from the start of the storage. + data_offset: u16, + /// Length of the packet data, in bytes. + data_len: u16, +} + +// Buffers are built by the socket; with the `runtime` feature off the crate +// is just the layout, exercised by its own tests. +#[cfg_attr(not(any(feature = "runtime", test)), allow(dead_code))] +impl XdpBuffer { + /// Wrap the frame at `frame_addr`, holding `data_len` bytes of packet data + /// at `data_offset`. + /// + /// # Safety + /// + /// `frame_addr` must be the offset of a frame of `capacity` bytes within + /// `region`, and the caller must be handing over exclusive ownership of + /// that frame: neither another buffer nor the kernel may touch it until + /// this buffer is dropped. + pub(crate) unsafe fn frame( + region: Arc, + frame_addr: usize, + data_offset: u16, + data_len: u16, + capacity: u16, + free_list: Sender, + ) -> Self { + Self { + storage: Storage::Frame { + region, + frame_addr, + free_list, + }, + capacity, + data_offset, + data_len, + } + } + + /// Wrap a buffer of its own, holding `data_len` bytes of packet data at + /// `data_offset`. + pub(crate) fn linear( + buffer: Box<[u8]>, + data_offset: u16, + data_len: u16, + free_list: Sender>, + ) -> Self { + let capacity = u16::try_from(buffer.len()).unwrap_or(u16::MAX); + Self { + storage: Storage::Linear { + buffer: Some(buffer), + free_list, + }, + capacity, + data_offset, + data_len, + } + } + + /// Offset of this buffer's frame within the UMEM region, if it has one. + #[must_use] + pub fn frame_addr(&self) -> Option { + match &self.storage { + Storage::Frame { frame_addr, .. } => Some(*frame_addr), + Storage::Linear { .. } => None, + } + } + + /// Length of the packet data, in bytes. + #[must_use] + pub fn data_len(&self) -> u16 { + self.data_len + } + + /// Bytes of the storage the packet is written into, headroom and all. + /// + /// Only for the code that fills a buffer as a packet is gathered; the + /// pipeline sees the packet through `AsRef` and `AsMut`. + #[cfg(any(feature = "runtime", test))] + pub(crate) fn storage_mut(&mut self) -> &mut [u8] { + let capacity = self.capacity as usize; + match &mut self.storage { + // SAFETY: the frame is ours alone and is `capacity` bytes long. + Storage::Frame { + region, frame_addr, .. + } => unsafe { std::slice::from_raw_parts_mut(region.ptr_at(*frame_addr), capacity) }, + // `buffer` is only `None` once `drop` has taken it, and nothing + // can reach a buffer after that. + Storage::Linear { buffer, .. } => buffer.as_deref_mut().unwrap_or(&mut []), + } + } + + /// Pointer to the first byte of packet data. + #[inline] + fn data_ptr(&self) -> *mut u8 { + let offset = self.data_offset as usize; + match &self.storage { + // SAFETY: the frame is within the region and ours alone, and + // `data_offset` is kept within it by everything that moves it. + Storage::Frame { + region, frame_addr, .. + } => unsafe { region.ptr_at(frame_addr + offset) }, + // As in `storage_mut`: `None` is unreachable outside `drop`, and + // an empty slice's pointer is still valid to offset by zero. + Storage::Linear { buffer, .. } => { + let base = buffer.as_deref().unwrap_or(&[]).as_ptr().cast_mut(); + // SAFETY: `data_offset` is within the buffer, as above. + unsafe { base.add(offset) } + } + } + } +} + +impl Drop for XdpBuffer { + fn drop(&mut self) { + // A disconnected channel means the pool is being torn down, and the + // storage goes away with it. Nothing to do about it. + match &mut self.storage { + Storage::Frame { + frame_addr, + free_list, + .. + } => { + let _ = free_list.send(*frame_addr); + } + Storage::Linear { buffer, free_list } => { + if let Some(buffer) = buffer.take() { + let _ = free_list.send(buffer); + } + } + } + } +} + +impl fmt::Debug for XdpBuffer { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let kind = match self.storage { + Storage::Frame { .. } => "frame", + Storage::Linear { .. } => "linear", + }; + f.debug_struct("XdpBuffer") + .field("storage", &kind) + .field("frame_addr", &self.frame_addr()) + .field("data_offset", &self.data_offset) + .field("data_len", &self.data_len) + .field("capacity", &self.capacity) + .finish() + } +} + +impl AsRef<[u8]> for XdpBuffer { + #[inline] + fn as_ref(&self) -> &[u8] { + // SAFETY: the storage is ours alone, and the data region is within it. + unsafe { std::slice::from_raw_parts(self.data_ptr(), self.data_len as usize) } + } +} + +impl AsMut<[u8]> for XdpBuffer { + #[inline] + fn as_mut(&mut self) -> &mut [u8] { + // SAFETY: as for `as_ref`, and `&mut self` rules out an overlapping + // shared borrow. + unsafe { std::slice::from_raw_parts_mut(self.data_ptr(), self.data_len as usize) } + } +} + +impl Headroom for XdpBuffer { + fn headroom(&self) -> u16 { + self.data_offset + } +} + +impl Tailroom for XdpBuffer { + fn tailroom(&self) -> u16 { + self.capacity + .saturating_sub(self.data_offset + self.data_len) + } +} + +impl Prepend for XdpBuffer { + type Error = NotEnoughHeadRoom; + + fn prepend(&mut self, len: u16) -> Result<&mut [u8], Self::Error> { + if self.headroom() < len { + return Err(NotEnoughHeadRoom); + } + self.data_offset -= len; + self.data_len += len; + Ok(self.as_mut()) + } +} + +impl TrimFromStart for XdpBuffer { + type Error = BufferNotLongEnough; + + fn trim_from_start(&mut self, len: u16) -> Result<&mut [u8], Self::Error> { + if len > self.data_len { + return Err(BufferNotLongEnough); + } + self.data_offset += len; + self.data_len -= len; + Ok(self.as_mut()) + } +} + +impl TrimFromEnd for XdpBuffer { + type Error = BufferNotLongEnough; + + fn trim_from_end(&mut self, len: u16) -> Result<&mut [u8], Self::Error> { + if len > self.data_len { + return Err(BufferNotLongEnough); + } + self.data_len -= len; + Ok(self.as_mut()) + } +} + +#[cfg(test)] +#[allow(clippy::expect_used, clippy::unwrap_used)] +mod test { + use super::{NotEnoughHeadRoom, XdpBuffer}; + use crate::umem::{DATA_OFFSET, FRAME_SIZE, LINEAR_SIZE, MAX_FRAME_PACKET_LEN, UmemRegion}; + + use concurrency::sync::Arc; + use concurrency::sync::mpsc::{Receiver, channel}; + use net::buffer::{Headroom, Prepend, Tailroom, TrimFromEnd, TrimFromStart}; + + use std::ptr::NonNull; + + /// A stand-in for a mapped UMEM, backed by a plain allocation. Returned + /// alongside the backing store, which must outlive the region. + fn test_region(num_frames: usize) -> (Arc, Vec) { + let len = num_frames * usize::from(FRAME_SIZE); + let mut backing = vec![0u8; len]; + let base = NonNull::new(backing.as_mut_ptr()).unwrap(); + // SAFETY: `backing` is `len` bytes and is returned to the caller, who + // holds it for as long as the region. + let region = unsafe { UmemRegion::new(base, len) }; + (Arc::new(region), backing) + } + + /// Build a frame-backed buffer over frame 0 of a fresh region. The + /// returned channel is where the frame goes when the buffer is dropped. + fn frame_buffer(data: &[u8]) -> (XdpBuffer, Receiver, Arc, Vec) { + let (region, backing) = test_region(1); + let data_len = u16::try_from(data.len()).expect("test packets fit in a frame"); + let (tx, rx) = channel(); + // SAFETY: frame 0 is within the region and nothing else refers to it. + let mut buf = + unsafe { XdpBuffer::frame(region.clone(), 0, DATA_OFFSET, data_len, FRAME_SIZE, tx) }; + buf.as_mut().copy_from_slice(data); + (buf, rx, region, backing) + } + + /// Build a buffer of its own, filled the way the socket fills one as it + /// gathers a packet that arrived across several frames. + fn linear_buffer(data: &[u8]) -> (XdpBuffer, Receiver>) { + let data_len = u16::try_from(data.len()).expect("test packets fit"); + let (tx, rx) = channel(); + let mut buf = XdpBuffer::linear( + vec![0u8; LINEAR_SIZE as usize].into_boxed_slice(), + DATA_OFFSET, + data_len, + tx, + ); + let start = DATA_OFFSET as usize; + buf.storage_mut()[start..start + data.len()].copy_from_slice(data); + (buf, rx) + } + + #[test] + fn an_empty_frame_buffer_offers_the_whole_frame() { + let (buf, _rx, _region, _backing) = frame_buffer(&[]); + + assert_eq!(buf.headroom(), DATA_OFFSET); + assert_eq!(buf.tailroom(), MAX_FRAME_PACKET_LEN); + assert_eq!(buf.data_len(), 0); + assert!(buf.as_ref().is_empty()); + } + + #[test] + fn data_is_readable_where_it_was_written() { + let data = b"Hello, AF_XDP!"; + let (buf, _rx, _region, _backing) = frame_buffer(data); + let len = u16::try_from(data.len()).expect("test packets fit in a frame"); + + assert_eq!(buf.as_ref(), data); + assert_eq!(buf.data_len(), len); + assert_eq!(buf.headroom(), DATA_OFFSET); + assert_eq!(buf.tailroom(), MAX_FRAME_PACKET_LEN - len); + } + + /// A buffer of its own has to behave exactly as one over a frame does, or + /// a packet would be handled differently for having arrived in fragments. + #[test] + fn a_linear_buffer_behaves_as_a_frame_does() { + let data = [0xAB; 5000]; + let (mut buf, _rx) = linear_buffer(&data); + + assert_eq!(buf.as_ref(), &data[..]); + assert_eq!(buf.headroom(), DATA_OFFSET); + assert_eq!(buf.tailroom(), LINEAR_SIZE - DATA_OFFSET - 5000); + assert!(buf.frame_addr().is_none()); + + buf.prepend(14).expect("headroom was available"); + assert_eq!(buf.data_len(), 5014); + assert_eq!(&buf.as_ref()[14..], &data[..]); + + buf.trim_from_start(14).expect("the bytes are there"); + assert_eq!(buf.as_ref(), &data[..]); + } + + #[test] + fn prepend_then_trim_returns_the_buffer_to_where_it_started() { + let (mut buf, _rx, _region, _backing) = frame_buffer(&[0xAA; 100]); + let headroom = buf.headroom(); + let len = buf.data_len(); + + let slice = buf.prepend(10).expect("headroom was available"); + assert_eq!(slice.len(), (len + 10) as usize); + assert_eq!(buf.headroom(), headroom - 10); + assert_eq!(buf.data_len(), len + 10); + + buf.trim_from_start(10).expect("the bytes are there"); + assert_eq!(buf.headroom(), headroom); + assert_eq!(buf.data_len(), len); + + buf.trim_from_end(50).expect("the bytes are there"); + assert_eq!(buf.data_len(), len - 50); + assert_eq!(buf.tailroom(), MAX_FRAME_PACKET_LEN - (len - 50)); + } + + #[test] + fn prepend_beyond_the_start_of_the_storage_is_refused() { + let (mut buf, _rx, _region, _backing) = frame_buffer(&[0xAA; 100]); + + assert!(matches!( + buf.prepend(DATA_OFFSET + 1), + Err(NotEnoughHeadRoom) + )); + // The failed prepend left the buffer alone. + assert_eq!(buf.headroom(), DATA_OFFSET); + assert_eq!(buf.data_len(), 100); + } + + #[test] + fn trimming_more_than_the_packet_holds_is_refused() { + let (mut buf, _rx, _region, _backing) = frame_buffer(&[0xAA; 10]); + + assert!(buf.trim_from_start(11).is_err()); + assert!(buf.trim_from_end(11).is_err()); + assert_eq!(buf.data_len(), 10); + } + + #[test] + fn a_dropped_buffer_hands_its_frame_back() { + let (buf, rx, _region, _backing) = frame_buffer(&[]); + drop(buf); + assert_eq!(rx.try_recv(), Ok(0)); + } + + #[test] + fn a_dropped_buffer_hands_its_own_storage_back() { + let (buf, rx) = linear_buffer(&[0xAA; 5000]); + drop(buf); + assert_eq!( + rx.try_recv().expect("the buffer came back").len(), + LINEAR_SIZE as usize + ); + } +} diff --git a/xdp/src/lib.rs b/xdp/src/lib.rs new file mode 100644 index 0000000000..b1e5be728f --- /dev/null +++ b/xdp/src/lib.rs @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright Open Network Fabric Authors + +//! `AF_XDP` userspace support for the Hedgehog dataplane. +//! +//! The crate is split so that the parts which describe how a UMEM frame is +//! laid out build on their own, without libxdp: +//! +//! - [`umem`]: frame geometry and the mapped region frames are carved from +//! - [`buffer`]: [`buffer::XdpBuffer`], a `PacketBufferMut` over one frame +//! - [`socket`]: `AF_XDP` sockets and their rings (feature `runtime`) +//! - [`program`]: how packets reach the sockets (feature `runtime`) + +#![deny( + clippy::all, + clippy::pedantic, + clippy::unwrap_used, + clippy::expect_used, + clippy::panic, + rustdoc::all +)] +#![allow(rustdoc::missing_crate_level_docs)] + +pub mod buffer; +#[cfg(feature = "runtime")] +pub mod program; +#[cfg(feature = "runtime")] +pub mod socket; +pub mod umem; diff --git a/xdp/src/program.rs b/xdp/src/program.rs new file mode 100644 index 0000000000..47d1ec5512 --- /dev/null +++ b/xdp/src/program.rs @@ -0,0 +1,239 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright Open Network Fabric Authors + +//! Getting received packets from an interface to the sockets bound to it. +//! +//! An `AF_XDP` socket only ever sees the packets an XDP program redirects to +//! it, and redirecting is final: the network stack never sees that packet +//! again. So the program decides, for every packet, whether the dataplane gets +//! it or the host does, and [`Redirect`] is that program -- loaded, attached, +//! and told which sockets and which addresses to send things to. +//! +//! libxdp will load a redirect program of its own when a socket is bound, and +//! that is what happens if we do not inhibit it. We do: its program redirects +//! everything, which leaves the host receiving nothing on its own interfaces, +//! and it does not handle packets that span more than one buffer, which the +//! fabric's MTU guarantees. + +use std::collections::{HashMap, HashSet}; +use std::hash::Hash; +use std::io; +use std::net::IpAddr; +use std::os::unix::io::AsRawFd; + +use aya::Ebpf; +use aya::maps::{HashMap as BpfHashMap, XskMap}; +use aya::programs::{Xdp, XdpMode}; +use concurrency::sync::Mutex; +use tracing::{debug, info}; + +/// Most RX queues a redirect can serve on one interface. +/// +/// The socket map of the program has this many entries, so a socket bound to a +/// queue above it could never be redirected to. `xdp-ebpf` declares the same +/// number; it is a `no_std` crate for another target and cannot share this. +pub const MAX_QUEUES: u32 = 64; + +/// The compiled eBPF object, placed in `OUT_DIR` by our build script. +static PROGRAM: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/dataplane-xdp-ebpf")); + +/// Name of the XDP function within the object. +const PROGRAM_NAME: &str = "xdp_redirect"; + +/// Name of the socket map within the object. +const SOCKET_MAP_NAME: &str = "XSKMAP"; + +/// Names of the maps holding the host's own addresses. +const LOCAL_IPV4_MAP_NAME: &str = "LOCAL_IPV4"; +const LOCAL_IPV6_MAP_NAME: &str = "LOCAL_IPV6"; + +/// The XDP program that decides where each packet goes, attached to every +/// interface the driver serves. +/// +/// Dropping this detaches it, so it must be held for as long as packets are +/// expected. +pub struct Redirect { + /// One loaded and attached program per interface. Behind a lock because + /// workers register their sockets from their own threads. + programs: Mutex>, +} + +impl Redirect { + /// Load the program and attach it to each of `interfaces`. + /// + /// # Errors + /// + /// Returns an error if the program cannot be loaded or attached. + pub fn attach<'a>(interfaces: impl IntoIterator) -> io::Result { + let mut programs = HashMap::new(); + for if_name in interfaces { + if programs.contains_key(if_name) { + continue; + } + programs.insert(if_name.to_owned(), attach_to(if_name)?); + } + Ok(Self { + programs: Mutex::new(programs), + }) + } + + /// Tell the program where to send the packets of `if_name`, queue + /// `queue_id`. + /// + /// # Errors + /// + /// Returns an error if the kernel refuses the map entry. + pub fn register(&self, if_name: &str, queue_id: u32, socket: &impl AsRawFd) -> io::Result<()> { + let mut programs = self.programs.lock(); + let ebpf = programs + .get_mut(if_name) + .ok_or_else(|| io::Error::other(format!("no XDP program attached to {if_name}")))?; + + let map = ebpf.map_mut(SOCKET_MAP_NAME).ok_or_else(|| { + io::Error::other(format!("no map named '{SOCKET_MAP_NAME}' in the object")) + })?; + let mut sockets = XskMap::try_from(map) + .map_err(|e| io::Error::other(format!("'{SOCKET_MAP_NAME}' is not an XskMap: {e}")))?; + + sockets.set(queue_id, socket.as_raw_fd(), 0).map_err(|e| { + io::Error::other(format!( + "could not register the socket for {if_name}:q{queue_id}: {e}" + )) + })?; + + debug!("Registered the socket for {if_name}:q{queue_id}"); + Ok(()) + } + + /// Tell the program which addresses are the host's own, so that traffic to + /// them is left for the kernel instead of being redirected to a socket. + /// + /// Replaces whatever was there. Call it again whenever the set changes: an + /// address the program does not know about is an address whose traffic the + /// host stops receiving. + /// + /// # Errors + /// + /// Returns an error if the maps cannot be updated. + pub fn set_local_addresses(&self, addresses: &[IpAddr]) -> io::Result<()> { + let mut programs = self.programs.lock(); + // The same set goes to every interface: an address is the host's + // wherever the packet carrying it arrived. + for (if_name, ebpf) in programs.iter_mut() { + replace_addresses(ebpf, addresses) + .map_err(|e| io::Error::other(format!("on {if_name}: {e}")))?; + } + Ok(()) + } +} + +/// Put `addresses`, and only those, in one program's address maps. +fn replace_addresses(ebpf: &mut Ebpf, addresses: &[IpAddr]) -> io::Result<()> { + sync_family( + ebpf, + LOCAL_IPV4_MAP_NAME, + |address| match address { + // Stored as the address appears in the header, which is what the + // program reads out of the packet. + IpAddr::V4(v4) => Some(u32::from_ne_bytes(v4.octets())), + IpAddr::V6(_) => None, + }, + addresses, + )?; + sync_family( + ebpf, + LOCAL_IPV6_MAP_NAME, + |address| match address { + IpAddr::V4(_) => None, + IpAddr::V6(v6) => Some(v6.octets()), + }, + addresses, + ) +} + +/// Make the named map hold exactly the addresses `key_of` accepts. +/// +/// Updated in place rather than cleared and refilled, so that an address +/// that is not changing is never briefly absent -- traffic to it would be +/// redirected away from the host for as long as the gap lasted. +fn sync_family( + ebpf: &mut Ebpf, + name: &str, + key_of: impl Fn(&IpAddr) -> Option, + addresses: &[IpAddr], +) -> io::Result<()> { + let map = ebpf + .map_mut(name) + .ok_or_else(|| io::Error::other(format!("no map named '{name}' in the object")))?; + let mut map: BpfHashMap<_, K, u8> = BpfHashMap::try_from(map) + .map_err(|e| io::Error::other(format!("'{name}' is not a hash map: {e}")))?; + + let wanted: HashSet = addresses.iter().filter_map(key_of).collect(); + let present: HashSet = map + .keys() + .collect::>() + .map_err(|e| io::Error::other(format!("could not read the addresses already set: {e}")))?; + + for key in wanted.difference(&present) { + map.insert(key, 1, 0) + .map_err(|e| io::Error::other(format!("could not add an address: {e}")))?; + } + for key in present.difference(&wanted) { + map.remove(key) + .map_err(|e| io::Error::other(format!("could not drop an address: {e}")))?; + } + Ok(()) +} + +/// Load the redirect program and attach it to `if_name`. +/// +/// Native XDP is tried first, and generic XDP -- which every interface +/// supports, at the cost of running after the kernel has built an `skb` -- +/// second. +fn attach_to(if_name: &str) -> io::Result { + let mut ebpf = Ebpf::load(PROGRAM) + .map_err(|e| io::Error::other(format!("could not load the XDP program: {e}")))?; + + let program: &mut Xdp = ebpf + .program_mut(PROGRAM_NAME) + .ok_or_else(|| { + io::Error::other(format!("no function named '{PROGRAM_NAME}' in the object")) + })? + .try_into() + .map_err(|e| io::Error::other(format!("'{PROGRAM_NAME}' is not an XDP program: {e}")))?; + + program + .load() + .map_err(|e| io::Error::other(format!("the kernel rejected the XDP program: {e}")))?; + + match program.attach(if_name, XdpMode::Driver) { + Ok(_) => info!("XDP program attached to {if_name} in native mode"), + Err(native) => { + debug!("Native XDP attach to {if_name} failed ({native}); trying generic XDP"); + program.attach(if_name, XdpMode::Skb).map_err(|generic| { + io::Error::other(format!( + "could not attach the XDP program to {if_name}: \ + native mode said '{native}', generic mode said '{generic}'" + )) + })?; + info!("XDP program attached to {if_name} in generic mode"); + } + } + + Ok(ebpf) +} + +#[cfg(test)] +mod test { + /// The XDP program has to recognise VXLAN to keep it away from the kernel, + /// and it carries its own copy of the port because it is built for another + /// target and cannot use `net`. Nothing links the two but this. + #[test] + fn the_program_agrees_with_net_on_the_vxlan_port() { + assert_eq!( + u16::from(net::vxlan::Vxlan::PORT), + 4789, + "xdp-ebpf/src/main.rs hardcodes this as VXLAN_PORT" + ); + } +} diff --git a/xdp/src/socket.rs b/xdp/src/socket.rs new file mode 100644 index 0000000000..8ba6bd04b6 --- /dev/null +++ b/xdp/src/socket.rs @@ -0,0 +1,947 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright Open Network Fabric Authors + +//! `AF_XDP` sockets and the rings a worker moves packets over. +//! +//! A worker owns one [`XskUmem`] and one [`XskSocket`] per interface it serves. +//! All of its sockets are bound to the same UMEM, which is what lets a packet +//! received on one interface be forwarded out of another without leaving the +//! mapping, and which gives the worker a single pool of free frames to draw +//! from rather than one per interface. +//! +//! A frame moves through the rings like this: +//! +//! ```text +//! free frames +//! | ^ +//! fill ring v \ completion ring +//! (kernel writes) \ (kernel has sent) +//! | \ +//! RX ring TX ring +//! | ^ +//! v / +//! XdpBuffer -> pipeline -> send() +//! \ (copies into a free frame) +//! \ dropped by the pipeline +//! -> back to free frames +//! ``` +//! +//! A buffer hands its frame back over a channel when it is dropped, wherever +//! in the pipeline that happens; [`XskUmem::reclaim_dropped`] moves those +//! frames back to the free list. + +#![allow(unsafe_code)] // ring operations and UMEM access are unsafe by nature + +use concurrency::sync::Arc; +use concurrency::sync::mpsc::{Receiver, Sender, channel}; +use std::collections::VecDeque; +use std::io::Write; +use std::num::NonZeroU32; +use std::os::fd::BorrowedFd; +use std::os::unix::io::{AsRawFd, RawFd}; +use std::ptr::NonNull; + +use nix::poll::{PollFd, PollFlags, PollTimeout, poll}; +use tracing::{debug, info, trace, warn}; +use xsk_rs::config::{ + BindFlags, Interface, LibxdpFlags, SocketConfig, SocketConfigBuilder, UmemConfig, + UmemConfigBuilder, +}; +use xsk_rs::socket::{Socket, TxQueue, XdpStatistics}; +use xsk_rs::umem::frame::FrameDesc; +use xsk_rs::{CompQueue, FillQueue, RxQueue, Umem}; + +use crate::buffer::XdpBuffer; +use crate::program::Redirect; +use crate::umem::{ + DATA_OFFSET, FRAME_SIZE, LINEAR_SIZE, MAX_FRAME_PACKET_LEN, MAX_PACKET_LEN, UmemRegion, +}; + +/// Frames handled in one ring operation. +pub const BATCH_SIZE: usize = 64; + +/// The most frames one packet can span, which is what it takes to hold +/// [`MAX_PACKET_LEN`] at [`MAX_FRAME_PACKET_LEN`] a frame. +const MAX_FRAGMENTS: usize = MAX_PACKET_LEN.div_ceil(MAX_FRAME_PACKET_LEN) as usize; + +/// The descriptor option that says a packet continues in the next descriptor. +/// +/// xsk-rs reads it back through `FrameDesc::has_more_frames`, but does not +/// give a way to set it by name. +const XDP_PKT_CONTD: u32 = 1; + +/// Start of the frame the byte at `addr` falls in. +/// +/// Masking rather than subtracting the headroom, so that a frame is still +/// found when its packet does not start where the UMEM put it, which is the +/// case for any frame an XDP program has prepended to. Valid because the +/// kernel only takes a power of two frame size. +const fn frame_start(addr: usize) -> usize { + addr & !(FRAME_SIZE as usize - 1) +} + +/// An error, followed by the chain of causes behind it. +/// +/// libxdp's errors keep the errno that actually says what went wrong -- a +/// permission problem, a driver that will not do zero-copy -- in their source +/// rather than their message, so reporting only the message loses it. +fn with_causes(error: &dyn std::error::Error) -> String { + use std::fmt::Write as _; + + let mut message = error.to_string(); + let mut cause = error.source(); + while let Some(error) = cause { + // Writing to a String cannot fail. + let _ = write!(message, ": {error}"); + cause = error.source(); + } + message +} + +/// Size of each of the four rings, in descriptors. Must be a power of two. +pub const DEFAULT_RING_SIZE: u32 = 1024; + +/// Frames to give the UMEM for each socket bound to it. +/// +/// Enough to keep a fill ring full, with the rest covering the frames in +/// flight through the pipeline and on the TX ring. +pub const DEFAULT_FRAMES_PER_SOCKET: u32 = 2048; + +/// Buffers kept for packets that arrive across more than one frame. +/// +/// Only packets too long for a frame need one, and only for as long as they +/// are in the pipeline, so this is sized for a batch of them in flight rather +/// than for the frame pool. +pub const DEFAULT_LINEAR_BUFFERS: usize = 256; + +/// Anything that can go wrong setting up or driving an `AF_XDP` socket. +#[derive(Debug, thiserror::Error)] +pub enum XskError { + /// The requested geometry is one libxdp or the kernel will not accept. + #[error("invalid AF_XDP configuration: {0}")] + Config(String), + /// The UMEM could not be created. + #[error("could not create the UMEM: {0}")] + Umem(String), + /// The socket could not be created or bound. + #[error("could not bind an AF_XDP socket to {if_name}:q{queue_id}: {reason}")] + Bind { + /// Interface we tried to bind to. + if_name: String, + /// Queue we tried to bind to. + queue_id: u32, + /// What libxdp reported. + reason: String, + }, + /// A syscall failed. + #[error("AF_XDP I/O error: {0}")] + Io(#[from] std::io::Error), + /// Every frame is either on a ring or in the pipeline. + #[error("no free UMEM frames")] + NoFreeFrames, + /// The packet does not fit in a frame. + #[error("packet of {len} bytes exceeds the {MAX_PACKET_LEN} byte frame capacity")] + PacketTooLong { + /// Length of the packet we were asked to send. + len: usize, + }, + /// The kernel has not caught up with what we already queued. + #[error("TX ring full")] + TxRingFull, +} + +/// Geometry and bind options shared by every socket on one UMEM. +#[derive(Debug, Clone, Copy)] +pub struct XskConfig { + /// Number of frames in the UMEM. + pub frames: u32, + /// Number of descriptors in each socket's fill ring. + pub fill_ring_size: u32, + /// Number of descriptors in each socket's completion ring. + pub completion_ring_size: u32, + /// Number of descriptors in each socket's RX ring. + pub rx_ring_size: u32, + /// Number of descriptors in each socket's TX ring. + pub tx_ring_size: u32, + /// Bind in copy mode without trying zero-copy first. + pub force_copy: bool, + /// Buffers to keep for packets that span more than one frame. + pub linear_buffers: usize, +} + +impl Default for XskConfig { + fn default() -> Self { + Self { + frames: DEFAULT_FRAMES_PER_SOCKET, + fill_ring_size: DEFAULT_RING_SIZE, + completion_ring_size: DEFAULT_RING_SIZE, + rx_ring_size: DEFAULT_RING_SIZE, + tx_ring_size: DEFAULT_RING_SIZE, + force_copy: false, + linear_buffers: DEFAULT_LINEAR_BUFFERS, + } + } +} + +impl XskConfig { + /// Size the UMEM for `sockets` sockets, leaving the rest as it is. + #[must_use] + pub fn for_sockets(mut self, sockets: u32) -> Self { + self.frames = DEFAULT_FRAMES_PER_SOCKET.saturating_mul(sockets.max(1)); + self + } + + fn umem_config(self) -> Result { + UmemConfigBuilder::new() + .frame_size( + u32::from(FRAME_SIZE) + .try_into() + .map_err(|e| XskError::Config(format!("frame size: {e:?}")))?, + ) + .fill_queue_size( + self.fill_ring_size + .try_into() + .map_err(|e| XskError::Config(format!("fill ring size: {e:?}")))?, + ) + .comp_queue_size( + self.completion_ring_size + .try_into() + .map_err(|e| XskError::Config(format!("completion ring size: {e:?}")))?, + ) + // Our buffers prepend into the kernel's own headroom, which is + // free once a frame reaches userspace, so we ask for none of our + // own on top of it. + .frame_headroom(0) + .build() + .map_err(|e| XskError::Config(format!("UMEM: {e}"))) + } + + fn socket_config(self, zero_copy: bool) -> Result { + let mut builder = SocketConfigBuilder::new(); + builder + .rx_queue_size( + self.rx_ring_size + .try_into() + .map_err(|e| XskError::Config(format!("RX ring size: {e:?}")))?, + ) + .tx_queue_size( + self.tx_ring_size + .try_into() + .map_err(|e| XskError::Config(format!("TX ring size: {e:?}")))?, + ) + .bind_flags( + if zero_copy { + BindFlags::XDP_ZEROCOPY + } else { + BindFlags::XDP_COPY + } | BindFlags::XDP_USE_NEED_WAKEUP + // Without this the kernel drops anything that does not fit + // in one frame rather than splitting it across several, + // which is every packet above ~3.8KB. + | BindFlags::XDP_USE_SG, + ); + + // We bring our own program, so libxdp must not load one of its own. + builder.libxdp_flags(LibxdpFlags::XSK_LIBXDP_FLAGS_INHIBIT_PROG_LOAD); + + Ok(builder.build()) + } +} + +/// The UMEM a worker's sockets share, and the pool of frames within it. +/// +/// Frames are named by their index, which is their offset in the region +/// divided by [`FRAME_SIZE`]. A frame is on exactly one of: the free list, a +/// fill ring, an RX ring, a TX ring, a completion ring, or an [`XdpBuffer`] +/// somewhere in the pipeline. +pub struct XskUmem { + /// The xsk-rs UMEM. Keeps the mapping alive and owns the frame layout. + umem: Umem, + /// The same mapping, addressed by offset, for [`XdpBuffer`] to read. + region: Arc, + /// One descriptor per frame, indexed by frame index. + descs: Vec, + /// Frames not currently owned by a ring or a buffer. + free: VecDeque, + /// Handed to each buffer so it can return its frame when dropped. + returned_tx: Sender, + /// The frames buffers have handed back, not yet on the free list. + returned_rx: Receiver, + /// Buffers for packets that span more than one frame. + linear: Vec>, + /// Handed to each such buffer so it comes back when dropped. + linear_tx: Sender>, + /// The ones handed back, not yet on the free list. + linear_rx: Receiver>, + /// Geometry and bind options for the sockets created from this UMEM. + config: XskConfig, +} + +impl XskUmem { + /// Map a UMEM and carve it into frames, all of them free. + /// + /// # Errors + /// + /// Returns [`XskError::Config`] if the geometry is not one libxdp accepts, + /// or [`XskError::Umem`] if the mapping or registration fails. + pub fn new(config: XskConfig) -> Result { + let umem_config = config.umem_config()?; + let frames = NonZeroU32::new(config.frames) + .ok_or_else(|| XskError::Config("a UMEM needs at least one frame".to_owned()))?; + + let (umem, descs) = + Umem::new(umem_config, frames, false).map_err(|e| XskError::Umem(with_causes(&e)))?; + + let first = descs + .first() + .ok_or_else(|| XskError::Umem("UMEM has no frames".to_owned()))?; + + // xsk-rs hands out descriptors, not the base of the mapping, but + // XdpBuffer works in offsets. The data of the first frame sits exactly + // `addr` bytes into the mapping, so subtracting gets us back to the + // start of it. + let region = { + let addr = first.addr(); + // SAFETY: `first` belongs to `umem`, and no frame has been given + // to a ring yet, so nothing else is looking at it. + let data = unsafe { umem.data(first) }; + let base = data.contents().as_ptr().cast_mut(); + // SAFETY: `base` points `addr` bytes into the mapping, so the + // subtraction lands on its first byte. + let base = unsafe { base.sub(addr) }; + let base = NonNull::new(base) + .ok_or_else(|| XskError::Umem("UMEM starts at a null address".to_owned()))?; + let len = + usize::try_from(config.frames).unwrap_or(usize::MAX) * usize::from(FRAME_SIZE); + // SAFETY: the mapping is `len` bytes and is kept alive by `umem`, + // which this `XskUmem` owns for as long as the region. + Arc::new(unsafe { UmemRegion::new(base, len) }) + }; + + let (returned_tx, returned_rx) = channel(); + let (linear_tx, linear_rx) = channel(); + let linear = (0..config.linear_buffers) + .map(|_| vec![0u8; LINEAR_SIZE as usize].into_boxed_slice()) + .collect(); + + debug!( + "UMEM mapped: {} frames of {FRAME_SIZE} bytes ({} MiB)", + descs.len(), + (descs.len() * usize::from(FRAME_SIZE)) >> 20, + ); + + Ok(Self { + umem, + region, + free: (0..descs.len()).collect(), + descs, + returned_tx, + returned_rx, + linear, + linear_tx, + linear_rx, + config, + }) + } + + /// Bind a socket to `if_name`, queue `queue_id`, fill its RX ring, and + /// tell `redirect` where the packets of that queue should go. + /// + /// The two belong together: a socket bound without telling the redirect + /// about it is a socket that receives nothing. + /// + /// Zero-copy is tried first unless [`XskConfig::force_copy`] is set. A + /// driver that cannot do zero-copy fails the bind, and we fall back to + /// copy mode rather than refusing to run. + /// + /// # Errors + /// + /// Returns [`XskError::Bind`] if neither mode binds, or an I/O error if + /// the socket cannot be registered with the redirect. + pub fn bind( + &mut self, + if_name: &str, + queue_id: u32, + redirect: &Redirect, + ) -> Result { + let interface: Interface = if_name.parse().map_err(|e| XskError::Bind { + if_name: if_name.to_owned(), + queue_id, + reason: format!("invalid interface name: {e}"), + })?; + + let bind = |zero_copy: bool| { + let config = self.config.socket_config(zero_copy)?; + // SAFETY: every socket we create is bound to a distinct + // (interface, queue) pair of this UMEM, which is the case libxdp + // hands fresh fill and completion queues back for. + unsafe { Socket::new(config, &self.umem, &interface, queue_id) }.map_err(|e| { + XskError::Bind { + if_name: if_name.to_owned(), + queue_id, + reason: with_causes(&e), + } + }) + }; + + let (zero_copy, (tx, rx, queues)) = if self.config.force_copy { + (false, bind(false)?) + } else { + match bind(true) { + Ok(socket) => (true, socket), + Err(e) => { + info!("Zero-copy bind failed ({e}); falling back to copy mode"); + (false, bind(false)?) + } + } + }; + + let (fq, cq) = queues.ok_or_else(|| XskError::Bind { + if_name: if_name.to_owned(), + queue_id, + reason: "libxdp returned no fill and completion queues, which means \ + the pair is already bound from this UMEM" + .to_owned(), + })?; + + info!( + "AF_XDP socket bound to {if_name}:q{queue_id} in {} mode", + if zero_copy { "zero-copy" } else { "copy" }, + ); + + let mut socket = XskSocket { + if_name: if_name.to_owned(), + queue_id, + fq, + cq, + rx, + tx, + rx_descs: vec![FrameDesc::default(); BATCH_SIZE], + cq_descs: vec![FrameDesc::default(); BATCH_SIZE], + fill_descs: Vec::with_capacity(BATCH_SIZE), + tx_descs: Vec::with_capacity(MAX_FRAGMENTS), + rx_packet: Vec::with_capacity(MAX_FRAGMENTS), + tx_queued: false, + }; + + // Give the kernel somewhere to put the packets it receives; without + // this the socket is bound but deaf. Never more than half of what is + // free, though: the frames left over are what packets are transmitted + // from and what the pipeline holds, and a socket that took the lot + // would leave the ones bound after it, and every send, with nothing. + let to_post = (self.config.fill_ring_size as usize).min(self.free.len() / 2); + let posted = socket.replenish(self, to_post); + if posted == 0 { + warn!("No frames left to fill the RX ring of {if_name}:q{queue_id}"); + } + + // Last, because a socket has to be bound before the redirect will take + // it, and first in the sense that matters: until this happens the + // socket is bound and deaf, and the packets go to the kernel instead. + redirect.register(if_name, queue_id, &socket)?; + + Ok(socket) + } + + /// Move the frames of dropped buffers back to the free list. + /// + /// Returns how many frames came back. + pub fn reclaim_dropped(&mut self) -> usize { + let mut reclaimed = 0; + while let Ok(frame_addr) = self.returned_rx.try_recv() { + self.release(frame_addr); + reclaimed += 1; + } + while let Ok(buffer) = self.linear_rx.try_recv() { + self.linear.push(buffer); + reclaimed += 1; + } + reclaimed + } + + /// Number of frames in the UMEM. + #[must_use] + pub fn total_frames(&self) -> usize { + self.descs.len() + } + + /// Put the frame starting at `frame_addr` back on the free list. + fn release(&mut self, frame_addr: usize) { + let index = frame_addr / usize::from(FRAME_SIZE); + if index < self.descs.len() { + self.free.push_back(index); + } else { + // Only reachable if a descriptor came back with an address outside + // the UMEM, which would mean the ring is corrupt. Losing the frame + // is better than handing out one that is not ours. + warn!("Discarding a frame at {frame_addr}, which is outside the UMEM"); + } + } + + /// Take a frame off the free list, if there is one. + fn claim(&mut self) -> Option { + self.free.pop_front() + } + + /// Wrap the frame the RX ring just handed us in a buffer. + #[allow(clippy::cast_possible_truncation)] // a frame holds well under 64KiB + fn buffer(&self, desc: &FrameDesc) -> XdpBuffer { + let frame_addr = frame_start(desc.addr()); + let data_offset = (desc.addr() - frame_addr) as u16; + let data_len = desc.lengths().data() as u16; + // SAFETY: the frame was just consumed from the RX ring, so the kernel + // is done with it and no other buffer refers to it. Its address and + // length come from the descriptor the kernel filled in. + unsafe { + XdpBuffer::frame( + self.region.clone(), + frame_addr, + data_offset, + data_len, + FRAME_SIZE, + self.returned_tx.clone(), + ) + } + } + + /// Gather a packet that arrived across `descs` into one buffer. + /// + /// The pipeline needs the packet as one run of bytes, and the fragments + /// are in frames that are nowhere near each other, so this is a copy. It + /// happens only for packets too long for a single frame. + /// + /// The frames are handed straight back: unlike the single-frame case, the + /// buffer that comes out does not hold them. + /// + /// Returns `None` if there is no buffer free to gather into, or if the + /// packet is longer than one can hold, having released the frames either + /// way. + fn gather(&mut self, descs: &[FrameDesc]) -> Option { + let total: usize = descs.iter().map(|desc| desc.lengths().data()).sum(); + + let gathered = if total > MAX_PACKET_LEN as usize { + warn!( + "Dropping a {total} byte packet, which is longer than the {MAX_PACKET_LEN} \ + bytes a buffer holds" + ); + None + } else if let Some(storage) = self.linear.pop() { + #[allow(clippy::cast_possible_truncation)] // checked against MAX_PACKET_LEN above + let mut buffer = + XdpBuffer::linear(storage, DATA_OFFSET, total as u16, self.linear_tx.clone()); + let start = DATA_OFFSET as usize; + let storage = buffer.storage_mut(); + let mut at = start; + for desc in descs { + let len = desc.lengths().data(); + // SAFETY: the frame was just consumed from the RX ring, so it + // is ours and holds `len` bytes at `desc.addr()`. + let fragment = + unsafe { std::slice::from_raw_parts(self.region.ptr_at(desc.addr()), len) }; + storage[at..at + len].copy_from_slice(fragment); + at += len; + } + trace!("Gathered a {total} byte packet from {} frames", descs.len()); + Some(buffer) + } else { + warn!("Dropping a {total} byte packet: no buffer free to gather it into"); + None + }; + + // The fragments are copied out either way, so the frames go back + // whether or not a packet came of them. + for desc in descs { + self.release(frame_start(desc.addr())); + } + gathered + } + + /// Copy `data` into as many free frames as it needs, and return their + /// descriptors in the order they must be put on the TX ring. + /// + /// A packet that arrived in a frame of this same UMEM is copied into + /// another one, which the shared UMEM ought to make unnecessary. Sending + /// the frame where it lies would mean handing the ring a descriptor for + /// the packet's own address and length, and xsk-rs keeps `FrameDesc`'s + /// address private and its constructor crate-internal; the only way to + /// give a descriptor a length is to write through a cursor at the frame's + /// fixed data offset, which is this copy. Avoiding it needs a change + /// upstream, or dropping to libxdp-sys. + /// + /// A packet that fits in one frame takes one, which is the common case. + /// One that does not is split, and every descriptor but the last is marked + /// as having more to come, which is how the kernel is told they are one + /// packet. + /// + /// The frames are handed back on failure, so a caller that gets an error + /// has leaked nothing. + fn stage(&mut self, data: &[u8], staged: &mut Vec) -> Result<(), XskError> { + if data.len() > MAX_PACKET_LEN as usize { + return Err(XskError::PacketTooLong { len: data.len() }); + } + + staged.clear(); + let mut rest = data; + while !rest.is_empty() { + let Some(index) = self.claim() else { + self.unstage(staged); + return Err(XskError::NoFreeFrames); + }; + let take = rest.len().min(MAX_FRAME_PACKET_LEN as usize); + let (fragment, remainder) = rest.split_at(take); + rest = remainder; + + // SAFETY: the frame was just taken off the free list, so neither + // the kernel nor a buffer is looking at it. + let written = { + let desc = &mut self.descs[index]; + let mut data_mut = unsafe { self.umem.data_mut(desc) }; + let mut cursor = data_mut.cursor(); + // The frame may have carried a longer packet before; writing + // starts where the cursor is, and the descriptor's length is + // where it ends. + cursor.set_pos(0); + cursor.write_all(fragment) + }; + if let Err(e) = written { + self.free.push_back(index); + self.unstage(staged); + return Err(XskError::Io(e)); + } + + let mut desc = self.descs[index]; + // Every fragment but the last says there is more to come. The last + // one is fixed up below, once we know it is the last. + desc.set_options(XDP_PKT_CONTD); + staged.push(desc); + } + + if let Some(last) = staged.last_mut() { + last.set_options(0); + } + Ok(()) + } + + /// Hand staged frames back, for a packet that will not be sent after all. + fn unstage(&mut self, staged: &mut Vec) { + for desc in staged.drain(..) { + self.release(frame_start(desc.addr())); + } + } +} + +/// One `AF_XDP` socket, bound to an (interface, queue) pair./// One `AF_XDP` socket, bound to an (interface, queue) pair. +/// +/// The socket borrows the [`XskUmem`] it was bound to for every operation that +/// moves frames, which is what keeps a single free list behind sockets that +/// each have their own rings. +pub struct XskSocket { + /// Interface this socket is bound to. + if_name: String, + /// RX queue this socket is bound to. + queue_id: u32, + /// Free frames go here for the kernel to receive into. + fq: FillQueue, + /// Sent frames come back here. + cq: CompQueue, + /// Received frames come out here. + rx: RxQueue, + /// Frames to send go here. + tx: TxQueue, + /// Scratch for [`RxQueue::consume`]. + rx_descs: Vec, + /// Scratch for [`CompQueue::consume`]. + cq_descs: Vec, + /// Scratch for [`FillQueue::produce`]. + fill_descs: Vec, + /// Scratch for the descriptors of one packet being sent. + tx_descs: Vec, + /// Scratch for the descriptors of one packet being received. + rx_packet: Vec, + /// Whether anything has been put on the TX ring since the last flush. + tx_queued: bool, +} + +impl XskSocket { + /// Interface this socket is bound to. + #[must_use] + pub fn if_name(&self) -> &str { + &self.if_name + } + + /// Take up to [`BATCH_SIZE`] received packets off the RX ring, appending + /// them to `packets`. Returns how many were appended. + pub fn recv(&mut self, umem: &mut XskUmem, packets: &mut Vec) -> usize { + // SAFETY: the scratch descriptors belong to no ring; the RX ring + // overwrites them with the frames it has for us. + let received = unsafe { self.rx.consume(&mut self.rx_descs) }; + if received == 0 { + return 0; + } + + trace!( + "{}:q{}: {received} frames off the RX ring", + self.if_name, self.queue_id + ); + + let before = packets.len(); + for desc in &self.rx_descs[..received] { + let continues = desc.has_more_frames(); + + // The common case: a packet that fits one frame, kept where the + // kernel put it. + if !continues && self.rx_packet.is_empty() { + packets.push(umem.buffer(desc)); + continue; + } + + self.rx_packet.push(*desc); + if continues && self.rx_packet.len() < MAX_FRAGMENTS { + continue; + } + + if continues { + // More fragments than a packet we can hold has. Give up on it + // rather than growing without bound on a ring that is telling + // us something impossible. + warn!( + "{}:q{}: dropping a packet that spans more than {MAX_FRAGMENTS} frames", + self.if_name, self.queue_id + ); + for desc in &self.rx_packet { + umem.release(frame_start(desc.addr())); + } + } else if let Some(packet) = umem.gather(&self.rx_packet) { + packets.push(packet); + } + self.rx_packet.clear(); + } + + // A packet whose last fragment has not arrived yet stays in + // `rx_packet` and is finished on the next pass. + packets.len() - before + } + + /// Copy `data` into a free frame and put it on the TX ring. + /// + /// The kernel is not woken here: call [`flush_tx`](Self::flush_tx) once a + /// batch has been queued. + /// + /// # Errors + /// + /// Returns [`XskError::NoFreeFrames`] if the UMEM is exhausted, + /// [`XskError::PacketTooLong`] if the packet does not fit a frame, or + /// [`XskError::TxRingFull`] if the kernel has not kept up. + pub fn send(&mut self, umem: &mut XskUmem, data: &[u8]) -> Result<(), XskError> { + umem.stage(data, &mut self.tx_descs)?; + + // The fragments of one packet have to be next to each other on the + // ring, which is what producing them in one go gets us: the ring + // either takes them all or takes none. + // SAFETY: every frame is off the free list and holds part of the + // packet we just wrote, so none is on another ring. + let queued = unsafe { self.tx.produce(&self.tx_descs) }; + if queued != self.tx_descs.len() { + umem.unstage(&mut self.tx_descs); + return Err(XskError::TxRingFull); + } + + self.tx_descs.clear(); + self.tx_queued = true; + trace!( + "{}:q{}: queued {} bytes for transmission in {queued} frame(s)", + self.if_name, + self.queue_id, + data.len() + ); + Ok(()) + } + + /// Wake the kernel if it is asleep and we have queued something for it. + /// + /// # Errors + /// + /// Returns the error the wakeup syscall reported. + pub fn flush_tx(&mut self) -> std::io::Result<()> { + if !self.tx_queued { + return Ok(()); + } + self.tx_queued = false; + if self.tx.needs_wakeup() { + self.tx.wakeup()?; + } + Ok(()) + } + + /// Reclaim the frames the kernel is done sending and top the fill ring + /// back up. Call once per pass of the worker loop. + pub fn service(&mut self, umem: &mut XskUmem) { + self.collect_completions(umem); + self.replenish(umem, BATCH_SIZE); + } + + /// Statistics the kernel keeps for this socket, including the packets it + /// had to drop for want of a frame on the fill ring. + /// + /// # Errors + /// + /// Returns the error `getsockopt(2)` reported. + pub fn statistics(&self) -> std::io::Result { + self.rx.fd().xdp_statistics() + } + + /// Move frames the kernel has finished transmitting back to the free list. + fn collect_completions(&mut self, umem: &mut XskUmem) { + // SAFETY: the scratch descriptors belong to no ring; the completion + // ring overwrites them with the frames it is handing back. + let completed = unsafe { self.cq.consume(&mut self.cq_descs) }; + if completed == 0 { + return; + } + + trace!( + "{}:q{}: {completed} frames off the completion ring", + self.if_name, self.queue_id + ); + for desc in &self.cq_descs[..completed] { + umem.release(frame_start(desc.addr())); + } + } + + /// Post up to `count` free frames on the fill ring. Returns how many the + /// ring took. + fn replenish(&mut self, umem: &mut XskUmem, count: usize) -> usize { + // The ring takes all of what it is offered or none of it, so offering + // more than it has room for would pop frames off the free list only to + // push every one of them straight back. + let count = count.min(self.fq.nb_free_exact() as usize); + + self.fill_descs.clear(); + for _ in 0..count { + let Some(index) = umem.claim() else { break }; + match umem.descs.get(index) { + Some(desc) => self.fill_descs.push(*desc), + None => break, + } + } + if self.fill_descs.is_empty() { + return 0; + } + + // SAFETY: every frame here came off the free list, so it is on no + // other ring and no buffer refers to it. They all belong to `umem`, + // which is the UMEM this socket was bound to. + let posted = unsafe { + self.fq + .produce_and_wakeup(&self.fill_descs, self.rx.fd_mut(), 0) + } + .unwrap_or_else(|e| { + // The frames were still handed over; only the wakeup failed, and + // the next poll will find them. + debug!( + "{}:q{}: could not wake the kernel after filling: {e}", + self.if_name, self.queue_id + ); + self.fill_descs.len() + }); + + // A full ring takes nothing at all rather than a prefix, but handle + // the general case: whatever it refused stays free. + for desc in &self.fill_descs[posted..] { + umem.release(frame_start(desc.addr())); + } + + if posted > 0 { + trace!( + "{}:q{}: {posted} frames onto the fill ring", + self.if_name, self.queue_id + ); + } + posted + } +} + +impl AsRawFd for XskSocket { + fn as_raw_fd(&self) -> RawFd { + self.rx.fd().as_raw_fd() + } +} + +/// Wait until any of `sockets` has packets, or `timeout_ms` elapses. +/// +/// `ready` is filled with one flag per socket, in the order they were given, +/// saying which of them the kernel has something for. Returns whether any of +/// them does. +/// +/// # Errors +/// +/// Returns the error `poll(2)` reported, except that an interrupted call is +/// reported as nothing being ready: the caller is expected to come round +/// again, and it may want to look at why it was interrupted first. +pub fn wait_for_packets<'a>( + sockets: impl IntoIterator, + timeout_ms: u16, + ready: &mut Vec, +) -> std::io::Result { + let mut fds: Vec = sockets + .into_iter() + .map(|socket| { + // SAFETY: the socket owns the descriptor and outlives the borrow, + // which does not leave this function. + let fd = unsafe { BorrowedFd::borrow_raw(socket.as_raw_fd()) }; + PollFd::new(fd, PollFlags::POLLIN) + }) + .collect(); + + let woken = match poll(&mut fds, PollTimeout::from(timeout_ms)) { + Ok(woken) => woken, + Err(nix::errno::Errno::EINTR) => 0, + Err(e) => return Err(e.into()), + }; + + ready.clear(); + ready.extend(fds.iter().map(|fd| { + fd.revents() + .is_some_and(|events| events.contains(PollFlags::POLLIN)) + })); + Ok(woken > 0) +} + +#[cfg(test)] +#[allow(clippy::expect_used, clippy::unwrap_used)] +mod test { + use super::{frame_start, with_causes}; + use crate::umem::{DATA_OFFSET, FRAME_SIZE}; + + /// Whatever the kernel puts in a descriptor, we have to be able to say + /// which frame it names, including when the packet does not start where + /// the UMEM put it. + #[test] + fn every_address_in_a_frame_names_that_frame() { + let frame = 7 * usize::from(FRAME_SIZE); + + assert_eq!(frame_start(frame), frame); + assert_eq!(frame_start(frame + usize::from(DATA_OFFSET)), frame); + assert_eq!(frame_start(frame + usize::from(FRAME_SIZE) - 1), frame); + // One past the end is the next frame, not this one. + assert_eq!( + frame_start(frame + usize::from(FRAME_SIZE)), + frame + usize::from(FRAME_SIZE) + ); + } + + /// libxdp keeps the errno in the source of its errors, and that is the + /// part an operator needs, so it has to reach the log. + #[test] + fn the_cause_of_an_error_is_reported_with_it() { + #[derive(Debug, thiserror::Error)] + #[error("could not bind")] + struct Outer(#[source] std::io::Error); + + let error = Outer(std::io::Error::from_raw_os_error(1)); + let reported = with_causes(&error); + + assert!(reported.starts_with("could not bind: "), "{reported}"); + assert!(reported.contains("Operation not permitted"), "{reported}"); + } +} diff --git a/xdp/src/umem.rs b/xdp/src/umem.rs new file mode 100644 index 0000000000..c0dbb93889 --- /dev/null +++ b/xdp/src/umem.rs @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright Open Network Fabric Authors + +//! Geometry of the UMEM, the memory an `AF_XDP` socket shares with the kernel. +//! +//! The UMEM is one mapping split into equally sized frames. The kernel writes +//! received packets into frames userspace has posted on a fill ring, and reads +//! transmitted ones from frames userspace posts on a TX ring. Userspace reads +//! and writes them in place: there is no copy in or out of the mapping, which +//! is the whole point of `AF_XDP`. + +#![allow(unsafe_code)] // the mapping is addressed by pointer; see `UmemRegion` + +use std::ptr::NonNull; + +/// Size of a UMEM frame, in bytes. +/// +/// libxdp rejects frame sizes that are not a power of two. 4096 is a page, +/// which is what the kernel's zero-copy paths are happiest with. +pub const FRAME_SIZE: u16 = 4096; + +/// Headroom the kernel reserves ahead of the packet data of every frame +/// (`XDP_PACKET_HEADROOM`). +/// +/// It exists so that an XDP program can prepend to a packet before it reaches +/// the socket. Ours does not, so by the time a frame is handed to us the whole +/// of it is spare room the pipeline may prepend into. +pub const XDP_HEADROOM: u16 = 256; + +/// Offset of the packet data from the start of a frame. +/// +/// The UMEM is registered with no user headroom of its own, so this is just +/// the kernel's own reservation. +pub const DATA_OFFSET: u16 = XDP_HEADROOM; + +/// The largest packet a single frame can hold. +/// +/// The kernel will not register a UMEM whose chunks are larger than a page, so +/// this is a ceiling we cannot raise. A packet longer than this arrives across +/// several frames and leaves across several more; see [`LINEAR_SIZE`]. +pub const MAX_FRAME_PACKET_LEN: u16 = FRAME_SIZE - DATA_OFFSET; + +/// Size of the buffer a packet is gathered into when it arrives across more +/// than one frame. +/// +/// Large enough for a jumbo frame and the headroom a buffer carries, with room +/// to spare, and a power of two so the allocator has an easy time of it. +pub const LINEAR_SIZE: u16 = 16384; + +/// The largest packet the driver can carry. +pub const MAX_PACKET_LEN: u16 = LINEAR_SIZE - DATA_OFFSET; + +/// The mapped UMEM region, shared by every frame carved out of it. +/// +/// Frames are addressed by their byte offset within the region rather than by +/// pointer, which is also how the kernel names them on the rings. Holding the +/// region in an `Arc` keeps the mapping alive for as long as any buffer still +/// refers to a frame in it. +pub(crate) struct UmemRegion { + /// Start of the mapped region. + base: NonNull, + /// Length of the mapped region, in bytes. + len: usize, +} + +// SAFETY: a `UmemRegion` is only ever read through `ptr_at`, whose contract +// puts the burden of exclusive access on the caller. The region itself is a +// plain mapping with no interior state to race on. +unsafe impl Send for UmemRegion {} +// SAFETY: see above. +unsafe impl Sync for UmemRegion {} + +// The region is built by the socket; see the note on `impl XdpBuffer`. +#[cfg_attr(not(any(feature = "runtime", test)), allow(dead_code))] +impl UmemRegion { + /// Describe a UMEM region. + /// + /// # Safety + /// + /// `base` must point to a mapping of at least `len` bytes which stays + /// mapped for as long as this `UmemRegion`, and any buffer derived from + /// it, is alive. + pub(crate) unsafe fn new(base: NonNull, len: usize) -> Self { + Self { base, len } + } + + /// Pointer to the byte at `offset` within the region. + /// + /// # Safety + /// + /// `offset` must be within the region, and the caller must hold exclusive + /// access to the frame it falls in: neither another buffer nor the kernel + /// may be reading or writing it. + #[inline] + pub(crate) unsafe fn ptr_at(&self, offset: usize) -> *mut u8 { + debug_assert!(offset < self.len); + // SAFETY: the caller promises `offset` is within the region, so the + // result stays inside the same allocated object. + unsafe { self.base.as_ptr().add(offset) } + } +}