From 2052d2ce0949eb2320cd522574e11316b697f028 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 19 Oct 2026 02:25:37 +0000 Subject: [PATCH 1/9] chore(deps): update rust nightly --- .bazelrc | 2 +- MODULE.bazel | 6 +++--- rust-toolchain.toml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.bazelrc b/.bazelrc index 8fbc521a71..130eccbb42 100644 --- a/.bazelrc +++ b/.bazelrc @@ -15,7 +15,7 @@ common --repo_env=CARGO_NET_GIT_FETCH_WITH_CLI=true # host cargo would download another 418 MB toolchain on every CI job. common --repo_env=CARGO_UNSTABLE_GIT=shallow-deps common --repo_env=RUSTC_BOOTSTRAP=1 -# MODULE.bazel registers only nightly/2026-10-18. Without this, rules_rust +# MODULE.bazel registers only nightly/2026-10-19. Without this, rules_rust # resolves the stable channel and the build fails closed. # Hosts can extend this repository profile with their private fleet settings. build --config=tracedecay diff --git a/MODULE.bazel b/MODULE.bazel index 275174e15e..b9d2b9af10 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -47,9 +47,9 @@ rust.toolchain( "fastbuild": "1", "opt": "3", }, - versions = ["nightly/2026-10-18"], - rustfmt_version = "nightly/2026-10-18", - rust_analyzer_version = "nightly/2026-10-18", + versions = ["nightly/2026-10-19"], + rustfmt_version = "nightly/2026-10-19", + rust_analyzer_version = "nightly/2026-10-19", ) use_repo( rust, diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 5f396742b5..09be1af02d 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,6 +1,6 @@ [toolchain] # Dated nightly: asupersync's nightly-outcome-try default needs try_trait_v2. # 2026-09-27 includes rust-lang/rust#162275; 2026-08-25 does not. -channel = "nightly-2026-10-18" +channel = "nightly-2026-10-19" # LSP acceptance negotiates capabilities with the pinned upstream analyzer. components = ["clippy", "rustfmt", "rust-analyzer", "rust-src"] From f960a70b3783bfd036d6d333e7cb057be1fcca43 Mon Sep 17 00:00:00 2001 From: Zack Jackson Date: Sun, 18 Oct 2026 19:55:36 -0700 Subject: [PATCH 2/9] fix(build): update the analyzer import with Rust nightly --- MODULE.bazel | 2 +- renovate.json | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index b9d2b9af10..25a06f31a7 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -54,7 +54,7 @@ rust.toolchain( use_repo( rust, "rust_toolchains", - rust_analyzer_tools = "rust_analyzer_nightly-2026-10-18_tools", + rust_analyzer_tools = "rust_analyzer_nightly-2026-10-19_tools", ) register_toolchains("@rust_toolchains//:all") diff --git a/renovate.json b/renovate.json index 40ca9c23fd..d18c91f19e 100644 --- a/renovate.json +++ b/renovate.json @@ -23,7 +23,10 @@ "description": "rules_rust's nightly pin, and the .bazelrc note that names it, must match rust-toolchain.toml", "customType": "regex", "managerFilePatterns": ["/^MODULE\\.bazel$/", "/^\\.bazelrc$/"], - "matchStrings": ["nightly/(?\\d{4}-\\d{2}-\\d{2})"], + "matchStrings": [ + "nightly/(?\\d{4}-\\d{2}-\\d{2})", + "rust_analyzer_nightly-(?\\d{4}-\\d{2}-\\d{2})_tools" + ], "depNameTemplate": "rust", "datasourceTemplate": "rust-version", "extractVersionTemplate": "^nightly-(?\\d{4}-\\d{2}-\\d{2})$", From 71fe47b48d43ad649e22c48d92f6b439323369d4 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Wed, 21 Oct 2026 00:10:42 -0700 Subject: [PATCH 3/9] fix(memory): reserve text builds below the admission ceiling Text admission sized from shared cgroup headroom, then reserved an extra watermark band against that same room. A 1.5 GiB build was refused with more than 1.5 GiB available because the reserve requested another 0.6 GiB. Add ceiling-aware reservation to the canonical process authority and use it for text artifacts. The ledger and measured process still obey the watermark, while the cgroup is charged only for the actual allocation. Remove the synthetic margin charge and its immediate shrink. Exercise actual concurrent reservations, a cgroup-constrained admission, and text builds with exactly their required cgroup headroom. Native regressions run in CI; local rustfmt and diff checks pass. --- .../src/code_index_scheduler.rs | 3 +- .../src/code_index_scheduler/serving.rs | 98 ++++++------------- .../tests/code_index_scheduler/serving.rs | 89 ++++++++++------- .../src/resident_memory.rs | 54 +++++++--- .../src/resident_memory/tests.rs | 39 ++++++++ 5 files changed, 168 insertions(+), 115 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler.rs index baca65a33f..a325e59813 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler.rs @@ -300,8 +300,7 @@ pub use serving::{ use serving::{ CodeIndexCommittedProgressSampleV1, CodeTextProjectionSlotV1, map_sealed_page_source_error, sha256_private_file_and_size, text_artifact_admitted_build_budget, - text_artifact_builder_budget, text_artifact_resident_memory_charge, - text_artifact_source_batch_limits, + text_artifact_builder_budget, text_artifact_source_batch_limits, }; #[cfg(test)] use tracedecay_query::retrieval::ports::TEXT_ARTIFACT_PAGE_CHUNKS_V1; diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/serving.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/serving.rs index de14bd949d..26a77bcc2d 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/serving.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/serving.rs @@ -947,29 +947,6 @@ pub struct DaemonCodeTextArtifactStoreV1 { worktree_id: WorktreeId, } -/// The bytes one artifact admission reserves: its ceiling plus the band above -/// the admission watermark, so the reserve call enforces that watermark -/// against both the ledger and the measured resident set. The band is not -/// memory the artifact owns and is released right after the reserve; -/// retaining it in every overlapping build would charge the same -/// process-wide margin repeatedly. The process's live set is not charged: -/// the reserve already compares the request with the measured resident set, -/// which contains it. -pub(super) fn text_artifact_resident_memory_charge( - requested: NonZeroU64, - watermark_headroom: u64, -) -> Result { - requested - .get() - .checked_add(watermark_headroom) - .and_then(NonZeroU64::new) - .ok_or_else(|| { - RetrievalPortError::Contract( - "text-artifact resident-memory accounting overflowed".to_owned(), - ) - }) -} - pub(super) fn text_artifact_admitted_build_budget( preferred_bytes: u64, minimum_bytes: u64, @@ -1170,10 +1147,9 @@ impl DaemonCodeTextArtifactStoreV1 { } } - /// Reserve one artifact memory ceiling. The atomic reserve also includes - /// the process-wide high-watermark headroom, then releases that - /// check-only margin before returning while the component ceiling stays - /// charged. + /// Reserve one artifact memory ceiling below the process-wide admission + /// watermark. Only the component's bytes are charged to the ledger and + /// checked against shared cgroup headroom. fn reserve_resident_memory( &self, generation_id: &CodeGenerationId, @@ -1184,20 +1160,14 @@ impl DaemonCodeTextArtifactStoreV1 { .map(|(reservation, _)| reservation) } - /// Measure headroom and size the build: `(observed, watermark headroom, - /// admitted)` bytes. + /// Measure headroom and size the build below the admission watermark. fn text_artifact_admission( &self, preferred: NonZeroU64, minimum: NonZeroU64, - ) -> Result<(u64, u64), RetrievalPortError> { + ) -> Result { let admission_watermark = self.resident_memory.admission_watermark_bytes(); let headroom = self.resident_memory.headroom_below(admission_watermark); - let watermark_headroom = self - .resident_memory - .snapshot() - .limit_bytes - .saturating_sub(admission_watermark); let admitted_bytes = text_artifact_admitted_build_budget( preferred.get(), minimum.get(), @@ -1207,7 +1177,7 @@ impl DaemonCodeTextArtifactStoreV1 { self.resident_memory .wait_for_headroom(minimum.get(), admission_watermark); })?; - Ok((watermark_headroom, admitted_bytes)) + Ok(admitted_bytes) } fn reserve_resident_memory_up_to( @@ -1235,56 +1205,50 @@ impl DaemonCodeTextArtifactStoreV1 { "text-artifact minimum resident-memory reservation must be nonzero".to_owned(), ) })?; - let (watermark_headroom, admitted_bytes) = - match self.text_artifact_admission(preferred, minimum) { - Err(RetrievalPortError::ResidentMemoryRefused(detail)) => { - let released = self - .resident_owners - .shed(minimum.get(), std::time::Instant::now()); - for release in &released { - log_resident_owner_release_v1(release); - } - // Freed pages sit in the pool workers' allocator heaps - // whether or not an owner was shed. - let trim = release_process_allocator_memory_v1(); - tracing::info!( - event = "code_text_artifact_build_shed_retained_state", - released_owners = released.len(), - trimmed_bytes = trim.released_bytes(), - refused = %detail, - "text-artifact build shed retained state before re-measuring headroom" - ); - self.text_artifact_admission(preferred, minimum) + let admitted_bytes = match self.text_artifact_admission(preferred, minimum) { + Err(RetrievalPortError::ResidentMemoryRefused(detail)) => { + let released = self + .resident_owners + .shed(minimum.get(), std::time::Instant::now()); + for release in &released { + log_resident_owner_release_v1(release); } - admission => admission, - }?; + // Freed pages sit in the pool workers' allocator heaps + // whether or not an owner was shed. + let trim = release_process_allocator_memory_v1(); + tracing::info!( + event = "code_text_artifact_build_shed_retained_state", + released_owners = released.len(), + trimmed_bytes = trim.released_bytes(), + refused = %detail, + "text-artifact build shed retained state before re-measuring headroom" + ); + self.text_artifact_admission(preferred, minimum) + } + admission => admission, + }?; let admitted = NonZeroU64::new(admitted_bytes).ok_or_else(|| { RetrievalPortError::Contract( "text-artifact admitted resident-memory reservation must be nonzero".to_owned(), ) })?; - let accounted = text_artifact_resident_memory_charge(admitted, watermark_headroom)?; - let mut reservation = self + let reservation = self .resident_memory - .reserve( + .reserve_below( ResidentMemoryKeyV1 { project_id: self.project_id.clone(), worktree_id: self.worktree_id.clone(), generation_id: generation_id.clone(), component, }, - accounted, + admitted, + self.resident_memory.admission_watermark_bytes(), ) .map_err(|error| { RetrievalPortError::ResidentMemoryRefused(format!( "text-artifact resident-memory admission was refused: {error}" )) })?; - reservation.shrink_to(admitted.get()).map_err(|error| { - RetrievalPortError::Contract(format!( - "text-artifact resident-memory headroom release failed: {error}" - )) - })?; let admitted = usize::try_from(admitted.get()).map_err(|error| { RetrievalPortError::Contract(format!( "text-artifact admitted reservation exceeds the platform limit: {error}" diff --git a/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/serving.rs b/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/serving.rs index 05bf0e812e..6e9b36212f 100644 --- a/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/serving.rs +++ b/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/serving.rs @@ -2087,41 +2087,53 @@ fn reader_reservation_refusal_precedes_missing_artifact_access() { #[test] fn overlapping_text_builds_share_one_admission_watermark_headroom() { - let limit_bytes = 1_000_u64; - let watermark_headroom = 100_u64; - let requested = NonZeroU64::new(300).expect("nonzero build request"); - let mut used_bytes = 0_u64; + use tracedecay_domain::WorktreeId; + use tracedecay_runtime_core::resident_memory::{ + ResidentMemoryAdmissionFailureV1, ResidentMemoryComponentIdV1, ResidentMemoryKeyV1, + }; - for _ in 0..3 { - let accounted = - super::super::text_artifact_resident_memory_charge(requested, watermark_headroom) - .expect("bounded admission accounting"); - assert!( - used_bytes + accounted.get() <= limit_bytes, - "each overlapping build fits beneath the same 900-byte high watermark" - ); - used_bytes += requested.get(); - } + let limit = NonZeroU64::new(1_000).expect("nonzero process ceiling"); + let authority = Arc::new(ProcessResidentMemoryV1::with_pressure( + limit, + Arc::new(ResidentMemoryPressureV1::new(limit)), + )); + let key = ResidentMemoryKeyV1 { + project_id: ProjectId::new("project.overlapping-text-builds").expect("project"), + worktree_id: WorktreeId::new("worktree.overlapping-text-builds").expect("worktree"), + generation_id: CodeGenerationId::new("generation.overlapping-text-builds") + .expect("generation"), + component: ResidentMemoryComponentIdV1::new("code-text-artifact-build").expect("component"), + }; + let requested = NonZeroU64::new(300).expect("nonzero build request"); + let builds = [(); 3].map(|()| { + authority + .reserve_below( + key.clone(), + requested, + authority.admission_watermark_bytes(), + ) + .expect("each build fits beneath the shared 900-byte watermark") + }); assert_eq!( - used_bytes, 900, + authority.snapshot().used_bytes, + 900, "the retained ledger owns the three build ceilings, not three margins" ); - for overflow in [ - super::super::text_artifact_resident_memory_charge( - NonZeroU64::new(u64::MAX).expect("maximum nonzero request"), - 1, + assert!(matches!( + authority.reserve_below( + key, + NonZeroU64::new(100).expect("remaining hard-limit room"), + authority.admission_watermark_bytes(), ), - super::super::text_artifact_resident_memory_charge(requested, u64::MAX), - ] { - assert!( - matches!( - overflow, - Err(tracedecay_query::retrieval::RetrievalPortError::Contract(_)) - ), - "overflow must remain a typed contract refusal: {overflow:?}" - ); - } + Err(ResidentMemoryAdmissionFailureV1::ReservationCeiling { + used_bytes: 900, + requested_bytes: 100, + limit_bytes: 900, + }) + )); + drop(builds); + assert_eq!(authority.snapshot().used_bytes, 0); } #[test] @@ -2199,7 +2211,14 @@ fn text_artifact_ceilings_reserve_through_process_resident_memory() { RESIDENT_MEMORY_PRESSURE_HIGH_WATERMARK_PERMILLE_V1, ); let fitting_live = watermark - build_floor; - for (live_bytes, admitted) in [(fitting_live, true), (fitting_live + 1, false)] { + for (live_bytes, cgroup_headroom, admitted) in [ + (fitting_live, None, true), + (fitting_live + 1, None, false), + // Shared cgroup room must cover the actual build, not a second copy + // of the band above the process admission watermark. + (0, Some(build_floor), true), + (0, Some(build_floor - 1), false), + ] { let pressure = Arc::new(ResidentMemoryPressureV1::with_sampler( measured_limit, Arc::new(move || { @@ -2207,7 +2226,7 @@ fn text_artifact_ceilings_reserve_through_process_resident_memory() { resident_bytes: live_bytes, unreclaimable_bytes: live_bytes, swapped_bytes: 0, - cgroup_headroom_bytes: None, + cgroup_headroom_bytes: cgroup_headroom, }) }), )); @@ -2224,11 +2243,13 @@ fn text_artifact_ceilings_reserve_through_process_resident_memory() { let latest = scheduler .latest_complete() .expect("measured latest generation"); - let advanced = latest.advance_text_serving(1); + // Open the build and its reservation without consuming source pages, + // so every case tests admission of the same unpublished artifact. + let advanced = latest.advance_text_serving(0); if admitted { assert!( advanced.is_ok(), - "{live_bytes} B live leaves the build floor below the watermark: {advanced:?}" + "{live_bytes} B live with cgroup room {cgroup_headroom:?} admits the build floor: {advanced:?}" ); } else { assert!( @@ -2236,7 +2257,7 @@ fn text_artifact_ceilings_reserve_through_process_resident_memory() { advanced, Err(tracedecay_query::retrieval::RetrievalPortError::ResidentMemoryRefused(_)) ), - "{live_bytes} B live leaves less than the build floor: {advanced:?}" + "{live_bytes} B live with cgroup room {cgroup_headroom:?} leaves less than the build floor: {advanced:?}" ); assert_eq!( measured.snapshot().used_bytes, diff --git a/crates/tracedecay-runtime-core/src/resident_memory.rs b/crates/tracedecay-runtime-core/src/resident_memory.rs index 34a9c445be..606ac2e31b 100644 --- a/crates/tracedecay-runtime-core/src/resident_memory.rs +++ b/crates/tracedecay-runtime-core/src/resident_memory.rs @@ -1521,14 +1521,39 @@ impl ProcessResidentMemoryV1 { key: ResidentMemoryKeyV1, requested_bytes: NonZeroU64, ) -> Result { - if let Some(failure) = self.observed_over_budget_refusal(requested_bytes) { + self.reserve_below(key, requested_bytes, u64::MAX) + } + + /// Reserve below a caller's admission ceiling using the same occupancy + /// as [`Self::headroom_below`]. The ceiling constrains both the ledger and + /// the process measurement; shared cgroup capacity constrains only the + /// requested allocation. Reserving the band above the ceiling as fake + /// component bytes would charge that band against cgroup room as well. + #[tracing::instrument( + name = "runtime_core.resident.reserve_below", + level = "trace", + skip_all + )] + pub fn reserve_below( + self: &Arc, + key: ResidentMemoryKeyV1, + requested_bytes: NonZeroU64, + ceiling: u64, + ) -> Result { + if let Some(failure) = self.observed_over_budget_refusal_below(requested_bytes, ceiling) { return Err(self.refused(failure)); } - if let Some(reservation) = self.try_reserve(&key, requested_bytes) { + let ledger_ceiling = ceiling.min(self.limit_bytes.get()); + if let Some(reservation) = self.try_reserve(&key, requested_bytes, ledger_ceiling) { return Ok(reservation); } - Err(self.refused(self.admission_failure(requested_bytes))) + let failure = ResidentMemoryAdmissionFailureV1::ReservationCeiling { + used_bytes: self.lock_state().used_bytes, + requested_bytes: requested_bytes.get(), + limit_bytes: ledger_ceiling, + }; + Err(self.refused(failure)) } /// Reserves one process-shared component without fabricating a project, @@ -1637,10 +1662,11 @@ impl ProcessResidentMemoryV1 { self: &Arc, key: &ResidentMemoryKeyV1, requested_bytes: NonZeroU64, + ceiling: u64, ) -> Option { let mut state = self.lock_state(); let next_used = state.used_bytes.checked_add(requested_bytes.get())?; - if next_used > self.limit_bytes.get() { + if next_used > ceiling { return None; } state.used_bytes = next_used; @@ -1662,30 +1688,34 @@ impl ProcessResidentMemoryV1 { fn observed_over_budget_refusal( &self, requested_bytes: NonZeroU64, + ) -> Option { + self.observed_over_budget_refusal_below(requested_bytes, u64::MAX) + } + + fn observed_over_budget_refusal_below( + &self, + requested_bytes: NonZeroU64, + ceiling: u64, ) -> Option { let pressure = self.pressure.state(); - let observed_bytes = pressure.observed_bytes()?; + let measured_ceiling = ceiling.min(self.pressure.limit_bytes()); + let observed_bytes = self.pressure.observed_bytes_for_limit(measured_ceiling)?; let high_watermark_bytes = self.pressure.high_watermark_bytes(); if requested_bytes.get() <= RESIDENT_MEMORY_PRESSURE_ADMISSION_FLOOR_BYTES_V1 || (!pressure.is_over_budget() - && requested_bytes.get() - <= self.pressure.limit_bytes().saturating_sub(observed_bytes)) + && requested_bytes.get() <= measured_ceiling.saturating_sub(observed_bytes)) { return None; } Some(ResidentMemoryAdmissionFailureV1::ObservedOverBudget { observed_bytes, - limit_bytes: self.pressure.limit_bytes(), + limit_bytes: measured_ceiling, high_watermark_bytes, requested_bytes: requested_bytes.get(), floor_bytes: RESIDENT_MEMORY_PRESSURE_ADMISSION_FLOOR_BYTES_V1, }) } - fn admission_failure(&self, requested_bytes: NonZeroU64) -> ResidentMemoryAdmissionFailureV1 { - self.admission_failure_from_used(self.lock_state().used_bytes, requested_bytes) - } - fn admission_failure_from_used( &self, used_bytes: u64, diff --git a/crates/tracedecay-runtime-core/src/resident_memory/tests.rs b/crates/tracedecay-runtime-core/src/resident_memory/tests.rs index 557293b4ec..93e1948c3e 100644 --- a/crates/tracedecay-runtime-core/src/resident_memory/tests.rs +++ b/crates/tracedecay-runtime-core/src/resident_memory/tests.rs @@ -1335,6 +1335,45 @@ fn cgroup_headroom_respects_a_larger_but_fuller_ancestor() { ); } +#[test] +fn reservations_below_a_watermark_use_the_headroom_measurement() { + const MIB: u64 = 1024 * 1024; + let limit = bytes(800 * MIB); + let pressure = Arc::new(ResidentMemoryPressureV1::with_sampler( + limit, + Arc::new(|| { + Some(ProcessResidentSampleV1 { + resident_bytes: 100 * MIB, + unreclaimable_bytes: 100 * MIB, + swapped_bytes: 0, + cgroup_headroom_bytes: Some(200 * MIB), + }) + }), + )); + let authority = Arc::new(ProcessResidentMemoryV1::with_pressure(limit, pressure)); + let watermark = authority.admission_watermark_bytes(); + let headroom = authority.headroom_below(watermark); + assert_eq!(headroom.available_bytes, 200 * MIB); + let owner = key("project-a", "worktree-a", "generation-a", "text-build"); + let refused = authority + .reserve_below( + owner.clone(), + bytes(headroom.available_bytes + 1), + watermark, + ) + .expect_err("growth beyond shared cgroup headroom must be refused"); + assert!(refused.is_observed_over_budget()); + assert_eq!(authority.snapshot().used_bytes, 0); + + let held = authority + .reserve_below(owner, bytes(headroom.available_bytes), watermark) + .expect("the actual allocation fits without charging the 80 MiB watermark band"); + assert_eq!(held.reserved_bytes(), 200 * MIB); + assert_eq!(authority.snapshot().used_bytes, 200 * MIB); + drop(held); + assert_eq!(authority.snapshot().used_bytes, 0); +} + #[test] fn narrowed_budgets_and_waiters_share_cgroup_capacity_without_charging_peers() { const MIB: u64 = 1024 * 1024; From 5a37781ed4f7cad8b35301e878b9c2c21dcced8d Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Wed, 21 Oct 2026 00:21:18 -0700 Subject: [PATCH 4/9] fix(memory): size text reservations inside ledger admission Reuse the reviewed reserve_up_to implementation from 863e868d3894ac2c0d1f51248e8b35b176040ef0 so text builders can reserve a supported smaller budget when headroom changes. Keep sizing, charging, minimum-floor validation, and the bounded shedding retry on one admission path, then park only for the minimum working set. Remove the unused reserve_below facade. Preserve cgroup headroom and real artifact floor regressions, and carry the existing sized-admission and shared-ledger behavior tests. --- .../src/code_index_scheduler/serving.rs | 90 ++++++-------- .../tests/code_index_scheduler/serving.rs | 19 ++- .../src/resident_memory.rs | 108 +++++++++++------ .../src/resident_memory/tests.rs | 112 ++++++++++++++++-- 4 files changed, 218 insertions(+), 111 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/serving.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/serving.rs index 26a77bcc2d..e6a5be1c4a 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/serving.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/serving.rs @@ -5,7 +5,6 @@ mod family_report; use std::{ collections::{BTreeSet, VecDeque}, io::Read, - num::NonZeroU64, path::{Path, PathBuf}, sync::{ Arc, Condvar, Mutex, MutexGuard, OnceLock, PoisonError, RwLock, @@ -1147,9 +1146,8 @@ impl DaemonCodeTextArtifactStoreV1 { } } - /// Reserve one artifact memory ceiling below the process-wide admission - /// watermark. Only the component's bytes are charged to the ledger and - /// checked against shared cgroup headroom. + /// Charge the artifact's ceiling below the process admission watermark. + /// Sizing and charging share one measured headroom reading. fn reserve_resident_memory( &self, generation_id: &CodeGenerationId, @@ -1160,26 +1158,6 @@ impl DaemonCodeTextArtifactStoreV1 { .map(|(reservation, _)| reservation) } - /// Measure headroom and size the build below the admission watermark. - fn text_artifact_admission( - &self, - preferred: NonZeroU64, - minimum: NonZeroU64, - ) -> Result { - let admission_watermark = self.resident_memory.admission_watermark_bytes(); - let headroom = self.resident_memory.headroom_below(admission_watermark); - let admitted_bytes = text_artifact_admitted_build_budget( - preferred.get(), - minimum.get(), - headroom.available_bytes, - ) - .inspect_err(|_| { - self.resident_memory - .wait_for_headroom(minimum.get(), admission_watermark); - })?; - Ok(admitted_bytes) - } - fn reserve_resident_memory_up_to( &self, generation_id: &CodeGenerationId, @@ -1205,7 +1183,33 @@ impl DaemonCodeTextArtifactStoreV1 { "text-artifact minimum resident-memory reservation must be nonzero".to_owned(), ) })?; - let admitted_bytes = match self.text_artifact_admission(preferred, minimum) { + let admission_watermark = self.resident_memory.admission_watermark_bytes(); + let reserve = || { + let reservation = self + .resident_memory + .reserve_up_to( + ResidentMemoryKeyV1 { + project_id: self.project_id.clone(), + worktree_id: self.worktree_id.clone(), + generation_id: generation_id.clone(), + component, + }, + preferred, + admission_watermark, + ) + .map_err(|error| { + RetrievalPortError::ResidentMemoryRefused(format!( + "text-artifact resident-memory admission was refused: {error}" + )) + })?; + let admitted = text_artifact_admitted_build_budget( + preferred.get(), + minimum.get(), + reservation.reserved_bytes(), + )?; + Ok((reservation, admitted)) + }; + let (reservation, admitted) = match reserve() { Err(RetrievalPortError::ResidentMemoryRefused(detail)) => { let released = self .resident_owners @@ -1213,8 +1217,6 @@ impl DaemonCodeTextArtifactStoreV1 { for release in &released { log_resident_owner_release_v1(release); } - // Freed pages sit in the pool workers' allocator heaps - // whether or not an owner was shed. let trim = release_process_allocator_memory_v1(); tracing::info!( event = "code_text_artifact_build_shed_retained_state", @@ -1223,33 +1225,17 @@ impl DaemonCodeTextArtifactStoreV1 { refused = %detail, "text-artifact build shed retained state before re-measuring headroom" ); - self.text_artifact_admission(preferred, minimum) + reserve() + } + result => result, + } + .inspect_err(|error| { + if matches!(error, RetrievalPortError::ResidentMemoryRefused(_)) { + self.resident_memory + .wait_for_headroom(minimum.get(), admission_watermark); } - admission => admission, - }?; - let admitted = NonZeroU64::new(admitted_bytes).ok_or_else(|| { - RetrievalPortError::Contract( - "text-artifact admitted resident-memory reservation must be nonzero".to_owned(), - ) })?; - let reservation = self - .resident_memory - .reserve_below( - ResidentMemoryKeyV1 { - project_id: self.project_id.clone(), - worktree_id: self.worktree_id.clone(), - generation_id: generation_id.clone(), - component, - }, - admitted, - self.resident_memory.admission_watermark_bytes(), - ) - .map_err(|error| { - RetrievalPortError::ResidentMemoryRefused(format!( - "text-artifact resident-memory admission was refused: {error}" - )) - })?; - let admitted = usize::try_from(admitted.get()).map_err(|error| { + let admitted = usize::try_from(admitted).map_err(|error| { RetrievalPortError::Contract(format!( "text-artifact admitted reservation exceeds the platform limit: {error}" )) diff --git a/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/serving.rs b/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/serving.rs index 6e9b36212f..64b6aab9f1 100644 --- a/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/serving.rs +++ b/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/serving.rs @@ -32,7 +32,7 @@ use tracedecay_domain::{ PrivacyDomainId, ProjectId, ProviderEvaluationStateV1, PublicRetrieverStatus, QueryNormalizationRevision, RelationEdgeKindV1, RetrievalBudget, RetrievalRequest, RetrievalScope, RetrievalSnapshot, RetrieverKind, RetrieverOutcome, SanitizerRevision, - ScoreDomainId, SingleRootScopeV1, TemporalModeV1, UtcMicros, VectorWatermark, + ScoreDomainId, SingleRootScopeV1, TemporalModeV1, UtcMicros, VectorWatermark, WorktreeId, encode_lowercase_hex, sha256_hex_suffix, }; use tracedecay_query::retrieval::{ @@ -49,7 +49,8 @@ use tracedecay_query::retrieval::{ }; use tracedecay_runtime_core::resident_memory::{ DEFAULT_PROCESS_RESIDENT_MEMORY_LIMIT_V1, ProcessResidentMemoryV1, ProcessResidentSampleV1, - RESIDENT_MEMORY_PRESSURE_HIGH_WATERMARK_PERMILLE_V1, ResidentMemoryPressureV1, + RESIDENT_MEMORY_PRESSURE_HIGH_WATERMARK_PERMILLE_V1, ResidentMemoryAdmissionFailureV1, + ResidentMemoryComponentIdV1, ResidentMemoryKeyV1, ResidentMemoryPressureV1, resident_memory_watermark_bytes_v1, }; @@ -2087,15 +2088,13 @@ fn reader_reservation_refusal_precedes_missing_artifact_access() { #[test] fn overlapping_text_builds_share_one_admission_watermark_headroom() { - use tracedecay_domain::WorktreeId; - use tracedecay_runtime_core::resident_memory::{ - ResidentMemoryAdmissionFailureV1, ResidentMemoryComponentIdV1, ResidentMemoryKeyV1, - }; - let limit = NonZeroU64::new(1_000).expect("nonzero process ceiling"); let authority = Arc::new(ProcessResidentMemoryV1::with_pressure( limit, - Arc::new(ResidentMemoryPressureV1::new(limit)), + Arc::new(ResidentMemoryPressureV1::with_sampler( + limit, + Arc::new(|| None), + )), )); let key = ResidentMemoryKeyV1 { project_id: ProjectId::new("project.overlapping-text-builds").expect("project"), @@ -2107,7 +2106,7 @@ fn overlapping_text_builds_share_one_admission_watermark_headroom() { let requested = NonZeroU64::new(300).expect("nonzero build request"); let builds = [(); 3].map(|()| { authority - .reserve_below( + .reserve_up_to( key.clone(), requested, authority.admission_watermark_bytes(), @@ -2121,7 +2120,7 @@ fn overlapping_text_builds_share_one_admission_watermark_headroom() { "the retained ledger owns the three build ceilings, not three margins" ); assert!(matches!( - authority.reserve_below( + authority.reserve_up_to( key, NonZeroU64::new(100).expect("remaining hard-limit room"), authority.admission_watermark_bytes(), diff --git a/crates/tracedecay-runtime-core/src/resident_memory.rs b/crates/tracedecay-runtime-core/src/resident_memory.rs index 606ac2e31b..a16e42ca7d 100644 --- a/crates/tracedecay-runtime-core/src/resident_memory.rs +++ b/crates/tracedecay-runtime-core/src/resident_memory.rs @@ -1521,39 +1521,70 @@ impl ProcessResidentMemoryV1 { key: ResidentMemoryKeyV1, requested_bytes: NonZeroU64, ) -> Result { - self.reserve_below(key, requested_bytes, u64::MAX) + if let Some(failure) = self.observed_over_budget_refusal(requested_bytes) { + return Err(self.refused(failure)); + } + if let Some(reservation) = self.try_reserve(&key, requested_bytes) { + return Ok(reservation); + } + + Err(self.refused(self.admission_failure(requested_bytes))) } - /// Reserve below a caller's admission ceiling using the same occupancy - /// as [`Self::headroom_below`]. The ceiling constrains both the ledger and - /// the process measurement; shared cgroup capacity constrains only the - /// requested allocation. Reserving the band above the ceiling as fake - /// component bytes would charge that band against cgroup room as well. + /// Size and charge new work from the same measured headroom reading. + /// The returned ceiling can be smaller than `preferred_bytes`; consumers + /// with a minimum working set must release a reservation below that floor. + /// Concurrent ledger charges are checked under the charge's own lock. + /// The consumer registers its working-set minimum with `wait_for_headroom` + /// when the returned budget cannot support its next operation. #[tracing::instrument( - name = "runtime_core.resident.reserve_below", + name = "runtime_core.resident.reserve_up_to", level = "trace", skip_all )] - pub fn reserve_below( + pub fn reserve_up_to( self: &Arc, key: ResidentMemoryKeyV1, - requested_bytes: NonZeroU64, + preferred_bytes: NonZeroU64, ceiling: u64, ) -> Result { - if let Some(failure) = self.observed_over_budget_refusal_below(requested_bytes, ceiling) { - return Err(self.refused(failure)); - } - let ledger_ceiling = ceiling.min(self.limit_bytes.get()); - if let Some(reservation) = self.try_reserve(&key, requested_bytes, ledger_ceiling) { - return Ok(reservation); + self.pressure.measure_admission_bytes(); + let ceiling = ceiling.min(self.limit_bytes.get()); + let mut state = self.lock_state(); + let headroom = self.headroom_from_state(&state, ceiling); + let admitted = preferred_bytes + .get() + .min(headroom.available_bytes) + .min(ceiling.saturating_sub(state.used_bytes)); + if admitted == 0 { + let failure = if self.pressure.state().is_over_budget() + || headroom.observed_bytes >= ceiling.min(self.pressure.limit_bytes()) + { + ResidentMemoryAdmissionFailureV1::ObservedOverBudget { + observed_bytes: headroom.observed_bytes, + limit_bytes: self.pressure.limit_bytes(), + high_watermark_bytes: self.pressure.high_watermark_bytes(), + requested_bytes: preferred_bytes.get(), + floor_bytes: RESIDENT_MEMORY_PRESSURE_ADMISSION_FLOOR_BYTES_V1, + } + } else { + ResidentMemoryAdmissionFailureV1::ReservationCeiling { + used_bytes: state.used_bytes, + requested_bytes: preferred_bytes.get(), + limit_bytes: ceiling, + } + }; + drop(state); + return Err(failure); } - - let failure = ResidentMemoryAdmissionFailureV1::ReservationCeiling { - used_bytes: self.lock_state().used_bytes, - requested_bytes: requested_bytes.get(), - limit_bytes: ledger_ceiling, - }; - Err(self.refused(failure)) + // `admitted` is at most ceiling - used_bytes, so both ledger sums fit. + state.used_bytes += admitted; + *state.charges.entry(key.clone()).or_default() += admitted; + Ok(ResidentMemoryReservationV1 { + authority: Arc::clone(self), + key, + reserved_bytes: admitted, + }) } /// Reserves one process-shared component without fabricating a project, @@ -1621,12 +1652,20 @@ impl ProcessResidentMemoryV1 { /// [`Self::reserve`] holds them. pub fn headroom_below(&self, ceiling: u64) -> ResidentMemoryHeadroomV1 { self.pressure.measure_admission_bytes(); + self.headroom_from_state(&self.lock_state(), ceiling) + } + + fn headroom_from_state( + &self, + state: &ResidentMemoryStateV1, + ceiling: u64, + ) -> ResidentMemoryHeadroomV1 { let measured_ceiling = ceiling.min(self.pressure.limit_bytes()); let observed_bytes = self .pressure .observed_bytes_for_limit(measured_ceiling) .unwrap_or(0); - let used_bytes = self.lock_state().used_bytes; + let used_bytes = state.used_bytes; let available_bytes = if self.pressure.state().is_over_budget() { 0 } else { @@ -1662,11 +1701,10 @@ impl ProcessResidentMemoryV1 { self: &Arc, key: &ResidentMemoryKeyV1, requested_bytes: NonZeroU64, - ceiling: u64, ) -> Option { let mut state = self.lock_state(); let next_used = state.used_bytes.checked_add(requested_bytes.get())?; - if next_used > ceiling { + if next_used > self.limit_bytes.get() { return None; } state.used_bytes = next_used; @@ -1688,34 +1726,30 @@ impl ProcessResidentMemoryV1 { fn observed_over_budget_refusal( &self, requested_bytes: NonZeroU64, - ) -> Option { - self.observed_over_budget_refusal_below(requested_bytes, u64::MAX) - } - - fn observed_over_budget_refusal_below( - &self, - requested_bytes: NonZeroU64, - ceiling: u64, ) -> Option { let pressure = self.pressure.state(); - let measured_ceiling = ceiling.min(self.pressure.limit_bytes()); - let observed_bytes = self.pressure.observed_bytes_for_limit(measured_ceiling)?; + let observed_bytes = pressure.observed_bytes()?; let high_watermark_bytes = self.pressure.high_watermark_bytes(); if requested_bytes.get() <= RESIDENT_MEMORY_PRESSURE_ADMISSION_FLOOR_BYTES_V1 || (!pressure.is_over_budget() - && requested_bytes.get() <= measured_ceiling.saturating_sub(observed_bytes)) + && requested_bytes.get() + <= self.pressure.limit_bytes().saturating_sub(observed_bytes)) { return None; } Some(ResidentMemoryAdmissionFailureV1::ObservedOverBudget { observed_bytes, - limit_bytes: measured_ceiling, + limit_bytes: self.pressure.limit_bytes(), high_watermark_bytes, requested_bytes: requested_bytes.get(), floor_bytes: RESIDENT_MEMORY_PRESSURE_ADMISSION_FLOOR_BYTES_V1, }) } + fn admission_failure(&self, requested_bytes: NonZeroU64) -> ResidentMemoryAdmissionFailureV1 { + self.admission_failure_from_used(self.lock_state().used_bytes, requested_bytes) + } + fn admission_failure_from_used( &self, used_bytes: u64, diff --git a/crates/tracedecay-runtime-core/src/resident_memory/tests.rs b/crates/tracedecay-runtime-core/src/resident_memory/tests.rs index 93e1948c3e..10fe13f033 100644 --- a/crates/tracedecay-runtime-core/src/resident_memory/tests.rs +++ b/crates/tracedecay-runtime-core/src/resident_memory/tests.rs @@ -1336,7 +1336,7 @@ fn cgroup_headroom_respects_a_larger_but_fuller_ancestor() { } #[test] -fn reservations_below_a_watermark_use_the_headroom_measurement() { +fn sized_reservations_charge_only_the_measured_cgroup_headroom() { const MIB: u64 = 1024 * 1024; let limit = bytes(800 * MIB); let pressure = Arc::new(ResidentMemoryPressureV1::with_sampler( @@ -1355,18 +1355,8 @@ fn reservations_below_a_watermark_use_the_headroom_measurement() { let headroom = authority.headroom_below(watermark); assert_eq!(headroom.available_bytes, 200 * MIB); let owner = key("project-a", "worktree-a", "generation-a", "text-build"); - let refused = authority - .reserve_below( - owner.clone(), - bytes(headroom.available_bytes + 1), - watermark, - ) - .expect_err("growth beyond shared cgroup headroom must be refused"); - assert!(refused.is_observed_over_budget()); - assert_eq!(authority.snapshot().used_bytes, 0); - let held = authority - .reserve_below(owner, bytes(headroom.available_bytes), watermark) + .reserve_up_to(owner, bytes(headroom.available_bytes + 1), watermark) .expect("the actual allocation fits without charging the 80 MiB watermark band"); assert_eq!(held.reserved_bytes(), 200 * MIB); assert_eq!(authority.snapshot().used_bytes, 200 * MIB); @@ -1658,3 +1648,101 @@ fn allocator_reclaim_preserves_cgroup_pressure_until_sample_recovers() { .is_ok() ); } + +#[test] +fn sized_admission_remeasures_and_charges_the_supported_smaller_budget() { + const MIB: u64 = 1024 * 1024; + let measured = Arc::new(AtomicU64::new(300 * MIB)); + let sampled = Arc::clone(&measured); + let reads = Arc::new(AtomicU64::new(0)); + let sampled_reads = Arc::clone(&reads); + let pressure = Arc::new(ResidentMemoryPressureV1::with_sampler( + bytes(1000 * MIB), + Arc::new(move || { + sampled_reads.fetch_add(1, Ordering::SeqCst); + let observed = sampled.load(Ordering::SeqCst); + Some(ProcessResidentSampleV1 { + resident_bytes: observed, + unreclaimable_bytes: observed, + swapped_bytes: 0, + cgroup_headroom_bytes: None, + }) + }), + )); + let authority = Arc::new(ProcessResidentMemoryV1::with_pressure( + bytes(1000 * MIB), + Arc::clone(&pressure), + )); + let watermark = authority.admission_watermark_bytes(); + assert_eq!(watermark, 900 * MIB); + assert_eq!( + authority.headroom_below(watermark).available_bytes, + 600 * MIB + ); + measured.store(700 * MIB, Ordering::SeqCst); + let before = reads.load(Ordering::SeqCst); + let reservation = authority + .reserve_up_to( + key("project-a", "worktree-a", "generation-a", "text-build"), + bytes(600 * MIB), + watermark, + ) + .expect("the smaller build fits despite growth since planning"); + assert_eq!(reads.load(Ordering::SeqCst), before + 1); + assert_eq!(reservation.reserved_bytes(), 200 * MIB); + assert_eq!(authority.snapshot().used_bytes, 200 * MIB); + drop(reservation); + assert_eq!(authority.snapshot().used_bytes, 0); + + measured.store(950 * MIB, Ordering::SeqCst); + let refused = authority + .reserve_up_to( + key("project-a", "worktree-a", "generation-b", "text-build"), + bytes(600 * MIB), + watermark, + ) + .expect_err("sizing must still refuse measured pressure"); + assert!(refused.is_observed_over_budget()); + assert_eq!(authority.snapshot().used_bytes, 0); +} + +#[test] +fn sized_admissions_share_the_watermark_without_charging_its_margin() { + let pressure = Arc::new(ResidentMemoryPressureV1::with_sampler( + bytes(1000), + Arc::new(|| None), + )); + let authority = Arc::new(ProcessResidentMemoryV1::with_pressure( + bytes(1000), + pressure, + )); + let watermark = authority.admission_watermark_bytes(); + let owner = key("project-a", "worktree-a", "generation-a", "text-build"); + let mut reservations = (0..3) + .map(|_| { + authority + .reserve_up_to(owner.clone(), bytes(300), watermark) + .unwrap() + }) + .collect::>(); + assert_eq!(authority.snapshot().used_bytes, 900); + let refused = authority + .reserve_up_to(owner.clone(), bytes(300), watermark) + .unwrap_err(); + assert_eq!( + refused, + ResidentMemoryAdmissionFailureV1::ReservationCeiling { + used_bytes: 900, + requested_bytes: 300, + limit_bytes: 900, + } + ); + drop(reservations.pop()); + let maximum = authority + .reserve_up_to(owner, bytes(u64::MAX), watermark) + .expect("bounded sizing cannot overflow while reserving available bytes"); + assert_eq!(maximum.reserved_bytes(), 300); + assert_eq!(authority.snapshot().used_bytes, 900); + drop((reservations, maximum)); + assert_eq!(authority.snapshot().used_bytes, 0); +} From 365c3412546e779173b4440078eb4bc46e17a084 Mon Sep 17 00:00:00 2001 From: Zack Jackson Date: Wed, 21 Oct 2026 18:47:44 -0700 Subject: [PATCH 5/9] test(fs): verify batch barriers without disk latency ratios --- .../tracedecay-private-fs/tests/framed_log.rs | 60 ++++++++----------- 1 file changed, 25 insertions(+), 35 deletions(-) diff --git a/crates/tracedecay-private-fs/tests/framed_log.rs b/crates/tracedecay-private-fs/tests/framed_log.rs index a662770690..7e89393fd5 100644 --- a/crates/tracedecay-private-fs/tests/framed_log.rs +++ b/crates/tracedecay-private-fs/tests/framed_log.rs @@ -118,54 +118,44 @@ fn prepared_publish_is_private_and_readable_through_the_adapter() { } } -/// A batch makes its own members durable and nothing else: syncing one small -/// member must not wait for another writer's dirty data on the same -/// filesystem. Asserted as a latency class against the cost of flushing that -/// unrelated data afterwards, on disk under the target directory (a tmpfs -/// would make both free). +/// Observe the barriers themselves: a filesystem can coalesce unrelated +/// writeback with a member's fsync, so their relative elapsed times cannot +/// establish which files the batch requested to flush. #[cfg(target_os = "linux")] #[test] -fn durable_batch_sync_is_sized_to_its_members_not_the_filesystem() { - // The Bazel test launcher passes its scratch dir at run time. - let target_tmp = std::env::var_os("CARGO_TARGET_TMPDIR") - .unwrap_or_else(|| env!("CARGO_TARGET_TMPDIR").into()); - let root = tempfile::tempdir_in(target_tmp).expect("batch fixture root"); - // Incompressible, so a compressing filesystem still has to write it. - let mut state = 0x9e37_79b9_7f4a_7c15_u64; - let chunk = (0..1 << 20) - .map(|_| { - state ^= state << 13; - state ^= state >> 7; - state ^= state << 17; - state.to_le_bytes()[0] - }) - .collect::>(); - let mut unrelated = fs::File::create(root.path().join("unrelated")).expect("unrelated file"); - for round in 0..128_u8 { - let mut block = chunk.clone(); - block[0] = round; - unrelated.write_all(&block).expect("unrelated dirty data"); - } - +fn durable_batch_sync_flushes_only_its_members() { + let root = tempfile::tempdir().expect("batch fixture root"); + let unrelated_path = root.path().join("unrelated"); + fs::write(&unrelated_path, b"unrelated dirty data").expect("unrelated file"); let member_path = root.path().join("member.tmp"); let mut member = fs::File::create(&member_path).expect("member file"); member.write_all(b"sealed segment").expect("member bytes"); let mut batch = DurableFileBatch::new(); batch.written(&member_path, &member).expect("record member"); - let started = std::time::Instant::now(); + let member_barriers = sync_latency::inject(&member_path, Duration::ZERO); + let unrelated_barriers = sync_latency::inject(&unrelated_path, Duration::ZERO); + let all_barriers = sync_latency::inject(root.path(), Duration::ZERO); batch.sync().expect("batch sync"); - let batch_sync = started.elapsed(); - let started = std::time::Instant::now(); - unrelated.sync_all().expect("unrelated sync"); - let unrelated_sync = started.elapsed(); + + assert_eq!(member_barriers.syncs(), 1, "the member must be flushed"); + assert_eq!( + unrelated_barriers.syncs(), + 0, + "unrelated data is not a member" + ); + assert_eq!(all_barriers.syncs(), 0, "no additional filesystem barriers"); assert_eq!( fs::read(&member_path).expect("member readback"), b"sealed segment" ); - assert!( - batch_sync * 4 < unrelated_sync, - "one-member batch sync took {batch_sync:?}; flushing the unrelated 128 MiB took {unrelated_sync:?}" + fs::remove_file(&member_path).expect("remove member before another sync"); + assert_eq!( + batch + .sync() + .expect_err("missing member must refuse durability") + .kind(), + ErrorKind::NotFound ); } From 73fc292cd2ddc5a752667a4d163d614f374a6a7f Mon Sep 17 00:00:00 2001 From: Zack Jackson Date: Wed, 21 Oct 2026 18:47:46 -0700 Subject: [PATCH 6/9] test(daemon): retry in-flight discovery during detached warmup --- crates/tracedecay/src/daemon/tests/bootstrap.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/crates/tracedecay/src/daemon/tests/bootstrap.rs b/crates/tracedecay/src/daemon/tests/bootstrap.rs index 8b59087386..3ad4967cdc 100644 --- a/crates/tracedecay/src/daemon/tests/bootstrap.rs +++ b/crates/tracedecay/src/daemon/tests/bootstrap.rs @@ -3711,8 +3711,9 @@ async fn direct_tool_cache_miss_returns_warming_while_project_opens_in_backgroun assert!(message.contains("retry"), "{message}"); // Retry the warming hint the way a client does. Each retry rides the - // detached open's own publication, and a retry that had to start a new - // open would show up as a second attempt. + // detached open's own publication. Its topology walk can still own the + // discovery slot, which is another typed retryable response. A retry that + // had to start a new open would show up as a second attempt. loop { match engine .project_server_for_request(&handshake, super::super::ProjectServerRequirement::Core) @@ -3720,7 +3721,11 @@ async fn direct_tool_cache_miss_returns_warming_while_project_opens_in_backgroun { Ok(_) => break, Err(error) => assert!( - super::super::error_is_project_warming(&error), + super::super::error_is_project_warming(&error) + || (super::super::error_is_repository_discovery_deferred(&error) + && error + .project_route_context() + .is_some_and(|(_, retryable, _)| retryable)), "the detached warm-up must publish, not fail: {error:?}" ), } From 16d3514f7a9fab1e7a495ac131e320debfd5eab2 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Wed, 21 Oct 2026 19:55:43 -0700 Subject: [PATCH 7/9] build(bazel): refresh private filesystem test metadata --- crates/tracedecay-private-fs/BUILD.bazel | 2 -- 1 file changed, 2 deletions(-) diff --git a/crates/tracedecay-private-fs/BUILD.bazel b/crates/tracedecay-private-fs/BUILD.bazel index b329dfdb6d..179ba3b0c6 100644 --- a/crates/tracedecay-private-fs/BUILD.bazel +++ b/crates/tracedecay-private-fs/BUILD.bazel @@ -53,7 +53,6 @@ rust_test( crate_features = ["test-helpers"], compile_data = glob(["assets/**", "data/**", "fixtures/**", "resources/**", "tests/**", "tests/fixtures/**", "vendor/**"], allow_empty = True), data = glob(["assets/**", "data/**", "fixtures/**", "resources/**", "tests/**", "tests/fixtures/**", "vendor/**"], allow_empty = True), - rustc_env = {"CARGO_TARGET_TMPDIR": "/tmp"}, ) rust_test( @@ -69,7 +68,6 @@ rust_test( crate_features = ["test-helpers"], compile_data = glob(["assets/**", "data/**", "fixtures/**", "resources/**", "tests/**", "tests/fixtures/**", "vendor/**"], allow_empty = True), data = glob(["assets/**", "data/**", "fixtures/**", "resources/**", "tests/**", "tests/fixtures/**", "vendor/**"], allow_empty = True), - rustc_env = {"CARGO_TARGET_TMPDIR": "/tmp"}, ) rust_test( From 166e3874ffa1c90c0518294e7e1d2201d902038d Mon Sep 17 00:00:00 2001 From: Zack Jackson Date: Wed, 21 Oct 2026 21:46:53 -0700 Subject: [PATCH 8/9] fix(bazel): use metadata dependencies for Clippy checks --- .bazelrc | 2 + MODULE.bazel | 2 + .../rules-rust-clippy-metadata-inputs.patch | 114 ++++++++++++++++++ 3 files changed, 118 insertions(+) create mode 100644 scripts/bazel/patches/rules-rust-clippy-metadata-inputs.patch diff --git a/.bazelrc b/.bazelrc index 130eccbb42..fe70ed2a12 100644 --- a/.bazelrc +++ b/.bazelrc @@ -64,6 +64,8 @@ test --default_test_resources=cpu=HOST_CPUS build:clippy --aspects=@rules_rust//rust:defs.bzl%rust_clippy_aspect # Lint outputs need dependency metadata, not linked test executables. build:clippy --output_groups=clippy_checks +# Produce dependency metadata so linting does not require full Rust libraries. +build:clippy --@rules_rust//rust/settings:pipelined_compilation=true build:clippy --@rules_rust//rust/settings:clippy_flag=-Dwarnings build:clippy --@rules_rust//rust/settings:clippy.toml=//:clippy.toml diff --git a/MODULE.bazel b/MODULE.bazel index e048bf9b5d..d642ecec34 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -18,6 +18,8 @@ single_version_override( "//scripts:bazel/patches/rules-rust-windows-file-url-path-deps.patch", # Genrule tools must include Rustfmt's compiler runtime libraries. "//scripts:bazel/patches/rules-rust-rustfmt-runtime-inputs.patch", + # No-link lint actions can use Rust metadata for executable targets. + "//scripts:bazel/patches/rules-rust-clippy-metadata-inputs.patch", ], patch_strip = 1, ) diff --git a/scripts/bazel/patches/rules-rust-clippy-metadata-inputs.patch b/scripts/bazel/patches/rules-rust-clippy-metadata-inputs.patch new file mode 100644 index 0000000000..98924c6cc1 --- /dev/null +++ b/scripts/bazel/patches/rules-rust-clippy-metadata-inputs.patch @@ -0,0 +1,114 @@ +diff --git a/rust/private/rustc.bzl b/rust/private/rustc.bzl +--- a/rust/private/rustc.bzl ++++ b/rust/private/rustc.bzl +@@ -645,7 +645,7 @@ + visited_libs[name] = artifact + return ambiguous_libs + +-def _depend_on_metadata(crate_info, force_depend_on_objects): ++def _depend_on_metadata(crate_info, force_depend_on_objects, force_depend_on_metadata = False): + """Determines if we can depend on metadata for this crate. + + By default (when pipelining is disabled or when the crate type needs to link against +@@ -658,6 +658,8 @@ + Args: + crate_info (CrateInfo): The Crate to determine this for. + force_depend_on_objects (bool): if set we will not depend on metadata. ++ force_depend_on_metadata (bool): A no-link action, such as Clippy, can use ++ metadata even when checking an executable or procedural macro. + + Returns: + Whether we can depend on metadata for this crate. +@@ -665,7 +667,7 @@ + if force_depend_on_objects: + return False + +- return crate_info.type in ("rlib", "lib") ++ return force_depend_on_metadata or crate_info.type in ("rlib", "lib") + + def collect_inputs( + ctx, +@@ -683,7 +685,8 @@ + force_depend_on_objects = False, + experimental_use_cc_common_link = False, + include_linker_inputs = False, +- include_link_flags = True): ++ include_link_flags = True, ++ force_depend_on_metadata = False): + """Gather's the inputs and required input information for a rustc action + + Args: +@@ -707,6 +710,8 @@ + rust binaries. + include_linker_inputs (bool, optional): Whether to include linker inputs in transitive dependencies. + include_link_flags (bool, optional): Whether to include flags like `-l` that instruct the linker to search for a library. ++ force_depend_on_metadata (bool, optional): Select metadata dependencies and ++ omit transitive native linker artifacts for a no-link action. + + Returns: + tuple: A tuple: A tuple of the following items: +@@ -747,7 +752,7 @@ + # flattened on each transitive rust_library dependency. + libs_from_linker_inputs = [] + ambiguous_libs = {} +- if crate_info.type not in ("lib", "rlib") or include_linker_inputs: ++ if (not force_depend_on_metadata and crate_info.type not in ("lib", "rlib")) or include_linker_inputs: + linker_inputs = dep_info.transitive_noncrates.to_list() + ambiguous_libs = _disambiguate_libs(ctx.actions, toolchain, crate_info, dep_info, use_pic) + libs_from_linker_inputs = _collect_libs_from_linker_inputs(linker_inputs, use_pic) + [ +@@ -762,7 +767,7 @@ + linkstamp_outs = [] + + transitive_crate_outputs = dep_info.transitive_crate_outputs +- if _depend_on_metadata(crate_info, force_depend_on_objects): ++ if _depend_on_metadata(crate_info, force_depend_on_objects, force_depend_on_metadata): + transitive_crate_outputs = dep_info.transitive_metadata_outputs + + dependency_search_artifacts = None +@@ -1009,7 +1014,8 @@ + error_format = None, + allowed_unstable_rust_features = None, + link_std_dylib = False, +- dependency_search_artifacts = None): ++ dependency_search_artifacts = None, ++ force_depend_on_metadata = False): + """Builds an Args object containing common rustc flags + + Args: +@@ -1078,6 +1084,8 @@ + use_json_output (bool): Have rustc emit json and process_wrapper parse json messages to output rendered output. + build_metadata (bool): Generate CLI arguments for building *only* .rmeta files. This requires use_json_output. + force_depend_on_objects (bool): Force using `.rlib` object files instead of metadata (`.rmeta`) files even if they are available. ++ force_depend_on_metadata (bool): Use metadata dependencies for an action ++ that checks an executable without linking it. + skip_expanding_rustc_env (bool): Whether to skip expanding CrateInfo.rustc_env_attr + require_explicit_unstable_features (bool): Whether to require all unstable features to be explicitly opted in to using `-Zallow-features=...`. + error_format (str, optional): Error format to pass to the `--error-format` command line argument. If set to None, uses the "_error_format" entry in `attr`. +@@ -1421,7 +1429,7 @@ + if crate_info.type == "dylib" and toolchain.target_os in ["macos", "darwin"]: + rustc_flags.add("--codegen=link-arg=-Wl,-install_name,@rpath/" + crate_info.output.basename) + +- use_metadata = _depend_on_metadata(crate_info, force_depend_on_objects) ++ use_metadata = _depend_on_metadata(crate_info, force_depend_on_objects, force_depend_on_metadata) + + # These always need to be added, even if not linking this crate. + add_crate_link_flags(rustc_flags, dep_info, force_all_deps_direct, use_metadata, dependency_search_artifacts) +diff --git a/rust/private/clippy.bzl b/rust/private/clippy.bzl +--- a/rust/private/clippy.bzl ++++ b/rust/private/clippy.bzl +@@ -161,6 +161,7 @@ + dep_info, + build_info, + lint_files, ++ force_depend_on_metadata = True, + ) + + if clippy_diagnostics_file: +@@ -187,6 +188,7 @@ + build_env_files = build_env_files, + build_flags_files = build_flags_files, + emit = ["dep-info", "metadata"], ++ force_depend_on_metadata = True, + skip_expanding_rustc_env = True, + use_json_output = bool(clippy_diagnostics_file), + error_format = error_format, From eab82a2e0cd924f8869cfda292be6a7154b128ad Mon Sep 17 00:00:00 2001 From: Zack Jackson Date: Thu, 22 Oct 2026 02:21:17 -0700 Subject: [PATCH 9/9] test(daemon): synchronize graph listing lock assertions --- .../tracedecay/src/daemon/tests/ownership.rs | 38 +++++++++++-------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/crates/tracedecay/src/daemon/tests/ownership.rs b/crates/tracedecay/src/daemon/tests/ownership.rs index bb4b4c8641..92d0043209 100644 --- a/crates/tracedecay/src/daemon/tests/ownership.rs +++ b/crates/tracedecay/src/daemon/tests/ownership.rs @@ -366,30 +366,36 @@ async fn mounted_graph_listing_takes_no_server_lease_while_it_reads() { .expect("mount project server"); let swap = server.graph_slot().write_owned().await; + let mut registry_gate = Some(engine.store_administration.project_servers().lock().await); let listing = engine.store_administration.mounted_project_graphs(); let mut listing = std::pin::pin!(listing); let handles = Arc::strong_count(&server); - assert!( - std::future::Future::poll( - listing.as_mut(), - &mut std::task::Context::from_waker(std::task::Waker::noop()), - ) - .is_pending(), - "the listing parks on the held graph read" - ); + let registry_lock = engine.store_administration.project_servers().lock(); + let mut registry_lock = std::pin::pin!(registry_lock); + // Force the first listing poll to wait for another registry user. Tokio's + // cooperative budget can also defer it before the held graph slot. + // Drive both futures with real wakeups: the fair registry mutex admits + // the listing before this later lock request, then must release it while + // the graph write guard still prevents the listing from completing. + let registry = tokio::time::timeout( + std::time::Duration::from_secs(5), + std::future::poll_fn(|context| { + assert!( + std::future::Future::poll(listing.as_mut(), context).is_pending(), + "the listing parks on the held graph read" + ); + drop(registry_gate.take()); + std::future::Future::poll(registry_lock.as_mut(), context) + }), + ) + .await + .expect("a parked graph listing must release the registry lock for route binds"); assert_eq!( Arc::strong_count(&server), handles, "a parked graph listing must hold no server handle" ); - assert!( - engine - .store_administration - .project_servers() - .try_lock() - .is_ok(), - "a parked graph listing must not hold the registry lock, or every route bind waits on the swap" - ); + drop(registry); drop(swap); assert_eq!(listing.await.len(), 1); }