diff --git a/contracts/embedder-api.md b/contracts/embedder-api.md index 540e55a..089fd60 100644 --- a/contracts/embedder-api.md +++ b/contracts/embedder-api.md @@ -464,6 +464,13 @@ Obligations: examples/guests/resource-stream, runtime/tests/embedder/resource_stream_test.ts. +**Owned future payloads** (`future>`) follow the same delivery rule: the +producer pump releases a successfully lowered value if the reader drops or the +write ends without consuming it. Once consumed, ownership belongs to the +receiver; a later pump failure must not dispose it again. Cleanup preserves an +existing producer/write error and reports a standalone disposal failure. This +rule covers top-level `own` payloads, not owns nested in composite values. + **Stream and future values survive round trips.** Lifting an end the host already handled — a host-created stream passed back, or a guest stream on its second hop — is idempotent, yielding a handle over the same end. Hence: diff --git a/docs/architecture.md b/docs/architecture.md index 37032ba..e756b37 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -473,14 +473,10 @@ already converted to misses need not produce a callback. Component validation failures still propagate. Correctly populated entries can be read from a read-only cache, with a translator available for misses. -**Current directory-cache limitation:** `put` preserves adapter names such -as `adapters/0.wasm` beneath its own `adapters/` directory but does not -create the additional nested directory. Normal adapter-bearing writes -therefore fail and `translateCached` returns fresh artifacts without -populating that entry. Adapter-free writes are not implicated, and -correctly laid-out entries can still be read. Verify actual -`fromCache: true` results after prewarming; successful translation alone -does not establish that the cache was populated. +The directory backend preserves validated adapter paths beneath its +`adapters/` directory and creates their parent directories when writing. +Verify actual `fromCache: true` results after prewarming: successful +translation alone does not establish that cache writes succeeded. **Engine code caches.** These are independent, opportunistic platform optimizations; correctness and artifact caching do not depend on them. diff --git a/docs/security.md b/docs/security.md index 83ebd64..1ae7e99 100644 --- a/docs/security.md +++ b/docs/security.md @@ -192,12 +192,9 @@ provider, not through other grants or a compromised host. **Prefer a pre-warmed, read-only cache for fixed deployments.** Translate in a trusted build step and deny the production process write access to the cache and application artifacts. **Verify actual hits after warming:** -the current `dirCache.put` does not create the nested directory required -by normal `adapters/.wasm` names. Those adapter-bearing writes fail, -but `translateCached` still returns a successful fresh translation. -Adapter-free writes are unaffected by this limitation, and correctly -laid-out entries can still be read. Check `fromCache: true` before relying -on prewarming; this documentation does not resolve the backend limitation. +`translateCached` returns successful fresh artifacts even if a cache write +fails. Check `fromCache: true` before relying on prewarming, and use +`onCacheError` to observe write failures. The intended permission split is: diff --git a/runtime/src/cache/dir.ts b/runtime/src/cache/dir.ts index 6dfe61b..871683b 100644 --- a/runtime/src/cache/dir.ts +++ b/runtime/src/cache/dir.ts @@ -136,7 +136,11 @@ class DirCache implements ArtifactCache { ); for (const [file, bytes] of artifacts.adapters) { const name = safeRelName(file); - await Deno.writeFile(`${tmp}/adapters/${name}`, bytes); + const path = `${tmp}/adapters/${name}`; + await Deno.mkdir(path.slice(0, path.lastIndexOf("/")), { + recursive: true, + }); + await Deno.writeFile(path, bytes); } await rmIfExists(dir); diff --git a/runtime/src/embedder/streams.ts b/runtime/src/embedder/streams.ts index ca46e59..7788819 100644 --- a/runtime/src/embedder/streams.ts +++ b/runtime/src/embedder/streams.ts @@ -645,11 +645,13 @@ export class Future implements ProtocolFuture { } cancel(): void { - if (this.#host !== null) this.#host.cancel(); - // A deferred future whose host end never materialized has nothing to - // cancel; swallow that rejection rather than let `cancel()` produce an - // unhandled one (issue #182 — mirrors `drop()` below). - else void this.#hostP.then((h) => h.cancel(), () => {}); + if (this.#host !== null) { + try { + this.#host.cancel(); + } catch { + // Pump failures remain recorded; public disposal is silent. + } + } else void this.#hostP.then((h) => h.cancel()).catch(() => {}); } /** @@ -668,11 +670,13 @@ export class Future implements ProtocolFuture { drop(): void { if (this.#dropped) return; this.#dropped = true; - if (this.#host !== null) this.#host.drop(); - // A deferred future whose host end never materialized has nothing to - // release; swallow that rejection rather than let `drop()` produce an - // unhandled one. - else void this.#hostP.then((h) => h.drop(), () => {}); + if (this.#host !== null) { + try { + this.#host.drop(); + } catch { + // Pump failures remain recorded; public disposal is silent. + } + } else void this.#hostP.then((h) => h.drop()).catch(() => {}); } /** @internal — see `Stream.dropForTeardown` (#66). */ @@ -964,7 +968,22 @@ export function lowerFutureSource( void (async () => { try { const v = await (src as PromiseLike); - await host.write(codec.fromHost(v) as unknown as T); + const lowered = codec.fromHost(v); + const info = codec.release === undefined ? undefined : { progress: 0 }; + try { + await host.write(lowered as unknown as T, info); + if (info?.progress === 0) { + throwIfPeerTrapped(host.value, codec.where ?? "future producer", 0); + } + } catch (e) { + try { + if (info?.progress === 0) codec.release?.(lowered); + } catch { + // Preserve the write failure if cleanup also fails. + } + throw e; + } + if (info?.progress === 0) codec.release?.(lowered); } catch (e) { // Report the producer cause rather than replace it with a generic // abandonment trap. A bound store receives the failure; only an unbound diff --git a/runtime/src/exec/host_streams.ts b/runtime/src/exec/host_streams.ts index d517b4e..0cda67d 100644 --- a/runtime/src/exec/host_streams.ts +++ b/runtime/src/exec/host_streams.ts @@ -1169,8 +1169,8 @@ export function hostStreamFor(value: ComponentValue): HostStream { } export interface HostFuture { - /** Deliver the future's single value. */ - write(value: T): Promise; + /** Deliver one value; optional internal output tracks source consumption, even on failure. */ + write(value: T, info?: { progress: number }): Promise; /** Await the future's single value. */ read(): Promise; /** @@ -1270,7 +1270,8 @@ function mkFuture( activity.notify(); }; const self: HostFuture = { - write(v: T): Promise { + write(v: T, info?: { progress: number }): Promise { + if (info !== undefined) info.progress = 0; // One operation per direction; opposite ends may rendezvous after a // guest round trip. This is a busy guard, not a delivered-value guard. if (parked.write) { @@ -1286,12 +1287,14 @@ function mkFuture( parked.write = true; try { shared.write(writeInst, buf as never, (result: CopyResult) => { + if (info !== undefined) info.progress = buf.progress; settle("write", result); resolve(); }); activity.notify(); activity.pump(); } catch (e) { + if (info !== undefined) info.progress = buf.progress; reject(e); withdraw("write", buf); } diff --git a/runtime/tests/cache_test.ts b/runtime/tests/cache_test.ts index bc2284a..171b766 100644 --- a/runtime/tests/cache_test.ts +++ b/runtime/tests/cache_test.ts @@ -6,6 +6,8 @@ // cargo build -p translator-shim --release --target wasm32-unknown-unknown // - examples/guests/build/{hello,test-suite}.component.wasm // ./examples/build.sh +// - harness/generated/values/transcode.0.wasm +// just corpus import { assertEq } from "./support/asserts.ts"; import { Translator } from "../src/shim/mod.ts"; @@ -119,6 +121,60 @@ Deno.test("dirCache: round-trip, cache hit skips the translator entirely", async await roundTrip(cache, "dirCache"); }); +Deno.test("dirCache: real adapters persist with full paths and execute on a warm hit", async () => { + const bytes = await readArtifact( + "harness/generated/values/transcode.0.wasm", + "just corpus", + ); + const dir = await tmpDir(); + try { + const spy = new SpyTranslator(await Translator.create(shimWasm)); + const errors: unknown[] = []; + const opts = { + onCacheError: (_op: "get" | "put", err: unknown) => errors.push(err), + }; + const first = await translateCached(spy, bytes, dirCache(dir), opts); + assertEq(first.fromCache, false, "cold miss"); + assertEq(spy.translateCalls, 1, "cold miss translates once"); + assert(first.adapters.size > 0, "fixture must emit real adapters"); + + const spy2 = new SpyTranslator(await Translator.create(shimWasm)); + const second = await translateCached(spy2, bytes, dirCache(dir), opts); + assertEq(second.fromCache, true, "fresh backend reads persisted artifacts"); + assertEq(spy2.translateCalls, 0, "warm hit never translates"); + assertEq(errors, [], "cache writes and reads succeed"); + assertEq(second.plan, first.plan, "cached plan matches translation"); + assertEq(second.adapters.size, first.adapters.size); + + const hex = await keyHex(await keyFor(spy, bytes)); + for (const [file, adapter] of first.adapters) { + assertEq( + second.adapters.get(file), + adapter, + "cached adapter bytes match", + ); + assertEq( + await Deno.readFile(`${dir}/${hex}/adapters/${file}`), + adapter, + "persisted layout preserves the full adapter name", + ); + } + + for (const result of [first, second]) { + const component = await instantiateComponent({ + plan: result.plan, + componentBytes: bytes, + adapters: result.adapters, + }); + const run = component.exports.run as () => number; + // The corpus guest checks transcoded string bytes on both sides. + assertEq(run(), 42, "fresh and cached adapters execute correctly"); + } + } finally { + await Deno.remove(dir, { recursive: true }); + } +}); + Deno.test("webCache: round-trip, cache hit skips the translator entirely", async () => { const cache = webCache(`artifact-cache-test-${crypto.randomUUID()}`); await roundTrip(cache, "webCache"); diff --git a/runtime/tests/embedder/future-disposal.wasm b/runtime/tests/embedder/future-disposal.wasm new file mode 100644 index 0000000..c422074 Binary files /dev/null and b/runtime/tests/embedder/future-disposal.wasm differ diff --git a/runtime/tests/embedder/future-disposal.wat b/runtime/tests/embedder/future-disposal.wat new file mode 100644 index 0000000..98b5d57 --- /dev/null +++ b/runtime/tests/embedder/future-disposal.wat @@ -0,0 +1,37 @@ +;; Reader abandonment legally wakes the parked writer; its callback then faults. +(component + (type $F (future u32)) + (core module $Mem (memory (export "mem") 1)) + (core instance $mem (instantiate $Mem)) + (canon future.new $F (core func $new)) + (canon future.write $F async (memory $mem "mem") (core func $write)) + (canon task.return (result $F) (core func $return)) + (canon waitable-set.new (core func $set)) + (canon waitable.join (core func $join)) + (core module $M + (import "" "new" (func $new (result i64))) + (import "" "write" (func $write (param i32 i32) (result i32))) + (import "" "return" (func $return (param i32))) + (import "" "set" (func $set (result i32))) + (import "" "join" (func $join (param i32 i32))) + (func (export "idle") (result i32) (i32.wrap_i64 (call $new))) + (func (export "run") (result i32) + (local $ends i64) (local $tx i32) (local $ws i32) + (local.set $ends (call $new)) + (local.set $tx (i32.wrap_i64 (i64.shr_u (local.get $ends) (i64.const 32)))) + (call $return (i32.wrap_i64 (local.get $ends))) + (if (i32.ne (call $write (local.get $tx) (i32.const 0)) (i32.const -1)) + (then unreachable)) + (local.set $ws (call $set)) + (call $join (local.get $tx) (local.get $ws)) + (i32.or (i32.const 2) (i32.shl (local.get $ws) (i32.const 4)))) + (func (export "cb") (param i32 i32 i32) (result i32) unreachable)) + (core instance $m (instantiate $M (with "" (instance + (export "new" (func $new)) + (export "write" (func $write)) + (export "return" (func $return)) + (export "set" (func $set)) + (export "join" (func $join)))))) + (func (export "run") async (result $F) + (canon lift (core func $m "run") async (callback (core func $m "cb")))) + (func (export "idle") (result $F) (canon lift (core func $m "idle")))) diff --git a/runtime/tests/embedder/future-own.wasm b/runtime/tests/embedder/future-own.wasm new file mode 100644 index 0000000..6a92736 Binary files /dev/null and b/runtime/tests/embedder/future-own.wasm differ diff --git a/runtime/tests/embedder/future-own.wat b/runtime/tests/embedder/future-own.wat new file mode 100644 index 0000000..f28dbfe --- /dev/null +++ b/runtime/tests/embedder/future-own.wat @@ -0,0 +1,31 @@ +;; Readable-end abandonment, delivery, and peer faults for future>. +(component + (import "r" (type $R (sub resource))) + (type $F (future (own $R))) + (core module $Mem (memory (export "mem") 1)) + (core instance $mem (instantiate $Mem)) + (canon future.drop-readable $F (core func $drop)) + (canon future.read $F async (memory $mem "mem") (core func $read)) + (canon resource.drop $R (core func $drop-r)) + (core module $M + (import "" "mem" (memory 1)) + (import "" "drop" (func $drop (param i32))) + (import "" "read" (func $read (param i32 i32) (result i32))) + (import "" "drop-r" (func $drop-r (param i32))) + (func (export "drop") (param i32) (call $drop (local.get 0))) + (func (export "pass") (param i32) (result i32) (local.get 0)) + (func (export "fail") (param i32) unreachable) + (func (export "take") (param i32 i32) + (if (call $read (local.get 0) (i32.const 0)) (then unreachable)) + (call $drop-r (i32.load (i32.const 0))) + (call $drop (local.get 0)) + (if (local.get 1) (then unreachable)))) + (core instance $m (instantiate $M (with "" (instance + (export "mem" (memory $mem "mem")) + (export "drop" (func $drop)) + (export "read" (func $read)) + (export "drop-r" (func $drop-r)))))) + (func (export "drop") (param "f" $F) (canon lift (core func $m "drop"))) + (func (export "pass") (param "f" $F) (result $F) (canon lift (core func $m "pass"))) + (func (export "fail") (param "f" $F) (canon lift (core func $m "fail"))) + (func (export "take") (param "f" $F) (param "trap" bool) (canon lift (core func $m "take")))) diff --git a/runtime/tests/embedder/future_result_test.ts b/runtime/tests/embedder/future_result_test.ts index 3d4fa83..db40a48 100644 --- a/runtime/tests/embedder/future_result_test.ts +++ b/runtime/tests/embedder/future_result_test.ts @@ -12,13 +12,177 @@ // returns. import { assertEq } from "../support/asserts.ts"; -import { guest, haveFixture, instantiateFixture } from "./support.ts"; -import { Future, Stream } from "../../src/embedder/streams.ts"; -import { hostFuture } from "../../src/exec/host_streams.ts"; +import { caught, guest, haveFixture, instantiateFixture } from "./support.ts"; +import { + Future, + lowerFutureSource, + Stream, +} from "../../src/embedder/streams.ts"; +import { hostFuture, hostFutureFor } from "../../src/exec/host_streams.ts"; import { HostResourceRegistry } from "../../src/embedder/resources.ts"; import { INTERNAL_HOST_REGISTRIES } from "../../src/embedder/instantiate.ts"; import { sync } from "../../src/embedder/sync.ts"; -import { Trap } from "@polyengine/protocol"; +import { StreamProducerError, Trap } from "@polyengine/protocol"; +import { SharedFutureImpl } from "../../src/task/mod.ts"; + +const turn = () => new Promise((resolve) => setTimeout(resolve, 0)); +const ownFixture = "runtime/tests/embedder/future-own.wasm"; +const ownReady = await haveFixture(ownFixture); +for ( + const mode of [ + "drop-before-source", + "drop-parked", + "cancel-parked", + "take", + "fail", + "take-fail", + ] as const +) { + Deno.test({ + name: `future own cleanup: ${mode}`, + ignore: !ownReady, + async fn() { + let disposed = 0; + class R { + [Symbol.dispose]() { + disposed++; + } + } + const c = await instantiateFixture(ownFixture, { r: R }); + const registry = + (c as unknown as Record>)[ + INTERNAL_HOST_REGISTRIES + ].get(0)!; + let resolve!: (value: R) => void; + const source = new Promise((r) => resolve = r); + if (mode === "drop-before-source") { + await c.exports.drop(source); + resolve(new R()); + } else { + const f = c.exports.pass(source) as Future; + resolve(new R()); + await turn(); + assertEq([disposed, registry.liveCount], [0, 1]); + if (mode === "cancel-parked") { + f.cancel(); + } else { + const error = await caught(() => + mode === "drop-parked" + ? c.exports.drop(f) + : mode === "fail" + ? c.exports.fail(f) + : c.exports.take(f, mode === "take-fail") + ); + assertEq( + error instanceof Error, + mode === "fail" || mode === "take-fail", + ); + } + } + await turn(); + assertEq([disposed, registry.liveCount], [1, 0]); + }, + }); +} + +Deno.test({ + name: + "future own cleanup: mapped destructor failure is recorded exactly once", + ignore: !ownReady, + async fn() { + for (const cleanupError of [undefined, new Error("own cleanup failed")]) { + let disposed = 0; + class R { + [Symbol.dispose]() { + disposed++; + throw cleanupError; + } + } + const c = await instantiateFixture(ownFixture, { r: R }); + const registry = + (c as unknown as Record>)[ + INTERNAL_HOST_REGISTRIES + ].get(0)!; + const f = c.exports.pass(Promise.resolve(new R())) as Future; + await turn(); + assertEq([disposed, registry.liveCount], [0, 1]); + assertEq(sync(c.exports.drop)(f), undefined); + await turn(); + assertEq([disposed, registry.liveCount], [1, 0]); + // Sync drop returns before pump cleanup; no export consumes the + // resulting producer-failure slot before this observation. + const reported = c.handle.componentInstances[0].store.hostFailure; + assertEq( + reported instanceof StreamProducerError, + true, + String(reported), + ); + assertEq((reported as StreamProducerError).cause, cleanupError); + f.drop(); + await turn(); + assertEq([disposed, registry.liveCount], [1, 0]); + } + }, +}); + +// Narrow seam tests: source consumption is not a transactional destination commit. +for ( + const stage of [ + "before-read", + "after-read", + "after-completion", + "drop", + ] as const +) { + for (const cleanupThrows of [false, true]) { + Deno.test(`future source accounting: ${stage}, cleanupThrows=${cleanupThrows}`, async () => { + const primary = new Error("injected write failure"); + let releases = 0; + const value = lowerFutureSource(Promise.resolve(7), { + element: { kind: "u32" }, + where: "test own future producer", + fromHost: (v: number) => v, + toHost: (v) => v as number, + release: (v) => { + assertEq(v, 7); + releases++; + if (cleanupThrows) throw undefined; + }, + }); + const shared = value as SharedFutureImpl; + // Observe producer reporting without a scheduler or fabricated pump. + const store: { hostFailure?: unknown } = {}; + shared.boundStore = store; + const write = shared.write; + if (stage === "drop") shared.drop(); + else {shared.write = (inst, src, done) => { + if (stage === "after-read") src.read(1); + if (stage === "after-completion") write.call(shared, inst, src, done); + throw primary; + };} + const host = hostFutureFor(value); + const reading = stage === "after-completion" ? host.read() : null; + await turn(); + if (reading !== null) assertEq(await reading, 7); + const consumed = stage === "after-read" || stage === "after-completion"; + assertEq(releases, consumed ? 0 : 1); + if ( + stage === "after-completion" || (stage === "drop" && !cleanupThrows) + ) { + assertEq(store.hostFailure, undefined); + } else { + assertEq(store.hostFailure instanceof StreamProducerError, true); + assertEq( + (store.hostFailure as StreamProducerError).cause, + stage === "drop" ? undefined : primary, + ); + } + host.drop(); + await turn(); + assertEq(releases, consumed ? 0 : 1, "no second release on disposal"); + }); + } +} const FIXTURE = guest("future-import"); const have = await haveFixture(FIXTURE); diff --git a/runtime/tests/embedder/streams_test.ts b/runtime/tests/embedder/streams_test.ts index 93b5cd2..48907e7 100644 --- a/runtime/tests/embedder/streams_test.ts +++ b/runtime/tests/embedder/streams_test.ts @@ -3,9 +3,13 @@ import { assertEq } from "../support/asserts.ts"; import { caught, guest, haveFixture, instantiateFixture } from "./support.ts"; -import { DroppedError, StreamProducerError } from "@polyengine/protocol"; +import { + DroppedError, + PeerTrappedError, + StreamProducerError, +} from "@polyengine/protocol"; import { Future, Stream } from "../../src/embedder/streams.ts"; -import { hostStream, hostStreamFor } from "../../src/exec/mod.ts"; +import { hostFuture, hostStream, hostStreamFor } from "../../src/exec/mod.ts"; import { LiftLowerContext, mkCanonicalOptions, @@ -411,10 +415,72 @@ Deno.test({ const dummyCodec = { element: null, toHost: (v: unknown) => v as number, - fromHost: (v: number) => v as unknown, + fromHost: (v: number) => v, where: "test future", }; +const disposalFixture = "runtime/tests/embedder/future-disposal.wasm"; +const disposalReady = await haveFixture(disposalFixture); +const turn = () => new Promise((resolve) => setTimeout(resolve, 0)); + +for (const jspi of [undefined, false, true]) { + for (const deferred of [false, true]) { + Deno.test({ + name: `public future disposal: jspi=${jspi}, deferred=${deferred}`, + ignore: !disposalReady, + async fn() { + for (const method of ["drop", Symbol.dispose] as const) { + const c = await instantiateFixture(disposalFixture, {}, { jspi }); + const f = c.exports.run() as Future; + if (!deferred) await turn(); + assertEq(f[method](), undefined); + assertEq(f[method](), undefined); + await turn(); // Deno rejects any unhandled derived disposal promise. + const error = await caught(() => Promise.resolve(f)); + assertEq(error instanceof PeerTrappedError, true, String(error)); + assertEq(await caught(() => Promise.resolve(f)), error); + } + }, + }); + } + Deno.test({ + name: `public future cancellation: jspi=${jspi}`, + ignore: !disposalReady, + async fn() { + const c = await instantiateFixture(disposalFixture, {}, { jspi }); + const f = c.exports.idle() as Future; + const reading = caught(() => Promise.resolve(f)); + await turn(); + assertEq(f.cancel(), undefined); + assertEq(f.cancel(), undefined); + const error = await reading; + assertEq(error instanceof DroppedError, true); + assertEq(String(error).includes("cancelled"), true); + f.drop(); + }, + }); +} + +Deno.test("future cancel boundary: injected raw failure stays silent, bound and deferred", async () => { + for (const deferred of [false, true]) { + const h = hostFuture(null); + let calls = 0; + h.cancel = () => { + calls++; + throw new Error("injected cancel failure"); + }; + const f = deferred + ? Future.deferred(Promise.resolve(h.value), dummyCodec) + : Future.fromHostFuture(h, dummyCodec); + assertEq(f.cancel(), undefined); + await turn(); + assertEq(calls, 1); + assertEq(f.cancel(), undefined); + assertEq(calls, 2); + f.drop(); + } +}); + Deno.test({ name: "futures (#182): cancel() on a deferred future whose producing call rejected does not raise an unhandled rejection",