diff --git a/.changeset/fix-arraybuffer-serialization.md b/.changeset/fix-arraybuffer-serialization.md new file mode 100644 index 0000000..3102890 --- /dev/null +++ b/.changeset/fix-arraybuffer-serialization.md @@ -0,0 +1,5 @@ +--- +"capnweb": minor +--- + +Support exact ArrayBuffer, DataView, and typed array serialization over RPC. diff --git a/README.md b/README.md index 21fb10b..02434c1 100644 --- a/README.md +++ b/README.md @@ -199,7 +199,7 @@ The following types can be passed over RPC (in arguments or return values), and * Arrays * `bigint` * `Date` -* `Uint8Array` +* `ArrayBuffer`, `DataView`, and typed arrays * `Error` and its well-known subclasses * `Blob` * `ReadableStream` and `WritableStream`, with automatic flow control. @@ -207,7 +207,6 @@ The following types can be passed over RPC (in arguments or return values), and The following types are not supported as of this writing, but may be added in the future: * `Map` and `Set` -* `ArrayBuffer` and typed arrays other than `Uint8Array` * `RegExp` The following are intentionally NOT supported: diff --git a/__tests__/index.test.ts b/__tests__/index.test.ts index 486c029..0d28f20 100644 --- a/__tests__/index.test.ts +++ b/__tests__/index.test.ts @@ -7,6 +7,7 @@ import { deserialize, serialize, RpcSession, type RpcSessionOptions, RpcTranspor type RpcTransportWithCustomEncoding, RpcTarget, RpcStub, newWebSocketRpcSession, newMessagePortRpcSession, newHttpBatchRpcSession} from "../src/index.js" +import { swapByteOrder } from "../src/serialize.js" import { MAX_CLOSE_REASON_BYTES } from "../src/websocket.js" import { Counter, TestTarget } from "./test-util.js"; @@ -164,6 +165,22 @@ describe("simple serialization", () => { expect(new Uint8Array(deserialized)).toStrictEqual(bytes); }) + it("can serialize Uint8Array as legacy bytes without a type marker", () => { + let bytes = new Uint8Array([72, 101, 108, 108, 111]); + let serialized = serialize(bytes); + expect(serialized).toBe('["bytes","SGVsbG8"]'); + + let deserialized = deserialize(serialized) as Uint8Array; + expect(deserialized).toBeInstanceOf(Uint8Array); + expect(new Uint8Array(deserialized)).toStrictEqual(bytes); + + // Accept the explicit marker from other implementations, while continuing + // to emit the markerless form for backwards compatibility. + let explicitlyTyped = deserialize('["bytes","SGVsbG8","Uint8Array"]'); + expect(Object.getPrototypeOf(explicitlyTyped)).toBe(Uint8Array.prototype); + expect(explicitlyTyped).toStrictEqual(bytes); + }) + it("can serialize Node.js Buffer as bytes", () => { if (typeof Buffer === "undefined") return; // skip in browsers let buf = Buffer.from("hello!"); @@ -174,6 +191,110 @@ describe("simple serialization", () => { expect(new Uint8Array(deserialized)).toStrictEqual(new Uint8Array(buf)); }) + it("can serialize ArrayBuffer as bytes with an ArrayBuffer marker", () => { + let bytes = new Uint8Array([72, 101, 108, 108, 111]); + let serialized = serialize(bytes.buffer); + expect(serialized).toBe('["bytes","SGVsbG8","ArrayBuffer"]'); + + let deserialized = deserialize(serialized); + expect(deserialized).toBeInstanceOf(ArrayBuffer); + expect(new Uint8Array(deserialized as ArrayBuffer)).toStrictEqual(bytes); + }) + + it("can serialize typed array views as bytes with type markers", () => { + let cases = [ + { + name: "DataView", + elementSize: 1, + makeView: (buffer: ArrayBuffer, offset: number, byteLength: number) => + new DataView(buffer, offset, byteLength), + }, + ...[ + Int8Array, + Uint8ClampedArray, + Int16Array, + Uint16Array, + Int32Array, + Uint32Array, + BigInt64Array, + BigUint64Array, + Float32Array, + Float64Array, + ].map(Type => ({ + name: Type.name, + elementSize: Type.BYTES_PER_ELEMENT, + makeView: (buffer: ArrayBuffer, offset: number, byteLength: number) => + new Type(buffer, offset, byteLength / Type.BYTES_PER_ELEMENT), + })), + ]; + + for (let {name, elementSize, makeView} of cases) { + // Use a non-zero offset and extra trailing byte to verify only the view's visible byte range + // is serialized, not the whole backing buffer. + let byteLength = elementSize * 2; + let offset = elementSize; + let backing = new ArrayBuffer(offset + byteLength + 1); + let bytes = new Uint8Array(byteLength); + for (let i = 0; i < bytes.length; i++) bytes[i] = i + 1; + new Uint8Array(backing, offset, byteLength).set(bytes); + + let view = makeView(backing, offset, byteLength); + let serialized = serialize(view); + let parsed = JSON.parse(serialized) as [string, string, string]; + expect(parsed[0]).toBe("bytes"); + expect(parsed[2]).toBe(name); + + let deserialized = deserialize(serialized) as ArrayBufferView; + expect(Object.getPrototypeOf(deserialized)).toBe(Object.getPrototypeOf(view)); + expect(new Uint8Array( + deserialized.buffer, deserialized.byteOffset, deserialized.byteLength)).toStrictEqual(bytes); + } + }) + + it("serializes multi-byte numbers in little-endian wire order", () => { + let serialized = serialize(new Uint32Array([0x01020304])); + let parsed = JSON.parse(serialized) as [string, string, string]; + expect(parsed[2]).toBe("Uint32Array"); + let wireBytes = Uint8Array.from(atob(parsed[1]), c => c.charCodeAt(0)); + expect([...wireBytes]).toStrictEqual([0x04, 0x03, 0x02, 0x01]); + }) + + it("can swap each multi-byte element's byte order", () => { + for (let elementSize of [2, 4, 8]) { + let bytes = Uint8Array.from( + { length: elementSize * 3 }, (_, index) => (index * 37 + 1) & 0xff); + let expected = bytes.slice(); + for (let offset = 0; offset < expected.length; offset += elementSize) { + expected.subarray(offset, offset + elementSize).reverse(); + } + + swapByteOrder(bytes, elementSize); + expect(bytes).toStrictEqual(expected); + } + + expect(() => swapByteOrder(new Uint8Array(), 3)).toThrowError( + "Unsupported element size: 3"); + }) + + it("rejects byte lengths that are misaligned for typed arrays", () => { + for (let [type, byteLength, elementSize] of [ + ["Int16Array", 1, 2], + ["Float32Array", 3, 4], + ["BigUint64Array", 7, 8], + ] as const) { + let base64 = btoa("\0".repeat(byteLength)); + expect(() => deserialize(`["bytes","${base64}","${type}"]`)).toThrowError( + `Invalid byte length ${byteLength} for ${type}; expected a multiple of ${elementSize}`); + } + }) + + it("throws for unknown bytes type markers", () => { + expect(() => deserialize('["bytes","SGVsbG8","invalidUint8Array"]')).toThrowError( + "Unknown bytes type marker: invalidUint8Array"); + expect(() => deserialize('["bytes","SGVsbG8",123]')).toThrowError( + "Unknown bytes type marker type: number"); + }) + it("preserves Invalid Date values through serialization", () => { let invalidDate = new Date(NaN); let serialized = serialize(invalidDate); @@ -2695,6 +2816,10 @@ describe("transport encoding levels", () => { let bytes = await stub.echo(new Uint8Array([1, 2, 3])) as Uint8Array; expect(new Uint8Array(bytes)).toStrictEqual(new Uint8Array([1, 2, 3])); + let typedArray = await stub.echo(new Uint32Array([0x01020304, 0xa0b0c0d0])) as Uint32Array; + expect(Object.getPrototypeOf(typedArray)).toBe(Uint32Array.prototype); + expect(typedArray).toStrictEqual(new Uint32Array([0x01020304, 0xa0b0c0d0])); + let date = await stub.echo(new Date(1234567890)) as Date; expect(date).toBeInstanceOf(Date); expect(date.getTime()).toBe(1234567890); diff --git a/protocol.md b/protocol.md index e3c9d99..c6c8abd 100644 --- a/protocol.md +++ b/protocol.md @@ -168,9 +168,15 @@ The literal value `undefined`. The values Infinity, -Infinity, and NaN. -`["bytes", base64]` - -A `Uint8Array`, represented as a base64-encoded string. +`["bytes", base64]`, `["bytes", base64, type]` + +A byte container, represented as a base64-encoded string. If `type` is omitted, the receiver +should deserialize bytes as its default `Uint8Array` for backwards compatibility. Otherwise, +`type` preserves the byte container type across the wire. The supported `type` values are +`ArrayBuffer`, `DataView`, `Int8Array`, `Uint8Array`, `Uint8ClampedArray`, +`Int16Array`, `Uint16Array`, `Int32Array`, `Uint32Array`, `BigInt64Array`, `BigUint64Array`, +`Float32Array`, and `Float64Array`. Multi-byte typed array elements are encoded in little-endian +byte order. `["blob", type, readableExpression]` diff --git a/src/core.ts b/src/core.ts index 2ea587e..d163a20 100644 --- a/src/core.ts +++ b/src/core.ts @@ -95,6 +95,18 @@ export function typeForRpc(value: unknown): TypeForRpc { case Uint8Array.prototype: case BUFFER_PROTOTYPE: + case ArrayBuffer.prototype: + case DataView.prototype: + case Int8Array.prototype: + case Uint8ClampedArray.prototype: + case Int16Array.prototype: + case Uint16Array.prototype: + case Int32Array.prototype: + case Uint32Array.prototype: + case BigInt64Array.prototype: + case BigUint64Array.prototype: + case Float32Array.prototype: + case Float64Array.prototype: return "bytes"; case WritableStream.prototype: diff --git a/src/serialize.ts b/src/serialize.ts index ad8ff76..9cd578c 100644 --- a/src/serialize.ts +++ b/src/serialize.ts @@ -73,6 +73,94 @@ export const DEFAULT_LIMITS: RpcLimits = { // ======================================================================================= +const NATIVE_LITTLE_ENDIAN = new Uint8Array(new Uint16Array([1]).buffer)[0] === 1; + +const BYTE_CONTAINER_TYPE_NAMES = [ + "ArrayBuffer", + "DataView", + "Int8Array", + "Uint8Array", + "Uint8ClampedArray", + "Int16Array", + "Uint16Array", + "Int32Array", + "Uint32Array", + "BigInt64Array", + "BigUint64Array", + "Float32Array", + "Float64Array", +] as const; + +type ByteContainerTypeName = typeof BYTE_CONTAINER_TYPE_NAMES[number]; +type MarkedByteContainerTypeName = Exclude; + +function isValidByteContainerName(value: string): value is ByteContainerTypeName { + return (BYTE_CONTAINER_TYPE_NAMES as readonly string[]).includes(value); +} + +// Listing every type, including those without multi-byte elements, makes adding +// a new ByteContainerTypeName a compile error until its element size is considered. +const TYPED_ARRAY_ELEMENT_SIZE: Record = { + ArrayBuffer: undefined, + DataView: undefined, + Int8Array: undefined, + Uint8Array: undefined, + Uint8ClampedArray: undefined, + Int16Array: 2, + Uint16Array: 2, + Int32Array: 4, + Uint32Array: 4, + BigInt64Array: 8, + BigUint64Array: 8, + Float32Array: 4, + Float64Array: 8, +}; + +// Uint8Array intentionally isn't included because it uses the markerless legacy form. +const BYTE_CONTAINER_PROTOTYPES: Record = { + ArrayBuffer: ArrayBuffer.prototype, + DataView: DataView.prototype, + Int8Array: Int8Array.prototype, + Uint8ClampedArray: Uint8ClampedArray.prototype, + Int16Array: Int16Array.prototype, + Uint16Array: Uint16Array.prototype, + Int32Array: Int32Array.prototype, + Uint32Array: Uint32Array.prototype, + BigInt64Array: BigInt64Array.prototype, + BigUint64Array: BigUint64Array.prototype, + Float32Array: Float32Array.prototype, + Float64Array: Float64Array.prototype, +}; + +const BYTE_CONTAINER_TYPE_BY_PROTOTYPE = new Map(); +for (let type of Object.keys(BYTE_CONTAINER_PROTOTYPES) as MarkedByteContainerTypeName[]) { + BYTE_CONTAINER_TYPE_BY_PROTOTYPE.set(BYTE_CONTAINER_PROTOTYPES[type], type); +} + +// Reverse each element's bytes in place. Production callers only need this on +// big-endian hosts; it is exported solely so the behavior can be tested on +// little-endian hosts. +export function swapByteOrder(bytes: Uint8Array, elementSize: number): void { + if (elementSize !== 2 && elementSize !== 4 && elementSize !== 8) { + throw new RangeError(`Unsupported element size: ${elementSize}`); + } + + let view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + for (let offset = 0; offset < bytes.byteLength; offset += elementSize) { + switch (elementSize) { + case 2: + view.setUint16(offset, view.getUint16(offset, false), true); + break; + case 4: + view.setUint32(offset, view.getUint32(offset, false), true); + break; + case 8: + view.setBigUint64(offset, view.getBigUint64(offset, false), true); + break; + } + } +} + export interface Exporter { exportStub(hook: StubHook): ExportId; exportPromise(hook: StubHook): ExportId; @@ -248,18 +336,33 @@ export class Devaluator { } case "bytes": { - let bytes = value as Uint8Array; - // At structuredClonable or jsonCompatibleWithBytes level, keep Uint8Array raw + let alternateTypeName = BYTE_CONTAINER_TYPE_BY_PROTOTYPE.get(Object.getPrototypeOf(value)); + let bytes: Uint8Array; + if (alternateTypeName === "ArrayBuffer") { + bytes = new Uint8Array(value as ArrayBuffer); + } else if (alternateTypeName === undefined) { + bytes = value as Uint8Array; + } else { + let view = value as ArrayBufferView; + bytes = new Uint8Array(view.buffer, view.byteOffset, view.byteLength); + let elementSize = TYPED_ARRAY_ELEMENT_SIZE[alternateTypeName]; + if (!NATIVE_LITTLE_ENDIAN && elementSize) { + bytes = bytes.slice(); + swapByteOrder(bytes, elementSize); + } + } + + // At structuredClonable or jsonCompatibleWithBytes level, keep the bytes raw. if (this.encodingLevel === "structuredClonable" || this.encodingLevel === "jsonCompatibleWithBytes") { - return ["bytes", bytes]; - } - // Otherwise encode as base64 - if (bytes.toBase64) { - return ["bytes", bytes.toBase64({omitPadding: true})]; + return alternateTypeName === undefined + ? ["bytes", bytes] : ["bytes", bytes, alternateTypeName]; } + let b64: string; - if (typeof Buffer !== "undefined") { + if (bytes.toBase64) { + b64 = bytes.toBase64({omitPadding: true}); + } else if (typeof Buffer !== "undefined") { let buf = bytes instanceof Buffer ? bytes : Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength); b64 = buf.toString("base64"); @@ -270,7 +373,8 @@ export class Devaluator { } b64 = btoa(binary); } - return ["bytes", b64.replace(/=+$/, "")]; + b64 = b64.replace(/=+$/, ""); + return alternateTypeName === undefined ? ["bytes", b64] : ["bytes", b64, alternateTypeName]; } case "headers": @@ -738,27 +842,68 @@ export class Evaluator { } break; case "bytes": { - // At jsonCompatibleWithBytes/structuredClonable level, bytes may already be a Uint8Array + let bytes: Uint8Array; + // At jsonCompatibleWithBytes/structuredClonable level, bytes may already be raw. if (value[1] instanceof Uint8Array) { - return value[1]; - } - // Otherwise decode from base64 - if (typeof value[1] == "string") { + bytes = value[1]; + } else if (typeof value[1] == "string") { if (typeof Buffer !== "undefined") { - return Buffer.from(value[1], "base64"); + bytes = Buffer.from(value[1], "base64"); } else if (Uint8Array.fromBase64) { - return Uint8Array.fromBase64(value[1]); + bytes = Uint8Array.fromBase64(value[1]); } else { let bs = atob(value[1]); let len = bs.length; - let bytes = new Uint8Array(len); + bytes = new Uint8Array(len); for (let i = 0; i < len; i++) { bytes[i] = bs.charCodeAt(i); } - return bytes; } + } else { + break; + } + + if (value.length === 2) { + return bytes; + } + if (typeof value[2] !== "string") { + throw new TypeError(`Unknown bytes type marker type: ${typeof value[2]}`); + } + + if (!isValidByteContainerName(value[2])) { + let marker = value[2].slice(0, 64); + throw new TypeError(`Unknown bytes type marker: ${marker}`); + } + + let marker = value[2]; + let elementSize = TYPED_ARRAY_ELEMENT_SIZE[marker]; + if (elementSize !== undefined && bytes.byteLength % elementSize !== 0) { + throw new TypeError( + `Invalid byte length ${bytes.byteLength} for ${marker}; ` + + `expected a multiple of ${elementSize}`); + } + + // Copy exactly the decoded range rather than exposing or aliasing a pooled Buffer. + let buffer = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength); + if (!NATIVE_LITTLE_ENDIAN && elementSize !== undefined) { + swapByteOrder(new Uint8Array(buffer), elementSize); + } + switch (marker) { + case "ArrayBuffer": return buffer; + case "DataView": return new DataView(buffer); + case "Int8Array": return new Int8Array(buffer); + case "Uint8Array": return new Uint8Array(buffer); + case "Uint8ClampedArray": return new Uint8ClampedArray(buffer); + case "Int16Array": return new Int16Array(buffer); + case "Uint16Array": return new Uint16Array(buffer); + case "Int32Array": return new Int32Array(buffer); + case "Uint32Array": return new Uint32Array(buffer); + case "BigInt64Array": return new BigInt64Array(buffer); + case "BigUint64Array": return new BigUint64Array(buffer); + case "Float32Array": return new Float32Array(buffer); + case "Float64Array": return new Float64Array(buffer); + default: marker satisfies never; } - break; } case "error": if (value.length >= 3 && typeof value[1] === "string" && typeof value[2] === "string") {