diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rs index eb7244e00b2b..fb8cba43e086 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit.rs @@ -550,6 +550,8 @@ pub(crate) fn emit( in_payload0, out_payload0, } => { + let stack_map = state.take_stack_map(); + // Note that we do not emit anything for preserving and restoring // ordinary registers here: That's taken care of by regalloc for us, // since we marked this instruction as clobbering all registers. @@ -643,6 +645,9 @@ pub(crate) fn emit( asm::inst::jmpq_m::new(tmp1.to_reg()).emit(sink, info, state); sink.bind_label(resume, state.ctrl_plane_mut()); + if let Some(s) = stack_map { + sink.push_user_stack_map(state, sink.cur_offset(), s); + } } Inst::JmpKnown { dst } => uncond_jmp(sink, *dst), diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs index 1ee94966b341..e758ee087cf0 100644 --- a/cranelift/codegen/src/isa/x64/inst/mod.rs +++ b/cranelift/codegen/src/isa/x64/inst/mod.rs @@ -1467,7 +1467,9 @@ impl MachInst for Inst { fn is_safepoint(&self) -> bool { match self { - Inst::CallKnown { .. } | Inst::CallUnknown { .. } => true, + Inst::CallKnown { .. } | Inst::CallUnknown { .. } | Inst::StackSwitchBasic { .. } => { + true + } _ => false, } } diff --git a/cranelift/filetests/filetests/isa/x64/stack_switch_stack_map.clif b/cranelift/filetests/filetests/isa/x64/stack_switch_stack_map.clif new file mode 100644 index 000000000000..dff66b81370e --- /dev/null +++ b/cranelift/filetests/filetests/isa/x64/stack_switch_stack_map.clif @@ -0,0 +1,14 @@ +test compile +set opt_level=speed +set stack_switch_model=basic +target x86_64 + +;; A stack switch is a GC safepoint. Its stack map is associated with the +;; resume PC saved in the outgoing control context. +function %switch_with_stack_map(i64, i64, i64) -> i64 { + ss0 = explicit_slot 4 + +block0(v0: i64, v1: i64, v2: i64): + v3 = stack_switch v0, v1, v2, stack_map=[i32 @ ss0+0] + return v3 +} diff --git a/crates/cranelift/src/alias_region.rs b/crates/cranelift/src/alias_region.rs index 3282f36ae27d..99a9af1a76aa 100644 --- a/crates/cranelift/src/alias_region.rs +++ b/crates/cranelift/src/alias_region.rs @@ -80,6 +80,9 @@ enum VmType { BuiltinFunctionsArray, ComponentBuiltinFunctionsArray, HostValRaw, + // Keep new variants at the end: `VmType`'s discriminant participates in + // stable alias-region IDs printed by disassembly tests. + VMPayloads, } /// A key that uniquely identifies an alias region across an entire compilation. diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs index a2a03e7df70f..86074d3a1098 100644 --- a/crates/cranelift/src/func_environ.rs +++ b/crates/cranelift/src/func_environ.rs @@ -46,6 +46,20 @@ use wasmtime_environ::{ }; use wasmtime_environ::{FUNCREF_INIT_BIT, FUNCREF_MASK}; +/// Function-local stack slots backing a continuation's +/// `VMPayloads::values`. +/// +/// The values and their GC-reference markers are logically one +/// payload descriptor but use distinct stack slots so that Cranelift +/// can assign them distinct alias regions. The marker slot is only +/// allocated when this function contains a stack switching site whose +/// payloads may contain GC references. +#[derive(Clone, Copy)] +pub(crate) struct VMPayloadStackSlots { + pub(crate) values: ir::StackSlot, + pub(crate) gc_ref_markers: Option, +} + #[derive(Copy, Clone, Debug)] pub(crate) enum Extension { Sign, @@ -216,10 +230,12 @@ pub struct FuncEnvironment<'module_environment> { /// current stack's `handler_list` field. stack_switching_handler_list_buffer: Option, - /// Used by the stack switching feature. If set, we have a allocated a - /// slot on this function's stack to be used for the - /// current continuation's `values` field. - stack_switching_values_buffer: Option, + /// Used by the stack switching feature. If set, these are the stack slots + /// backing the current continuation's `values` field. + stack_switching_values_storage: Option, + + /// Reusable storage for `get_interned_contref` builtin. + stack_switching_cont_ref_result_storage: Option, /// The stack-slot used for exposing Wasm state via debug /// instrumentation, if any, and the builder containing its metadata. @@ -300,7 +316,8 @@ impl<'module_environment> FuncEnvironment<'module_environment> { stack_limit_at_function_entry: None, stack_switching_handler_list_buffer: None, - stack_switching_values_buffer: None, + stack_switching_values_storage: None, + stack_switching_cont_ref_result_storage: None, state_slot: None, next_srcloc: ir::SourceLoc::default(), @@ -313,6 +330,18 @@ impl<'module_environment> FuncEnvironment<'module_environment> { } } + /// Returns the cached continuation-reference stack slot, creating it with + /// `data` if necessary. + pub(crate) fn get_or_create_contref_stack_slot( + &mut self, + builder: &mut FunctionBuilder<'_>, + data: ir::StackSlotData, + ) -> ir::StackSlot { + *self + .stack_switching_cont_ref_result_storage + .get_or_insert_with(|| builder.create_sized_stack_slot(data)) + } + /// Consume the branch hint for the instruction at module-relative `offset` /// (i.e. `builder.srcloc().bits()`), if any. The lazy decoder only moves /// forward, making this O(n) over a function body. @@ -5287,8 +5316,9 @@ impl FuncEnvironment<'_> { builder: &mut FunctionBuilder<'_>, contobj: ir::Value, args: &[ir::Value], + arg_types: &[WasmValType], ) -> ir::Value { - stack_switching::instructions::translate_cont_bind(self, builder, contobj, args) + stack_switching::instructions::translate_cont_bind(self, builder, contobj, args, arg_types) } pub fn translate_cont_new( @@ -5368,13 +5398,15 @@ impl FuncEnvironment<'_> { builder: &mut FunctionBuilder<'_>, tag_index: u32, suspend_args: &[ir::Value], - tag_return_types: &[ir::Type], + suspend_arg_types: &[WasmValType], + tag_return_types: &[WasmValType], ) -> WasmResult> { stack_switching::instructions::translate_suspend( self, builder, tag_index, suspend_args, + suspend_arg_types, tag_return_types, ) } @@ -5386,7 +5418,8 @@ impl FuncEnvironment<'_> { tag_index: u32, contobj: ir::Value, switch_args: &[ir::Value], - return_types: &[ir::Type], + switch_arg_types: &[WasmValType], + return_types: &[WasmValType], ) -> WasmResult> { stack_switching::instructions::translate_switch( self, @@ -5394,6 +5427,7 @@ impl FuncEnvironment<'_> { tag_index, contobj, switch_args, + switch_arg_types, return_types, ) } diff --git a/crates/cranelift/src/func_environ/gc.rs b/crates/cranelift/src/func_environ/gc.rs index 9177b11d590b..06e25f97dd9c 100644 --- a/crates/cranelift/src/func_environ/gc.rs +++ b/crates/cranelift/src/func_environ/gc.rs @@ -2,7 +2,7 @@ use crate::TRAP_ARRAY_OUT_OF_BOUNDS; use crate::bounds_checks::BoundsCheck; -use crate::func_environ::{CheckedEntity, Extension, FuncEnvironment}; +use crate::func_environ::{CheckedEntity, Extension, FuncEnvironment, stack_switching::fatpointer}; use crate::translate::{ Heap, HeapData, MemoryKind, StructFieldsVec, TargetEnvironment, VmctxLoadChain, }; @@ -377,10 +377,7 @@ pub fn read_field_at_addr( .call(get_interned_func_ref, &[vmctx, func_ref_id, expected_ty]); builder.func.dfg.first_result(call_inst) } - WasmHeapTopType::Cont => { - // TODO(#10248) GC integration for stack switching - return stack_switching_unsupported(); - } + WasmHeapTopType::Cont => read_cont_ref_at_addr(func_env, builder, addr, flags)?, }, }, }; @@ -431,6 +428,94 @@ pub fn intern_func_ref( Ok(builder.ins().ireduce(ir::types::I32, func_ref_id)) } +fn intern_cont_ref( + func_env: &mut FuncEnvironment<'_>, + builder: &mut FunctionBuilder<'_>, + ref_type: WasmRefType, + contobj: ir::Value, +) -> ir::Value { + assert_eq!(ref_type.heap_type.top(), WasmHeapTopType::Cont); + + if ref_type.heap_type == WasmHeapType::NoCont { + let zero = builder.ins().iconst(ir::types::I32, 0); + + if !ref_type.nullable { + builder.ins().trapz(zero, TRAP_INTERNAL_ASSERT); + } + + return zero; + } + + let (revision, contref) = fatpointer::deconstruct(func_env, &mut builder.cursor(), contobj); + let vmctx = func_env.vmctx_val(&mut builder.cursor()); + let intern = func_env + .builtin_functions + .intern_contref_for_gc_heap(builder.func); + let call = builder.ins().call(intern, &[vmctx, contref, revision]); + let id = builder.func.dfg.first_result(call); + builder.ins().ireduce(ir::types::I32, id) +} + +fn read_cont_ref_at_addr( + func_env: &mut FuncEnvironment<'_>, + builder: &mut FunctionBuilder<'_>, + addr: ir::Value, + flags: ir::MemFlagsData, +) -> WasmResult { + let id = builder.ins().load(ir::types::I32, flags, addr, 0); + + // We either create or fetch an existing a stack slot to hold the + // continuation values (16 bytes), and pass the address of this + // slot as the out parameter to the builtin. + let pointer_type = func_env.pointer_type(); + let pointer_bytes = pointer_type.bytes(); + let fatpointer_bytes = fatpointer::bytes(func_env); + + let slot = func_env.get_or_create_contref_stack_slot( + builder, + ir::StackSlotData::new( + ir::StackSlotKind::ExplicitSlot, + fatpointer_bytes, + u8::try_from(pointer_bytes.trailing_zeros()).unwrap(), + ), + ); + let out_result = builder.ins().stack_addr(pointer_type, slot, 0); + + let vmctx = func_env.vmctx_val(&mut builder.cursor()); + let get = func_env + .builtin_functions + .get_interned_contref(builder.func); + builder.ins().call(get, &[vmctx, id, out_result]); + + let region = func_env.alias_regions.stack_slot_region(builder.func, slot); + let flags = ir::MemFlagsData::trusted().with_alias_region(Some(region)); + let contref = builder.ins().load(pointer_type, flags, out_result, 0); + let revision = builder.ins().load( + pointer_type, + flags, + out_result, + i32::try_from(pointer_bytes).unwrap(), + ); + Ok(fatpointer::construct( + func_env, + &mut builder.cursor(), + revision, + contref, + )) +} + +fn write_cont_ref_at_addr( + func_env: &mut FuncEnvironment<'_>, + builder: &mut FunctionBuilder<'_>, + ref_type: WasmRefType, + flags: ir::MemFlagsData, + field_addr: ir::Value, + contobj: ir::Value, +) { + let id = intern_cont_ref(func_env, builder, ref_type, contobj); + builder.ins().store(flags, id, field_addr, 0); +} + fn write_func_ref_at_addr( func_env: &mut FuncEnvironment<'_>, builder: &mut FunctionBuilder<'_>, @@ -487,7 +572,9 @@ pub fn write_field_at_addr( func_env, builder, r, field_addr, new_val, flags, )?; } - WasmHeapTopType::Cont => return stack_switching_unsupported(), + WasmHeapTopType::Cont => { + write_cont_ref_at_addr(func_env, builder, r, flags, field_addr, new_val) + } }, WasmStorageType::Val(_) => { assert_eq!( @@ -527,6 +614,10 @@ fn default_value( } WasmValType::Ref(r) => { assert!(r.nullable); + if r.heap_type.top() == WasmHeapTopType::Cont { + let zero = cursor.ins().iconst(func_env.pointer_type(), 0); + return fatpointer::construct(func_env, cursor, zero, zero); + } let (ty, needs_stack_map) = func_env.reference_type(r.heap_type); // NB: The collector doesn't need to know about null references. diff --git a/crates/cranelift/src/func_environ/gc/copying.rs b/crates/cranelift/src/func_environ/gc/copying.rs index 7cf5628d0b7f..ec9b47a9e8ca 100644 --- a/crates/cranelift/src/func_environ/gc/copying.rs +++ b/crates/cranelift/src/func_environ/gc/copying.rs @@ -444,7 +444,9 @@ impl GcCompiler for CopyingCompiler { // store the reference directly. unbarriered_store_gc_ref(builder, r.heap_type, field_addr, val, flags)?; } - WasmHeapTopType::Cont => return super::stack_switching_unsupported(), + WasmHeapTopType::Cont => { + write_field_at_addr(func_env, builder, ty, field_addr, val)? + } }, WasmStorageType::I8 => { assert_eq!(builder.func.dfg.value_type(val), ir::types::I32); diff --git a/crates/cranelift/src/func_environ/stack_switching/fatpointer.rs b/crates/cranelift/src/func_environ/stack_switching/fatpointer.rs index 05b92e7779de..f8b6fb8379c9 100644 --- a/crates/cranelift/src/func_environ/stack_switching/fatpointer.rs +++ b/crates/cranelift/src/func_environ/stack_switching/fatpointer.rs @@ -1,6 +1,11 @@ use cranelift_codegen::ir; use cranelift_codegen::ir::InstBuilder; +/// Returns the platform size of a fat pointer. +pub fn bytes(env: &crate::func_environ::FuncEnvironment) -> u32 { + 2 * env.pointer_type().bytes() +} + /// Returns the Cranelift type used to represent all of the following: /// - wasm values of type `(ref null $ct)` and `(ref $ct)` /// - equivalently: runtime values of type `Option` and `VMContObj` @@ -34,7 +39,7 @@ pub(crate) fn deconstruct<'a>( /// Constructs a continuation object from a given contref and revision pointer. /// The contref_addr may be 0, to indicate that we want to build a wasm null reference. pub(crate) fn construct<'a>( - env: &mut crate::func_environ::FuncEnvironment<'a>, + env: &crate::func_environ::FuncEnvironment<'a>, pos: &mut cranelift_codegen::cursor::FuncCursor, revision_counter: ir::Value, contref_addr: ir::Value, diff --git a/crates/cranelift/src/func_environ/stack_switching/instructions.rs b/crates/cranelift/src/func_environ/stack_switching/instructions.rs index 0310f62bd1cc..5a6dac28c5de 100644 --- a/crates/cranelift/src/func_environ/stack_switching/instructions.rs +++ b/crates/cranelift/src/func_environ/stack_switching/instructions.rs @@ -8,7 +8,10 @@ use cranelift_codegen::ir::{self, MemFlagsData}; use cranelift_codegen::ir::{Block, BlockCall, InstBuilder, JumpTableData}; use cranelift_frontend::FunctionBuilder; use itertools::{Either, Itertools}; -use wasmtime_environ::{PtrSize, TagIndex, TypeIndex, WasmResult, WasmValType, wasm_unsupported}; +use wasmtime_environ::{ + PtrSize, TagIndex, TypeIndex, WasmHeapType, WasmRefType, WasmResult, WasmValType, + wasm_unsupported, +}; fn control_context_size(triple: &target_lexicon::Triple) -> WasmResult { match (triple.architecture, triple.operating_system) { @@ -31,7 +34,7 @@ pub(crate) mod stack_switching_helpers { use cranelift_codegen::ir::types::*; use cranelift_codegen::ir::{StackSlot, StackSlotKind::*}; use cranelift_frontend::FunctionBuilder; - use wasmtime_environ::PtrSize; + use wasmtime_environ::{PtrSize, WasmValType}; /// Provides information about the layout of a type when it is used as an /// element in a host array. This is used for `VMHostArrayRef`. @@ -69,7 +72,17 @@ pub(crate) mod stack_switching_helpers { phantom: PhantomData, } - pub type VMPayloads = VMHostArrayRef; + /// Compile-time reference to a runtime `VMPayloads` descriptor. + /// + /// This wrapper keeps the payload buffer and its GC-reference markers + /// paired. Its core invariant is that `gc_ref_data`, when present, points + /// to one marker byte for every slot in `buffer`; callers must update a + /// slot's value and marker together. + #[derive(Copy, Clone)] + pub struct VMPayloads { + address: ir::Value, + buffer: VMHostArrayRef, + } // Actually a vector of *mut VMTagDefinition pub type VMHandlerList = VMHostArrayRef<*mut u8>; @@ -105,7 +118,7 @@ pub(crate) mod stack_switching_helpers { ) -> VMPayloads { let offset: i64 = env.offsets.ptr.vm_cont_ref().args().into(); let address = builder.ins().iadd_imm_s(self.address, offset); - VMPayloads::new(address) + VMPayloads::new(env, address) } pub fn values<'a>( @@ -115,7 +128,7 @@ pub(crate) mod stack_switching_helpers { ) -> VMPayloads { let offset: i64 = env.offsets.ptr.vm_cont_ref().values().into(); let address = builder.ins().iadd_imm_s(self.address, offset); - VMPayloads::new(address) + VMPayloads::new(env, address) } pub fn common_stack_information<'a>( @@ -238,6 +251,147 @@ pub(crate) mod stack_switching_helpers { } } + impl VMPayloads { + pub fn new(env: &mut crate::func_environ::FuncEnvironment<'_>, address: ir::Value) -> Self { + debug_assert_eq!(env.offsets.ptr.vm_payloads().buffer(), 0); + Self { + address, + buffer: VMHostArrayRef::new(address), + } + } + + pub fn get_gc_ref_data( + &self, + env: &mut crate::func_environ::FuncEnvironment<'_>, + builder: &mut FunctionBuilder, + ) -> ir::Value { + env.alias_regions + .vm_payloads() + .gc_ref_data() + .load(&mut builder.cursor(), self.address) + } + + pub fn set_gc_ref_data( + &self, + env: &mut crate::func_environ::FuncEnvironment<'_>, + builder: &mut FunctionBuilder, + data: ir::Value, + ) { + env.alias_regions.vm_payloads().gc_ref_data().store( + &mut builder.cursor(), + self.address, + data, + ); + } + + /// Prepares stack storage for this payload buffer. + /// + /// Payload values and their GC-reference markers use distinct stack + /// slots so that they have distinct alias regions. The descriptor's + /// capacity and pointers always describe the storage selected at this + /// particular instruction site. Both slots retain their identities and + /// grow in place as later sites require more capacity. + pub fn prepare_stack_storage( + &self, + env: &mut crate::func_environ::FuncEnvironment<'_>, + builder: &mut FunctionBuilder, + capacity: u32, + initial_types: &[WasmValType], + needs_gc_ref_markers: bool, + existing_slots: Option, + ) -> crate::func_environ::VMPayloadStackSlots { + debug_assert!(capacity > 0); + debug_assert!(initial_types.len() <= usize::try_from(capacity).unwrap()); + + let (align, entry_size) = + ::vmhostarray_entry_layout(&env.offsets.ptr); + let values_size = capacity + .checked_mul(entry_size) + .expect("stack switching values buffer size should fit in u32"); + + let values_slot = match existing_slots.map(|slots| slots.values) { + Some(slot) => { + // Keep the slot identity stable: `stack_addr` instructions + // emitted at earlier sites continue to refer to this slot + // when a later site requires more storage. + let slot_data = &mut builder.func.sized_stack_slots[slot]; + debug_assert!(align <= slot_data.align_shift); + debug_assert_eq!(slot_data.kind, ExplicitSlot); + slot_data.size = slot_data.size.max(values_size); + slot + } + None => builder.create_sized_stack_slot(ir::StackSlotData::new( + ir::StackSlotKind::ExplicitSlot, + values_size, + align, + )), + }; + + let capacity_value = builder.ins().iconst(I32, i64::from(capacity)); + let values_data = builder.ins().stack_addr(env.pointer_type(), values_slot, 0); + self.set_capacity(env, builder, capacity_value); + self.set_data(env, builder, values_data); + + let mut gc_ref_markers = existing_slots.and_then(|slots| slots.gc_ref_markers); + if needs_gc_ref_markers { + let marker_slot = match gc_ref_markers { + Some(slot) => { + let slot_data = &mut builder.func.sized_stack_slots[slot]; + debug_assert_eq!(slot_data.kind, ExplicitSlot); + slot_data.size = slot_data.size.max(capacity); + slot + } + None => builder.create_sized_stack_slot(ir::StackSlotData::new( + ir::StackSlotKind::ExplicitSlot, + capacity, + 0, + )), + }; + gc_ref_markers = Some(marker_slot); + + let marker_data = builder.ins().stack_addr(env.pointer_type(), marker_slot, 0); + self.set_gc_ref_data(env, builder, marker_data); + + let region = env + .alias_regions + .stack_slot_region(builder.func, marker_slot); + let flags = ir::MemFlagsData::trusted().with_alias_region(Some(region)); + let zero = builder.ins().iconst(I8, 0); + for offset in 0..capacity { + builder + .ins() + .store(flags, zero, marker_data, i32::try_from(offset).unwrap()); + } + for (offset, ty) in initial_types.iter().enumerate() { + if ty.is_vmgcref_type_and_not_i31() { + let marker = builder + .ins() + .iconst(I8, i64::from(wasmtime_environ::CONTINUATION_PAYLOAD_GC_REF)); + builder.ins().store( + flags, + marker, + marker_data, + i32::try_from(offset).unwrap(), + ); + } + } + } + + crate::func_environ::VMPayloadStackSlots { + values: values_slot, + gc_ref_markers, + } + } + } + + impl core::ops::Deref for VMPayloads { + type Target = VMHostArrayRef; + + fn deref(&self) -> &Self::Target { + &self.buffer + } + } + impl VMHostArrayRef { pub(crate) fn new(address: ir::Value) -> Self { Self { @@ -315,7 +469,7 @@ pub(crate) mod stack_switching_helpers { env: &mut crate::func_environ::FuncEnvironment<'a>, builder: &mut FunctionBuilder, arg_count: i32, - ) -> ir::Value { + ) -> (ir::Value, ir::Value) { let data = self.get_data(env, builder); let original_length = self.get_length(env, builder); let new_length = builder @@ -328,7 +482,7 @@ pub(crate) mod stack_switching_helpers { let byte_offset = builder .ins() .imul_imm_s(original_length, i64::from(entry_size)); - builder.ins().iadd(data, byte_offset) + (builder.ins().iadd(data, byte_offset), original_length) } pub fn allocate_or_reuse_stack_slot<'a>( @@ -375,13 +529,13 @@ pub(crate) mod stack_switching_helpers { } /// Loads n entries from this Vector object, where n is the length of - /// `load_types`, which also gives the types of the values to load. + /// `load_types`, which also gives the Wasm types of the values to load. /// Loading starts at index 0 of the Vector object. pub fn load_data_entries<'a>( &self, env: &mut crate::func_environ::FuncEnvironment<'a>, builder: &mut FunctionBuilder, - load_types: &[ir::Type], + load_types: &[WasmValType], ) -> Vec { let region = env .alias_regions @@ -392,10 +546,14 @@ pub(crate) mod stack_switching_helpers { let mut values = vec![]; let mut offset = 0; let (_align, entry_size) = T::vmhostarray_entry_layout(&env.offsets.ptr); - for valtype in load_types { + for wasm_ty in load_types { + let valtype = crate::value_type(env.isa(), *wasm_ty); let val = builder .ins() - .load(*valtype, memflags, data_start_pointer, offset); + .load(valtype, memflags, data_start_pointer, offset); + if env.val_ty_needs_stack_map(*wasm_ty) { + builder.declare_value_needs_stack_map(val); + } values.push(val); offset += i32::try_from(entry_size).unwrap(); } @@ -915,6 +1073,10 @@ pub(crate) mod stack_switching_helpers { use helpers::VMStackChain; use stack_switching_helpers as helpers; +fn types_need_gc_ref_markers(types: &[WasmValType]) -> bool { + types.iter().any(|ty| ty.is_vmgcref_type_and_not_i31()) +} + /// Stores the given arguments in the appropriate `VMPayloads` object in the /// continuation. If the continuation was never invoked, use the `args` object. /// Otherwise, use the `values` object. @@ -922,8 +1084,11 @@ pub(crate) fn vmcontref_store_payloads<'a>( env: &mut crate::func_environ::FuncEnvironment<'a>, builder: &mut FunctionBuilder, values: &[ir::Value], + types: &[WasmValType], contref: ir::Value, ) { + debug_assert_eq!(values.len(), types.len()); + let needs_gc_ref_markers = types_need_gc_ref_markers(types); let count = i32::try_from(values.len()).expect("Number of stack switching payloads should fit in i32"); if values.len() > 0 { @@ -931,6 +1096,9 @@ pub(crate) fn vmcontref_store_payloads<'a>( let use_payloads_block = builder.create_block(); let store_data_block = builder.create_block(); builder.append_block_param(store_data_block, env.pointer_type()); + if needs_gc_ref_markers { + builder.append_block_param(store_data_block, env.pointer_type()); + } let co = helpers::VMContRef::new(contref); let csi = co.common_stack_information(env, builder); @@ -944,11 +1112,15 @@ pub(crate) fn vmcontref_store_payloads<'a>( builder.seal_block(use_args_block); let args = co.args(env, builder); - let ptr = args.occupy_next_slots(env, builder, count); + let (ptr, original_length) = args.occupy_next_slots(env, builder, count); + let mut block_args = vec![BlockArg::Value(ptr)]; + if needs_gc_ref_markers { + let gc_refs_ptr = args.get_gc_ref_data(env, builder); + let gc_refs_ptr = builder.ins().iadd(gc_refs_ptr, original_length); + block_args.push(BlockArg::Value(gc_refs_ptr)); + } - builder - .ins() - .jump(store_data_block, &[BlockArg::Value(ptr)]); + builder.ins().jump(store_data_block, &block_args); } { @@ -959,10 +1131,14 @@ pub(crate) fn vmcontref_store_payloads<'a>( // This also checks that the buffer is large enough to hold // `values.len()` more elements. - let ptr = payloads.occupy_next_slots(env, builder, count); - builder - .ins() - .jump(store_data_block, &[BlockArg::Value(ptr)]); + let (ptr, original_length) = payloads.occupy_next_slots(env, builder, count); + let mut block_args = vec![BlockArg::Value(ptr)]; + if needs_gc_ref_markers { + let gc_refs_ptr = payloads.get_gc_ref_data(env, builder); + let gc_refs_ptr = builder.ins().iadd(gc_refs_ptr, original_length); + block_args.push(BlockArg::Value(gc_refs_ptr)); + } + builder.ins().jump(store_data_block, &block_args); } { @@ -970,6 +1146,8 @@ pub(crate) fn vmcontref_store_payloads<'a>( builder.seal_block(store_data_block); let ptr = builder.block_params(store_data_block)[0]; + let gc_refs_ptr = + needs_gc_ref_markers.then(|| builder.block_params(store_data_block)[1]); // Store the values. let region = env @@ -977,8 +1155,22 @@ pub(crate) fn vmcontref_store_payloads<'a>( .continuation_stack_memory_region(builder.func); let memflags = ir::MemFlagsData::trusted().with_alias_region(Some(region)); let mut offset = 0; - for value in values { + for (index, (value, ty)) in values.iter().zip(types).enumerate() { builder.ins().store(memflags, *value, ptr, offset); + if let Some(gc_refs_ptr) = gc_refs_ptr { + let marker = if ty.is_vmgcref_type_and_not_i31() { + i64::from(wasmtime_environ::CONTINUATION_PAYLOAD_GC_REF) + } else { + 0 + }; + let marker = builder.ins().iconst(I8, marker); + builder.ins().store( + memflags, + marker, + gc_refs_ptr, + i32::try_from(index).unwrap(), + ); + } offset += i32::from(env.offsets.ptr.maximum_value_size()); } } @@ -1257,6 +1449,7 @@ pub(crate) fn translate_cont_bind<'a>( builder: &mut FunctionBuilder, contobj: ir::Value, args: &[ir::Value], + arg_types: &[WasmValType], ) -> ir::Value { let (witness, contref) = fatpointer::deconstruct(env, &mut builder.cursor(), contobj); @@ -1268,7 +1461,7 @@ pub(crate) fn translate_cont_bind<'a>( let evidence = builder.ins().icmp(IntCC::Equal, witness, revision); env.trapz(builder, evidence, crate::TRAP_CONTINUATION_ALREADY_CONSUMED); - vmcontref_store_payloads(env, builder, args, contref); + vmcontref_store_payloads(env, builder, args, arg_types, contref); let revision = vmcontref.incr_revision(env, builder, revision); let contobj = fatpointer::construct(env, &mut builder.cursor(), revision, contref); @@ -1291,12 +1484,16 @@ pub(crate) fn translate_cont_new<'a>( let nreturns = builder .ins() .iconst(I32, i64::try_from(return_types.len()).unwrap()); + let gc_refs = builder.ins().iconst( + I32, + i64::from(u8::from(types_need_gc_ref_markers(arg_types))), + ); let cont_new_func = env.builtin_functions.cont_new(&mut builder.func); let vmctx = env.vmctx_val(&mut builder.cursor()); let call_inst = builder .ins() - .call(cont_new_func, &[vmctx, func, nargs, nreturns]); + .call(cont_new_func, &[vmctx, func, nargs, nreturns, gc_refs]); let contref = *builder.func.dfg.inst_results(call_inst).first().unwrap(); let tag = helpers::VMContRef::new(contref).get_revision(env, builder); @@ -1380,6 +1577,10 @@ fn translate_resume_impl<'a>( resume_payload: ResumePayload<'_>, resumetable: &[(u32, Option)], ) -> WasmResult> { + let resume_arg_types = env + .continuation_arguments(TypeIndex::from_u32(type_index)) + .to_vec(); + // The resume instruction is the most involved instruction to // compile as it is responsible for both continuation application // and control tag dispatch. @@ -1461,10 +1662,17 @@ fn translate_resume_impl<'a>( match resume_payload { ResumePayload::Values(resume_args) => { + debug_assert_eq!(resume_args.len(), resume_arg_types.len()); let _next_revision = vmcontref.incr_revision(env, builder, revision); if resume_args.len() > 0 { // We store the arguments in the `VMContRef` to be resumed. - vmcontref_store_payloads(env, builder, resume_args, resume_contref); + vmcontref_store_payloads( + env, + builder, + resume_args, + &resume_arg_types, + resume_contref, + ); } } ResumePayload::Throw { .. } | ResumePayload::ThrowRef(_) => { @@ -1752,11 +1960,7 @@ fn translate_resume_impl<'a>( preamble_blocks.push(preamble_block); builder.switch_to_block(preamble_block); - let param_types = env.tag_params(TagIndex::from_u32(handle_tag)); - let param_types: Vec = param_types - .iter() - .map(|wty| crate::value_type(env.isa(), *wty)) - .collect(); + let param_types = env.tag_params(TagIndex::from_u32(handle_tag)).to_vec(); let values = suspended_contref.values(env, builder); let mut suspend_args: Vec = @@ -1821,11 +2025,9 @@ fn translate_resume_impl<'a>( returned_csi.set_state_returned(env, builder); // Load the values returned by the continuation. - let return_types: Vec<_> = env + let return_types = env .continuation_returns(TypeIndex::from_u32(type_index)) - .iter() - .map(|ty| crate::value_type(env.isa(), *ty)) - .collect(); + .to_vec(); let payloads = returned_contref.args(env, builder); let return_values = payloads.load_data_entries(env, builder, &return_types); payloads.clear(env, builder, true); @@ -1839,7 +2041,7 @@ fn load_resume_values_or_throw<'a>( builder: &mut FunctionBuilder, result: ControlEffect, continuation: helpers::VMContRef, - return_types: &[ir::Type], + return_types: &[WasmValType], ) -> WasmResult> { let throw_block = builder.create_block(); let values_block = builder.create_block(); @@ -1854,7 +2056,11 @@ fn load_resume_values_or_throw<'a>( let values = continuation.values(env, builder); // Exception references use Wasmtime's compressed, 32-bit GC-reference // representation. - let exnref = values.load_data_entries(env, builder, &[I32])[0]; + let exnref_ty = WasmValType::Ref(WasmRefType { + nullable: false, + heap_type: WasmHeapType::Exn, + }); + let exnref = values.load_data_entries(env, builder, &[exnref_ty])[0]; values.clear(env, builder, true); gc::translate_exn_throw_ref(env, builder, exnref)?; @@ -1871,8 +2077,10 @@ pub(crate) fn translate_suspend<'a>( builder: &mut FunctionBuilder, tag_index: u32, suspend_args: &[ir::Value], - tag_return_types: &[ir::Type], + suspend_arg_types: &[WasmValType], + tag_return_types: &[WasmValType], ) -> WasmResult> { + debug_assert_eq!(suspend_args.len(), suspend_arg_types.len()); let tag_addr = tag_address(env, builder, tag_index); let vmctx = env.vmctx_val(&mut builder.cursor()); @@ -1894,16 +2102,21 @@ pub(crate) fn translate_suspend<'a>( // return values including the possible exception reference. let values = active_contref.values(env, builder); let required_capacity = u32::try_from(std::cmp::max( - 1, // This account for the possible exception reference. + 1, // This accounts for the possible exception reference. std::cmp::max(suspend_args.len(), tag_return_types.len()), )) .expect("Number of stack switching payloads should fit in u32"); - env.stack_switching_values_buffer = Some(values.allocate_or_reuse_stack_slot( + let needs_gc_ref_markers = + types_need_gc_ref_markers(suspend_arg_types) || types_need_gc_ref_markers(tag_return_types); + let existing_storage = env.stack_switching_values_storage; + env.stack_switching_values_storage = Some(values.prepare_stack_storage( env, builder, required_capacity, - env.stack_switching_values_buffer, + suspend_arg_types, + needs_gc_ref_markers, + existing_storage, )); if suspend_args.len() > 0 { @@ -1933,13 +2146,20 @@ pub(crate) fn translate_suspend<'a>( // A normal resume supplies the tag's return values. `resume_throw_ref` // supplies an exception reference and throws it at this suspension point. - load_resume_values_or_throw( + let return_values = load_resume_values_or_throw( env, builder, ControlEffect::from_u64(result), active_contref, tag_return_types, - ) + ); + + if needs_gc_ref_markers { + let zero = builder.ins().iconst(env.pointer_type(), 0); + values.set_gc_ref_data(env, builder, zero); + } + + return_values } pub(crate) fn translate_switch<'a>( @@ -1948,7 +2168,8 @@ pub(crate) fn translate_switch<'a>( tag_index: u32, switchee_contobj: ir::Value, switch_args: &[ir::Value], - return_types: &[ir::Type], + switch_arg_types: &[WasmValType], + return_types: &[WasmValType], ) -> WasmResult> { let vmctx = env.vmctx_val(&mut builder.cursor()); @@ -1978,6 +2199,7 @@ pub(crate) fn translate_switch<'a>( // `switcher_contref`) to the immediate child (called // `switcher_contref_last_ancestor`) of the stack with the corresponding // handler (saved in `handler_stack_chain`). + let return_values_need_gc_ref_markers = types_need_gc_ref_markers(return_types); let ( switcher_contref, switcher_contobj, @@ -2004,11 +2226,17 @@ pub(crate) fn translate_switch<'a>( // reference. let values = switcher_contref.values(env, builder); let required_capacity = u32::try_from(std::cmp::max(1, return_types.len())).unwrap(); - env.stack_switching_values_buffer = Some(values.allocate_or_reuse_stack_slot( + let existing_storage = env.stack_switching_values_storage; + env.stack_switching_values_storage = Some(values.prepare_stack_storage( env, builder, required_capacity, - env.stack_switching_values_buffer, + // No GC ref marking is required at this point. Upon + // return to this site `vmcontref_store_payloads` will + // store the markers, if needed. + &[], + return_values_need_gc_ref_markers, + existing_storage, )); let switcher_contref_csi = switcher_contref.common_stack_information(env, builder); @@ -2054,7 +2282,13 @@ pub(crate) fn translate_switch<'a>( let (switchee_contref_csi, switchee_contref_last_ancestor) = { let mut combined_payloads = switch_args.to_vec(); combined_payloads.push(switcher_contobj); - vmcontref_store_payloads(env, builder, &combined_payloads, switchee_contref.address); + vmcontref_store_payloads( + env, + builder, + &combined_payloads, + switch_arg_types, + switchee_contref.address, + ); let switchee_contref_csi = switchee_contref.common_stack_information(env, builder); switchee_contref_csi.set_state_running(env, builder); @@ -2183,11 +2417,20 @@ pub(crate) fn translate_switch<'a>( // After switching back to the original stack, either load the values // supplied by an ordinary resume or throw the injected exception. - load_resume_values_or_throw( + let return_values = load_resume_values_or_throw( env, builder, ControlEffect::from_u64(result), switcher_contref, return_types, - ) + ); + + if return_values_need_gc_ref_markers { + let zero = builder.ins().iconst(env.pointer_type(), 0); + switcher_contref + .values(env, builder) + .set_gc_ref_data(env, builder, zero); + } + + return_values } diff --git a/crates/cranelift/src/translate/code_translator.rs b/crates/cranelift/src/translate/code_translator.rs index e86340ec2eab..0e3954297438 100644 --- a/crates/cranelift/src/translate/code_translator.rs +++ b/crates/cranelift/src/translate/code_translator.rs @@ -3158,25 +3158,14 @@ pub fn translate_operator( let arg_count = src_types.len() - dst_arity; let arg_types = &src_types[0..arg_count]; - for arg_type in arg_types { - // We can't bind GC objects using cont.bind at the moment: We - // don't have the necessary infrastructure to traverse the - // buffers used by cont.bind when looking for GC roots. Thus, - // this crude check ensures that these buffers can never contain - // GC roots to begin with. - if arg_type.is_vmgcref_type_and_not_i31() { - return Err(wasmtime_environ::WasmError::Unsupported( - "cont.bind does not support GC types at the moment".into(), - )); - } - } - let (original_contobj, args) = environ.stacks.peekn(arg_count + 1).split_last().unwrap(); let original_contobj = *original_contobj; let args = args.to_vec(); - let new_contobj = environ.translate_cont_bind(builder, original_contobj, &args); + let arg_types = arg_types.to_vec(); + let new_contobj = + environ.translate_cont_bind(builder, original_contobj, &args, &arg_types); environ.stacks.popn(arg_count + 1); environ.stacks.push1(new_contobj); @@ -3184,17 +3173,18 @@ pub fn translate_operator( Operator::Suspend { tag_index } => { let tag_index = TagIndex::from_u32(*tag_index); let param_types = environ.tag_params(tag_index).to_vec(); - let return_types: SmallVec<[_; 8]> = environ - .tag_returns(tag_index) - .iter() - .map(|ty| crate::value_type(environ.isa(), *ty)) - .collect(); + let return_types = environ.tag_returns(tag_index).to_vec(); let params = environ.stacks.peekn(param_types.len()).to_vec(); let param_count = params.len(); - let return_values = - environ.translate_suspend(builder, tag_index.as_u32(), ¶ms, &return_types)?; + let return_values = environ.translate_suspend( + builder, + tag_index.as_u32(), + ¶ms, + ¶m_types, + &return_types, + )?; environ.stacks.popn(param_count); environ.stacks.pushn(&return_values); @@ -3316,45 +3306,43 @@ pub fn translate_operator( tag_index, } => { // Arguments of the continuation we are going to switch to - let continuation_argument_types: SmallVec<[_; 8]> = environ + let switch_arg_types: SmallVec<[_; 8]> = environ .continuation_arguments(TypeIndex::from_u32(*cont_type_index)) .to_smallvec(); // Arity includes the continuation argument - let arity = continuation_argument_types.len(); + let arity = switch_arg_types.len(); let (contobj, switch_args) = environ.stacks.peekn(arity).split_last().unwrap(); let contobj = *contobj; let switch_args = switch_args.to_vec(); // Type of the continuation we are going to create by suspending the // currently running stack - let current_continuation_type = continuation_argument_types.last().unwrap(); + let current_continuation_type = switch_arg_types.last().unwrap(); let current_continuation_type = current_continuation_type.unwrap_ref_type(); // Argument types of current_continuation_type. These will in turn // be the types of the arguments we receive when someone switches // back to this switch instruction - let current_continuation_arg_types: SmallVec<[_; 8]> = - match current_continuation_type.heap_type { - WasmHeapType::ConcreteCont(index) => { - let mti = index - .as_module_type_index() - .expect("expected module-local type index"); - - environ - .continuation_arguments_from_interned(mti) - .iter() - .map(|ty| crate::value_type(environ.isa(), *ty)) - .collect() - } - _ => panic!("Invalid type on switch"), - }; + let return_types: SmallVec<[_; 8]> = match current_continuation_type.heap_type { + WasmHeapType::ConcreteCont(index) => { + let mti = index + .as_module_type_index() + .expect("expected module-local type index"); + + environ + .continuation_arguments_from_interned(mti) + .to_smallvec() + } + _ => panic!("Invalid type on switch"), + }; let switch_return_values = environ.translate_switch( builder, *tag_index, contobj, &switch_args, - ¤t_continuation_arg_types, + &switch_arg_types, + &return_types, )?; environ.stacks.popn(arity); diff --git a/crates/environ/src/builtin.rs b/crates/environ/src/builtin.rs index d59f3c16afee..fc390ef50965 100644 --- a/crates/environ/src/builtin.rs +++ b/crates/environ/src/builtin.rs @@ -154,7 +154,7 @@ macro_rules! foreach_builtin_function { // Creates a new continuation from a funcref. #[cfg(feature = "stack-switching")] - cont_new(vmctx: vmctx, r: pointer, param_count: u32, result_count: u32) -> pointer; + cont_new(vmctx: vmctx, r: pointer, param_count: u32, result_count: u32, gc_refs: u32) -> pointer; // Return the instance ID for a given vmctx. #[cfg(feature = "gc")] @@ -170,6 +170,23 @@ macro_rules! foreach_builtin_function { // Process a debug breakpoint. breakpoint(vmctx: vmctx) -> bool; + + // Intern a continuation reference into the GC heap's side table. + #[cfg(all(feature = "gc", feature = "stack-switching"))] + intern_contref_for_gc_heap( + vmctx: vmctx, + contref: pointer, + revision: pointer + ) -> u64; + + // Resolve an interned continuation-reference ID and write its + // pointer and revision witness to `result`. + #[cfg(all(feature = "gc", feature = "stack-switching"))] + get_interned_contref( + vmctx: vmctx, + contref_id: u32, + result: pointer + ) -> bool; } }; } @@ -387,6 +404,7 @@ impl BuiltinFunctionIndex { (@get ref_func pointer) => (return None); (@get table_get_lazy_init_func_ref pointer) => (return None); (@get intern_func_ref_for_gc_heap u64) => (return None); + (@get intern_contref_for_gc_heap u64) => (return None); (@get is_subtype u32) => (return None); (@get ceil_f32 f32) => (return None); (@get ceil_f64 f64) => (return None); diff --git a/crates/environ/src/stack_switching.rs b/crates/environ/src/stack_switching.rs index 8f43cbdd847c..a0d1c9d7def3 100644 --- a/crates/environ/src/stack_switching.rs +++ b/crates/environ/src/stack_switching.rs @@ -33,6 +33,10 @@ pub const STACK_STATE_TRAPPED_DISCRIMINANT: u32 = 5; /// Discriminant of variant `Return` in /// `runtime::vm::ControlEffect`. pub const CONTROL_EFFECT_RETURN_DISCRIMINANT: u32 = 0; + +/// Marker for a continuation payload slot that contains a GC-managed +/// reference and must be traced while the continuation is suspended. +pub const CONTINUATION_PAYLOAD_GC_REF: u8 = 1; /// Discriminant of variant `Resume` in /// `runtime::vm::ControlEffect`. pub const CONTROL_EFFECT_RESUME_DISCRIMINANT: u32 = 1; diff --git a/crates/environ/src/vmoffsets.rs b/crates/environ/src/vmoffsets.rs index 139309f46d00..741bf7e23d0c 100644 --- a/crates/environ/src/vmoffsets.rs +++ b/crates/environ/src/vmoffsets.rs @@ -71,6 +71,7 @@ macro_rules! define_vm_type_offsets { (@size ($p:expr) VMLazyThread) => { u32::from(($p).vm_lazy_thread().size()) }; (@size ($p:expr) VMStackLimits) => { u32::from(($p).vm_stack_limits().size()) }; (@size ($p:expr) VMHostArray) => { u32::from(($p).vm_host_array().size()) }; + (@size ($p:expr) VMPayloads) => { u32::from(($p).vm_payloads().size()) }; (@size ($p:expr) VMCommonStackInformation) => { u32::from(($p).vm_common_stack_information().size()) }; @@ -116,6 +117,7 @@ macro_rules! define_vm_type_offsets { (@align ($p:expr) VMLazyThread) => { u32::from(($p).vm_lazy_thread().align()) }; (@align ($p:expr) VMStackLimits) => { u32::from(($p).vm_stack_limits().align()) }; (@align ($p:expr) VMHostArray) => { u32::from(($p).vm_host_array().align()) }; + (@align ($p:expr) VMPayloads) => { u32::from(($p).vm_payloads().align()) }; (@align ($p:expr) VMCommonStackInformation) => { u32::from(($p).vm_common_stack_information().align()) }; diff --git a/crates/environ/src/vmtypes.rs b/crates/environ/src/vmtypes.rs index aca8cdcc3749..e1003de00378 100644 --- a/crates/environ/src/vmtypes.rs +++ b/crates/environ/src/vmtypes.rs @@ -651,12 +651,12 @@ macro_rules! for_each_vm_type { /// The arguments to, and return values of, the function passed /// to `cont.new`. #[aggregate] - pub args: VMHostArray, + pub args: VMPayloads, /// The payloads passed to and from this continuation once it /// has been suspended. #[aggregate] - pub values: VMHostArray, + pub values: VMPayloads, /// Tells the compiler that this structure has potential /// self-references, through `last_ancestor`. @@ -725,6 +725,21 @@ macro_rules! for_each_vm_type { /// The bump-allocation finger, an index into the GC heap. pub next: NonZeroU32, } + + /// Payload values exchanged with a continuation and the metadata + /// needed to trace GC references among them. + #[derive(Debug, Clone)] + #[repr(C)] + #[snake_name = vm_payloads] + pub struct VMPayloads { + /// The payload values themselves. + #[aggregate] + pub buffer: VMHostArray, + + /// One marker byte per buffer slot, indicating whether that + /// slot contains a GC reference, or `None` when no slots do. + pub gc_ref_data: Option>, + } } }; } diff --git a/crates/wasmtime/src/runtime/store/gc.rs b/crates/wasmtime/src/runtime/store/gc.rs index 364249346c2a..601c907a1b29 100644 --- a/crates/wasmtime/src/runtime/store/gc.rs +++ b/crates/wasmtime/src/runtime/store/gc.rs @@ -795,22 +795,38 @@ impl StoreOpaque { #[cfg(feature = "stack-switching")] fn trace_wasm_continuation_roots(&mut self, gc_roots_list: &mut GcRootsList) { - use crate::vm::VMStackState; + use crate::vm::{VMPayloads, VMStackState, ValRaw}; + + unsafe fn trace_payload_roots(gc_roots_list: &mut GcRootsList, payloads: &VMPayloads) { + let gc_ref_data = payloads.gc_ref_data; + let payloads = &payloads.buffer; + assert!(payloads.length <= payloads.capacity); + let Some(gc_ref_data) = gc_ref_data else { + return; + }; + let gc_ref_data = gc_ref_data.as_ptr(); + + for index in 0..usize::try_from(payloads.length).unwrap() { + let marker = unsafe { gc_ref_data.add(index).read() }; + if marker == wasmtime_environ::CONTINUATION_PAYLOAD_GC_REF { + let slot = unsafe { payloads.data.cast::().add(index).cast::() }; + unsafe { + StoreOpaque::trace_wasm_stack_slot(gc_roots_list, slot); + } + } + } + } log::trace!("Begin trace GC roots :: continuations"); for continuation in &self.continuations { let state = continuation.common_stack_information.state; - // FIXME(frank-emrich) In general, it is not enough to just trace - // through the stacks of continuations; we also need to look through - // their `cont.bind` arguments. However, we don't currently have - // enough RTTI information to check if any of the values in the - // buffers used by `cont.bind` are GC values. As a workaround, note - // that we currently disallow cont.bind-ing GC values altogether. - // This way, it is okay not to check them here. match state { VMStackState::Suspended => { + unsafe { + trace_payload_roots(gc_roots_list, &continuation.values); + } Backtrace::trace_suspended_continuation(self, continuation.deref(), |frame| { Self::trace_wasm_stack_frame(self.modules(), gc_roots_list, frame); core::ops::ControlFlow::Continue(()) @@ -824,8 +840,11 @@ impl StoreOpaque { // either case things should be handled correctly when traversing // further along in the chain, nothing required at this point. } - VMStackState::Fresh | VMStackState::Returned | VMStackState::Trapped => { - // Fresh/terminal continuations have no live GC values on their stack. + VMStackState::Fresh => unsafe { + trace_payload_roots(gc_roots_list, &continuation.args); + }, + VMStackState::Returned | VMStackState::Trapped => { + // Terminal continuations have no live GC values. } } } diff --git a/crates/wasmtime/src/runtime/vm.rs b/crates/wasmtime/src/runtime/vm.rs index e7f0c94d93e0..10526a14997b 100644 --- a/crates/wasmtime/src/runtime/vm.rs +++ b/crates/wasmtime/src/runtime/vm.rs @@ -137,7 +137,7 @@ pub use crate::runtime::vm::vmcontext::VMNullHeapData; pub use crate::runtime::vm::vmcontext::{ VMArrayCallHostFuncContext, VMCommonStackInformation, VMContRef, VMContext, VMFuncRef, VMFunctionImport, VMGlobalDefinition, VMGlobalImport, VMGlobalKind, VMHostArray, - VMMemoryDefinition, VMMemoryImport, VMOpaqueContext, VMStackLimits, VMStoreContext, + VMMemoryDefinition, VMMemoryImport, VMOpaqueContext, VMPayloads, VMStackLimits, VMStoreContext, VMTableImport, VMTagImport, VMWasmCallFunction, ValRaw, }; #[cfg(has_custom_sync)] diff --git a/crates/wasmtime/src/runtime/vm/gc.rs b/crates/wasmtime/src/runtime/vm/gc.rs index 5b0aae724863..28bf087928fa 100644 --- a/crates/wasmtime/src/runtime/vm/gc.rs +++ b/crates/wasmtime/src/runtime/vm/gc.rs @@ -8,6 +8,8 @@ mod disabled; #[cfg(not(feature = "gc"))] pub use disabled::*; +#[cfg(feature = "stack-switching")] +mod cont_ref; mod data; mod func_ref; mod gc_ref; @@ -15,6 +17,8 @@ mod gc_runtime; mod host_data; mod i31; +#[cfg(feature = "stack-switching")] +pub use cont_ref::*; pub use data::*; pub use func_ref::*; pub use gc_ref::*; @@ -54,6 +58,10 @@ pub struct GcStore { /// The function-references table for this GC heap. pub func_ref_table: FuncRefTable, + /// The continuation references table for this GC heap. + #[cfg(feature = "stack-switching")] + pub cont_ref_table: ContRefTable, + /// The total allocated bytes recorded after the last GC collection. /// `None` if no collection has been performed yet. Used by the /// grow-or-collect heuristic. @@ -96,6 +104,8 @@ impl GcStore { ) -> Self { let host_data_table = ExternRefHostDataTable::default(); let func_ref_table = FuncRefTable::default(); + #[cfg(feature = "stack-switching")] + let cont_ref_table = ContRefTable::default(); let _ = &gc_zeal_alloc_counter; @@ -104,6 +114,8 @@ impl GcStore { gc_heap, host_data_table, func_ref_table, + #[cfg(feature = "stack-switching")] + cont_ref_table, last_post_gc_allocated_bytes: None, #[cfg(gc_zeal)] gc_zeal_alloc_counter, diff --git a/crates/wasmtime/src/runtime/vm/gc/cont_ref.rs b/crates/wasmtime/src/runtime/vm/gc/cont_ref.rs new file mode 100644 index 000000000000..48c01b493526 --- /dev/null +++ b/crates/wasmtime/src/runtime/vm/gc/cont_ref.rs @@ -0,0 +1,56 @@ +//! Side table for continuation references stored in the GC heap. +//! +//! This follows the same idea as `VMFuncRef` and `externref`. We +//! cannot trust native addresses that come out of the GC heap. + +use crate::{Result, bail_bug, hash_map::HashMap, vm::VMContObj}; +use wasmtime_core::{ + alloc::PanicOnOom, + slab::{Id, Slab}, +}; + +/// Side table mapping the IDs stored in GC fields to continuation +/// objects. +/// +/// Raw ID zero is reserved for a null continuation reference. +/// Non-null IDs are one greater than their underlying slab IDs. +#[derive(Default)] +pub struct ContRefTable { + interned: HashMap, + slab: Slab, +} + +impl ContRefTable { + /// Intern a continuation object and return the ID to store in the + /// GC heap. Null continuation references use the reserved ID + /// zero. + /// + /// # Safety + /// + /// A non-null continuation's `VMContRef` pointer must remain valid for the + /// duration of this table's lifetime. + pub unsafe fn intern(&mut self, contobj: Option) -> u32 { + let Some(contobj) = contobj else { + return 0; + }; + + *self.interned.entry(contobj).or_insert_with(|| { + // TODO(dhil): Handle allocation failure here rather than panicking. + let id = self.slab.alloc(contobj).panic_on_oom().into_raw(); + id.checked_add(1).unwrap() + }) + } + + /// Resolve an ID loaded from the GC heap. + pub fn get(&self, raw: u32) -> Result> { + if raw == 0 { + return Ok(None); + } + + let id = Id::from_raw(raw - 1); + match self.slab.get(id).copied() { + Some(contobj) => Ok(Some(contobj)), + None => bail_bug!("bad continuation-reference table ID"), + } + } +} diff --git a/crates/wasmtime/src/runtime/vm/libcalls.rs b/crates/wasmtime/src/runtime/vm/libcalls.rs index 6964e405d62e..8d885265c5d7 100644 --- a/crates/wasmtime/src/runtime/vm/libcalls.rs +++ b/crates/wasmtime/src/runtime/vm/libcalls.rs @@ -604,6 +604,59 @@ fn get_interned_func_ref( Ok(func_ref.map_or(core::ptr::null_mut(), |f| f.as_ptr().cast())) } +// Intern a continuation reference into the GC heap's side table. +// +// This libcall may not GC. +#[cfg(all(feature = "gc", feature = "stack-switching"))] +unsafe fn intern_contref_for_gc_heap( + store: &mut dyn VMStore, + _instance: InstanceId, + contref: *mut u8, + revision: *mut u8, +) -> Result { + use crate::store::AutoAssertNoGc; + + let mut store = AutoAssertNoGc::new(store.store_opaque_mut()); + let contobj = unsafe { crate::vm::VMContObj::from_raw_parts(contref, revision.addr()) }; + let id = unsafe { store.require_gc_store_mut()?.cont_ref_table.intern(contobj) }; + Ok(id) +} + +// Resolve a continuation reference ID loaded from the GC heap and +// write its 16 bytes value into caller provided storage `out_result`. +// +// This libcall may not GC. +#[cfg(all(feature = "gc", feature = "stack-switching"))] +unsafe fn get_interned_contref( + store: &mut dyn VMStore, + _instance: InstanceId, + contref_id: u32, + out_result: *mut u8, +) -> Result<()> { + use crate::store::AutoAssertNoGc; + + #[repr(C)] + struct RawContObj { + contref: *mut u8, + revision: usize, + } + + let store = AutoAssertNoGc::new(store.store_opaque_mut()); + let contobj = store.unwrap_gc_store().cont_ref_table.get(contref_id)?; + let raw = match contobj { + Some(contobj) => RawContObj { + contref: contobj.contref.as_ptr().cast(), + revision: contobj.revision, + }, + None => RawContObj { + contref: core::ptr::null_mut(), + revision: 0, + }, + }; + unsafe { out_result.cast::().write(raw) }; + Ok(()) +} + #[cfg(feature = "gc")] fn is_subtype( store: &mut dyn VMStore, @@ -1119,9 +1172,16 @@ fn cont_new( func: *mut u8, param_count: u32, result_count: u32, + gc_refs: u32, ) -> Result> { - let ans = - crate::vm::stack_switching::cont_new(store, instance, func, param_count, result_count)?; + let ans = crate::vm::stack_switching::cont_new( + store, + instance, + func, + param_count, + result_count, + gc_refs != 0, + )?; Ok(Some(AllocationSize(ans.cast::() as usize))) } diff --git a/crates/wasmtime/src/runtime/vm/stack_switching.rs b/crates/wasmtime/src/runtime/vm/stack_switching.rs index 8a71d52e7cfb..06f61340d385 100644 --- a/crates/wasmtime/src/runtime/vm/stack_switching.rs +++ b/crates/wasmtime/src/runtime/vm/stack_switching.rs @@ -3,7 +3,7 @@ mod stack; -use crate::vm::{VMCommonStackInformation, VMContRef, VMHostArray, VMStackLimits}; +use crate::vm::{VMCommonStackInformation, VMContRef, VMHostArray, VMPayloads, VMStackLimits}; use core::{marker::PhantomPinned, ptr::NonNull}; pub use stack::*; @@ -51,7 +51,7 @@ pub const CONTROL_EFFECT_TRAP_ENCODING: u64 = /// (i.e., the one pointed to by the VMContObj) has a pointer to the /// other end of the chain (i.e., its last ancestor). #[repr(C)] -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub struct VMContObj { pub contref: NonNull, pub revision: usize, @@ -109,6 +109,27 @@ impl VMHostArray { data: core::ptr::null_mut(), } } + + /// Makes this array empty. + pub fn clear(&mut self) { + *self = Self::empty(); + } +} + +impl VMPayloads { + /// Creates an empty payload buffer with no GC metadata. + pub fn empty() -> Self { + Self { + buffer: VMHostArray::empty(), + gc_ref_data: None, + } + } + + /// Makes this payload buffer empty and invalidates its GC metadata. + pub fn clear(&mut self) { + self.buffer.clear(); + self.gc_ref_data = None; + } } impl VMContRef { @@ -135,8 +156,8 @@ impl VMContRef { let parent_chain = VMStackChain::Absent; let last_ancestor = core::ptr::null_mut(); let stack = VMContinuationStack::unallocated(); - let args = VMHostArray::empty(); - let values = VMHostArray::empty(); + let args = VMPayloads::empty(); + let values = VMPayloads::empty(); let revision = 0; let _marker = PhantomPinned; @@ -180,6 +201,7 @@ pub fn cont_new( func: *mut u8, param_count: u32, result_count: u32, + gc_refs: bool, ) -> crate::Result<*mut VMContRef> { let instance = store.instance_mut(instance); let caller_vmctx = instance.vmctx(); @@ -197,14 +219,14 @@ pub fn cont_new( // The initialization function will allocate the actual args/return value buffer and // update this object (if needed). - let contref_args_ptr = &mut contref.args as *mut VMHostArray; - + let contref_args_ptr = &mut contref.args as *mut VMPayloads; contref.stack.initialize( func.cast::(), caller_vmctx.as_ptr(), contref_args_ptr, param_count, result_count, + gc_refs, )?; // Now that the initial stack pointer was set by the initialization diff --git a/crates/wasmtime/src/runtime/vm/stack_switching/stack.rs b/crates/wasmtime/src/runtime/vm/stack_switching/stack.rs index fc24418c59e2..e8438a91d499 100644 --- a/crates/wasmtime/src/runtime/vm/stack_switching/stack.rs +++ b/crates/wasmtime/src/runtime/vm/stack_switching/stack.rs @@ -4,7 +4,7 @@ use crate::Result; use core::ops::Range; -use crate::runtime::vm::VMHostArray; +use crate::runtime::vm::VMPayloads; use crate::runtime::vm::{VMContext, VMFuncRef}; cfg_select! { @@ -95,7 +95,9 @@ impl VMContinuationStack { /// Initializes this stack, such that it will execute the function denoted /// by `func_ref`. `parameter_count` and `return_value_count` must be the /// corresponding number of parameters and return values of `func_ref`. - /// `args` must point to the `args` field of the `VMContRef` owning this pointer. + /// `args` must point to the `args` field of the `VMContRef` owning this + /// stack. When `gc_refs` is true, its `gc_ref_data` field receives the + /// corresponding root-marker buffer. /// /// It will be updated by this function to correctly describe /// the buffer used by this function for its arguments and return values. @@ -103,9 +105,10 @@ impl VMContinuationStack { &self, func_ref: *const VMFuncRef, caller_vmctx: *mut VMContext, - args: *mut VMHostArray, + args: *mut VMPayloads, parameter_count: u32, return_value_count: u32, + gc_refs: bool, ) -> Result<()> { self.0.initialize( func_ref, @@ -113,6 +116,7 @@ impl VMContinuationStack { args, parameter_count, return_value_count, + gc_refs, ) } } diff --git a/crates/wasmtime/src/runtime/vm/stack_switching/stack/dummy.rs b/crates/wasmtime/src/runtime/vm/stack_switching/stack/dummy.rs index 5c2cf049e490..1e5781f17bbb 100644 --- a/crates/wasmtime/src/runtime/vm/stack_switching/stack/dummy.rs +++ b/crates/wasmtime/src/runtime/vm/stack_switching/stack/dummy.rs @@ -1,7 +1,7 @@ use crate::Result; use core::ops::Range; -use crate::runtime::vm::VMHostArray; +use crate::runtime::vm::VMPayloads; use crate::runtime::vm::{VMContext, VMFuncRef}; /// Making sure that this has the same size as the non-dummy version, to @@ -59,9 +59,10 @@ impl VMContinuationStack { &self, _func_ref: *const VMFuncRef, _caller_vmctx: *mut VMContext, - _args: *mut VMHostArray, + _args: *mut VMPayloads, _parameter_count: u32, _return_value_count: u32, + _gc_refs: bool, ) -> Result<()> { Ok(()) } diff --git a/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix.rs b/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix.rs index 247495a5998f..580b104653c6 100644 --- a/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix.rs +++ b/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix.rs @@ -64,8 +64,7 @@ use std::ops::Range; use std::ptr; use crate::prelude::*; -use crate::runtime::vm::VMHostArray; -use crate::runtime::vm::{VMContext, VMFuncRef, ValRaw}; +use crate::runtime::vm::{VMContext, VMFuncRef, VMHostArray, VMPayloads, ValRaw, VmPtr}; #[derive(Debug, PartialEq, Eq)] pub enum Allocator { @@ -194,11 +193,14 @@ impl VMContinuationStack { /// calls `fiber_start` with the following arguments: /// TOS, func_ref, caller_vmctx, args_ptr, args_capacity /// - /// Note that at this point we also allocate the args buffer - /// (see picture at the top of this file). + /// Note that at this point we also allocate persistent launch storage for + /// the args buffer and, when `gc_refs` is true, its parallel + /// GC-reference-marker buffer (see picture at the top of this file). /// We define `args_capacity` as the max of parameter and return value count. - /// Then the size s of the actual buffer size is calculated as follows: - /// s = size_of(ValRaw) * `args_capacity`, + /// Their combined, 16-byte-aligned size `s` is calculated as follows when + /// `gc_refs` is true: + /// s = size_of(ValRaw) * `args_capacity` + /// + align_up(`args_capacity`, 16), /// /// Note that this value is used below, and we may have s = 0. /// @@ -222,15 +224,22 @@ impl VMContinuationStack { /// ---------------|------------------------------------------------------- /// -0x28 - s | func_ref /// -0x30 - s | caller_vmctx - /// -0x38 - s | args (of type *mut ArrayRef) + /// -0x38 - s | args (of type *mut VMHostArray) /// -0x40 - s | return_value_count + /// + /// The saved RSP points at `TOS - 0x40 - s`. On entry, + /// `wasmtime_continuation_start` pops the four launch-record words, leaving + /// RSP at `TOS - 0x20 - s`, immediately below the persistent buffers. + /// Subsequent stack frames grow towards lower addresses and may reuse the + /// consumed launch-record storage. pub fn initialize( &self, func_ref: *const VMFuncRef, caller_vmctx: *mut VMContext, - args: *mut VMHostArray, + args: *mut VMPayloads, parameter_count: u32, return_value_count: u32, + gc_refs: bool, ) -> Result<()> { let tos = self.top; @@ -240,22 +249,50 @@ impl VMContinuationStack { target.write(value) }; - let args_ref = &mut *args; + let payloads = &mut *args; + let args_ref = &mut payloads.buffer; let args_capacity = std::cmp::max(parameter_count, return_value_count); // The args object must currently be empty. debug_assert_eq!(args_ref.capacity, 0); debug_assert_eq!(args_ref.length, 0); - let total_control_size = usize::try_from(args_capacity)? + let args_data_size = usize::try_from(args_capacity)? .checked_mul(std::mem::size_of::()) - .and_then(|s| s.checked_add(0x40)) .ok_or_else(|| { format_err!( "continuation function type with {args_capacity} args \ overflows stack control data size calculation" ) })?; - let args_data_size = total_control_size - 0x40; + // Keep the fixed startup data 16-byte aligned. + let gc_refs_data_size = if cfg!(feature = "gc") && gc_refs { + usize::try_from(args_capacity)? + .checked_add(15) + .map(|s| s & !15) + .ok_or_else(|| { + format_err!( + "continuation function type with {args_capacity} args \ + overflows stack control data size calculation" + ) + })? + } else { + 0 + }; + let dynamic_data_size = + args_data_size + .checked_add(gc_refs_data_size) + .ok_or_else(|| { + format_err!( + "continuation function type with {args_capacity} args \ + overflows stack control data size calculation" + ) + })?; + let total_control_size = dynamic_data_size.checked_add(0x40).ok_or_else(|| { + format_err!( + "continuation function type with {args_capacity} args \ + overflows stack control data size calculation" + ) + })?; // Ensure the control data (fixed header + args buffer) fits // within the usable stack space. For Mmap allocations, @@ -282,6 +319,17 @@ impl VMContinuationStack { args_ref.capacity = args_capacity; args_ref.data = args_data_ptr; + if cfg!(feature = "gc") && gc_refs { + let data = if args_capacity == 0 { + ptr::null_mut() + } else { + tos.sub(0x20 + dynamic_data_size) + }; + if args_capacity > 0 { + data.write_bytes(0, usize::try_from(args_capacity)?); + } + payloads.gc_ref_data = NonNull::new(data).map(VmPtr::from); + } let to_store = [ // Data near top of stack: @@ -290,10 +338,16 @@ impl VMContinuationStack { (0x18, tos.sub(total_control_size).addr()), (0x20, usize::try_from(args_capacity)?), // Data after the args buffer: - (0x28 + args_data_size, func_ref.addr()), - (0x30 + args_data_size, caller_vmctx.addr()), - (0x38 + args_data_size, args.addr()), - (0x40 + args_data_size, usize::try_from(return_value_count)?), + (0x28 + dynamic_data_size, func_ref.addr()), + (0x30 + dynamic_data_size, caller_vmctx.addr()), + ( + 0x38 + dynamic_data_size, + (args_ref as *mut VMHostArray).addr(), + ), + ( + 0x40 + dynamic_data_size, + usize::try_from(return_value_count)?, + ), ]; for (offset, data) in to_store { diff --git a/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix/x86_64.rs b/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix/x86_64.rs index bc4f60a7d6a9..69337194d2c1 100644 --- a/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix/x86_64.rs +++ b/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix/x86_64.rs @@ -34,6 +34,7 @@ pub fn wasmtime_continuation_start_address() -> *const () { // values in various registers when execution of wasmtime_continuation_start begins: // // RSP: TOS - 0x40 - (16 * `args_capacity`) +// - align_up(`args_capacity`, 16) // RBP: TOS - 0x10 #[unsafe(naked)] diff --git a/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs b/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs index d2d530e2fb39..199efb34ba57 100644 --- a/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs +++ b/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs @@ -335,10 +335,15 @@ impl Backtrace { let mut continuations_iter = unsafe { chain.into_continuation_iter() }.peekable(); while let Some(continuation_ptr) = continuations_iter.next() { - let parent_limits_ptr = stack_limits_iter - .next() - .expect("expected one more VMStackLimits than continuations"); let continuation = unsafe { &*continuation_ptr }; + let Some(parent_limits_ptr) = stack_limits_iter.next() else { + // A detached suspended continuation chain ends in `Absent`, + // unlike a running chain which ends in an initial stack. The + // currently suspended stack was already traced above, and the + // last continuation has no parent stack left to trace. + debug_assert_eq!(continuation.parent_chain, VMStackChain::Absent); + break; + }; let parent_limits = unsafe { &*parent_limits_ptr }; // The parent of `continuation` if present and not the last in the chain. diff --git a/tests/all/stack_switching.rs b/tests/all/stack_switching.rs index dfc33714f8f1..3a075d471924 100644 --- a/tests/all/stack_switching.rs +++ b/tests/all/stack_switching.rs @@ -339,6 +339,71 @@ fn inter_instance_suspend() -> Result<()> { Ok(()) } +/// Regression test for https://github.com/bytecodealliance/wasmtime/issues/13750. +/// +/// A GC reference returned through a continuation payload must retain +/// its stack-map metadata while it is live on the operand stack +/// across subsequent GC safepoints. +#[cfg_attr(any(asan, miri), ignore)] +#[test] +fn continuation_result_gc_ref_on_operand_stack() -> Result<()> { + let mut config = Config::new(); + config + .wasm_stack_switching(true) + .wasm_exceptions(true) + .wasm_function_references(true) + .wasm_gc(true) + .collector(Collector::Copying); + + #[cfg(gc_zeal)] + { + let _ = config.gc_zeal_alloc_counter(std::num::NonZeroU32::new(1)); + } + + let engine = Engine::new(&config)?; + let module = Module::new( + &engine, + r#" + (module + (type $S (struct (field $x i32))) + (type $A (array i64)) + (type $ft (func (result (ref $S)))) + (type $ct (cont $ft)) + + (func $target (result (ref $S)) + (struct.new $S (i32.const 0x12345678))) + (elem declare func $target) + + (func (export "run") (result i32) + (local $k (ref null $ct)) + (local $i i32) + (local.set $k (cont.new $ct (ref.func $target))) + + ;; Keep the continuation's GC-reference result on the operand + ;; stack, rather than storing it in a GC-typed local. + (resume $ct (local.get $k)) + + ;; Force collections while that operand-stack value is the + ;; only live reference to the struct. + (local.set $i (i32.const 0)) + (loop $again + (drop (array.new_default $A (i32.const 256))) + (local.set $i (i32.add (local.get $i) (i32.const 1))) + (br_if $again + (i32.lt_u (local.get $i) (i32.const 2000)))) + + (struct.get $S $x)) + ) + "#, + )?; + + let mut store = Store::new(&engine, ()); + let instance = Instance::new(&mut store, &module, &[])?; + let run = instance.get_typed_func::<(), i32>(&mut store, "run")?; + assert_eq!(run.call(&mut store, ())?, 0x12345678); + Ok(()) +} + /// Tests interaction with host functions. Note that the interaction with host /// functions and traps is covered by the module `traps` further down. mod host { diff --git a/tests/disas/stack-switching/resume-suspend-data-passing.wat b/tests/disas/stack-switching/resume-suspend-data-passing.wat index 2606be8d129b..1dcad97b8de6 100644 --- a/tests/disas/stack-switching/resume-suspend-data-passing.wat +++ b/tests/disas/stack-switching/resume-suspend-data-passing.wat @@ -61,7 +61,7 @@ ;; block0(v0: i64, v1: i64): ;; @003c v3 = iconst.i32 10 ;; @0044 v7 = load.i64 notrap aligned readonly can_move region0 v0+8 -;; @0044 v31 = iconst.i64 136 +;; @0044 v31 = iconst.i64 144 ;; @0044 v34 = stack_addr.i64 ss0 ;; @0044 v37 = iconst.i64 0 ;; @0044 v47 = iconst.i64 96 @@ -118,8 +118,8 @@ ;; block8: ;; @0044 store.i64 notrap aligned region7 v11, v9+80 ;; v91 = iconst.i32 1 -;; v92 = iconst.i64 136 -;; v93 = iadd.i64 v9, v92 ; v92 = 136 +;; v92 = iconst.i64 144 +;; v93 = iadd.i64 v9, v92 ; v92 = 144 ;; @0044 store notrap aligned region8 v91, v93+4 ; v91 = 1 ;; @0044 store.i64 notrap aligned region4 v34, v93+8 ;; @0044 store.i32 notrap aligned region6 v4, v34 @@ -203,7 +203,7 @@ ;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 ;; gv2 = load.i64 notrap aligned region1 gv1+24 ;; sig0 = (i64 vmctx, i32) -> i64 tail -;; sig1 = (i64 vmctx, i64, i32, i32) -> i64 tail +;; sig1 = (i64 vmctx, i64, i32, i32, i32) -> i64 tail ;; sig2 = (i64 vmctx) tail ;; fn0 = colocated u805306368:6 sig0 ;; fn1 = colocated u805306368:42 sig1 @@ -214,195 +214,195 @@ ;; @0056 v2 = iconst.i32 0 ;; @0056 v3 = call fn0(v0, v2) ; v2 = 0 ;; @0058 trapz v3, user16 -;; @0058 v6 = call fn1(v0, v3, v2, v2) ; v2 = 0, v2 = 0 -;; @0058 v7 = load.i64 notrap aligned region2 v6+88 +;; @0058 v7 = call fn1(v0, v3, v2, v2, v2) ; v2 = 0, v2 = 0, v2 = 0 +;; @0058 v8 = load.i64 notrap aligned region2 v7+88 +;; @0058 v10 = uextend.i128 v8 +;; @0058 v11 = iconst.i64 64 +;; v139 = ishl v10, v11 ; v11 = 64 ;; @0058 v9 = uextend.i128 v7 -;; @0058 v10 = iconst.i64 64 -;; v138 = ishl v9, v10 ; v10 = 64 -;; @0058 v8 = uextend.i128 v6 -;; @0058 v13 = bor v138, v8 -;; @0062 v22 = iconst.i64 1 -;; @0062 v25 = load.i64 notrap aligned readonly can_move region0 v0+8 -;; @0062 v28 = iconst.i64 0 -;; @0062 v29 = iconst.i64 2 -;; @0062 v33 = iconst.i32 1 -;; @0062 v34 = iconst.i32 2 -;; @0062 v48 = iconst.i64 40 -;; @0062 v51 = stack_addr.i64 ss0 -;; @0062 v52 = iconst.i64 48 -;; @0062 v53 = iadd v0, v52 ; v52 = 48 -;; @0062 v60 = iconst.i64 96 -;; @0062 v63 = iconst.i64 -24 -;; v142 = iconst.i64 0x0001_0000_0000 -;; @005c jump block2(v13) +;; @0058 v14 = bor v139, v9 +;; @0062 v23 = iconst.i64 1 +;; @0062 v26 = load.i64 notrap aligned readonly can_move region0 v0+8 +;; @0062 v29 = iconst.i64 0 +;; @0062 v30 = iconst.i64 2 +;; @0062 v34 = iconst.i32 1 +;; @0062 v35 = iconst.i32 2 +;; @0062 v49 = iconst.i64 40 +;; @0062 v52 = stack_addr.i64 ss0 +;; @0062 v53 = iconst.i64 48 +;; @0062 v54 = iadd v0, v53 ; v53 = 48 +;; @0062 v61 = iconst.i64 96 +;; @0062 v64 = iconst.i64 -24 +;; v143 = iconst.i64 0x0001_0000_0000 +;; @005c jump block2(v14) ;; -;; block2(v14: i128): +;; block2(v15: i128): ;; @0062 jump block5 ;; ;; block5: -;; @0062 v15 = ireduce.i64 v14 -;; @0062 trapz v15, user16 -;; @0062 v20 = load.i64 notrap aligned region2 v15+88 -;; v145 = iconst.i64 64 -;; v146 = ushr.i128 v14, v145 ; v145 = 64 -;; @0062 v19 = ireduce.i64 v146 -;; @0062 v21 = icmp eq v20, v19 -;; @0062 trapz v21, user23 -;; v147 = iconst.i64 1 -;; v148 = iadd v20, v147 ; v147 = 1 -;; @0062 store notrap aligned region2 v148, v15+88 -;; @0062 v24 = load.i64 notrap aligned region3 v15+80 -;; @0062 v26 = load.i64 notrap aligned region4 v25+88 -;; @0062 v27 = load.i64 notrap aligned region4 v25+96 -;; @0062 store notrap aligned region5 v26, v24+64 -;; @0062 store notrap aligned region5 v27, v24+72 -;; v149 = iconst.i64 0 -;; @0062 store notrap aligned region3 v149, v15+80 ; v149 = 0 -;; v150 = iconst.i64 2 -;; @0062 store notrap aligned region4 v150, v25+88 ; v150 = 2 -;; @0062 store notrap aligned region4 v15, v25+96 -;; v151 = iconst.i32 1 -;; @0062 store notrap aligned region6 v151, v15+32 ; v151 = 1 -;; v152 = iconst.i32 2 -;; @0062 store notrap aligned region6 v152, v27+32 ; v152 = 2 -;; @0062 v38 = load.i64 notrap aligned region7 v25+72 -;; @0062 v39 = load.i64 notrap aligned region8 v25+64 -;; @0062 v40 = load.i64 notrap aligned region9 v25+80 -;; @0062 store notrap aligned region10 v38, v27+8 -;; @0062 store notrap aligned region11 v39, v27+16 -;; @0062 store notrap aligned region12 v40, v27+24 -;; @0062 v41 = load.i64 notrap aligned region1 v25+24 -;; @0062 store notrap aligned region13 v41, v27 -;; @0062 v44 = load.i64 notrap aligned region13 v15 -;; @0062 store notrap aligned region1 v44, v25+24 -;; @0062 v45 = load.i64 notrap aligned region10 v15+8 -;; @0062 store notrap aligned region7 v45, v25+72 -;; @0062 v46 = load.i64 notrap aligned region11 v15+16 -;; @0062 store notrap aligned region8 v46, v25+64 -;; @0062 v47 = load.i64 notrap aligned region12 v15+24 -;; @0062 store notrap aligned region9 v47, v25+80 -;; v153 = iconst.i64 40 -;; v154 = iadd v27, v153 ; v153 = 40 -;; @0062 store notrap aligned region14 v151, v154+4 ; v151 = 1 -;; @0062 store.i64 notrap aligned region15 v51, v154+8 -;; v155 = iadd.i64 v0, v52 ; v52 = 48 -;; @0062 store notrap aligned region16 v155, v51 -;; @0062 store notrap aligned region17 v151, v154 ; v151 = 1 -;; @0062 store notrap aligned region12 v151, v27+56 ; v151 = 1 -;; v156 = iconst.i64 96 -;; v157 = iadd v24, v156 ; v156 = 96 -;; @0062 v62 = load.i64 notrap aligned region18 v157 -;; v158 = iconst.i64 -24 -;; v159 = iadd v62, v158 ; v158 = -24 -;; v160 = iconst.i64 0x0001_0000_0000 -;; @0062 v65 = stack_switch v159, v159, v160 ; v160 = 0x0001_0000_0000 -;; @0062 v67 = load.i64 notrap aligned region4 v25+88 -;; @0062 v68 = load.i64 notrap aligned region4 v25+96 -;; @0062 store notrap aligned region4 v26, v25+88 -;; @0062 store notrap aligned region4 v27, v25+96 -;; @0062 store notrap aligned region6 v151, v27+32 ; v151 = 1 -;; v161 = iconst.i32 0 -;; @0062 store notrap aligned region17 v161, v154 ; v161 = 0 -;; @0062 store notrap aligned region14 v161, v154+4 ; v161 = 0 -;; @0062 store notrap aligned region15 v149, v154+8 ; v149 = 0 -;; @0062 store notrap aligned region12 v149, v27+56 ; v149 = 0 -;; @0062 brif v65, block9, block6 +;; @0062 v16 = ireduce.i64 v15 +;; @0062 trapz v16, user16 +;; @0062 v21 = load.i64 notrap aligned region2 v16+88 +;; v146 = iconst.i64 64 +;; v147 = ushr.i128 v15, v146 ; v146 = 64 +;; @0062 v20 = ireduce.i64 v147 +;; @0062 v22 = icmp eq v21, v20 +;; @0062 trapz v22, user23 +;; v148 = iconst.i64 1 +;; v149 = iadd v21, v148 ; v148 = 1 +;; @0062 store notrap aligned region2 v149, v16+88 +;; @0062 v25 = load.i64 notrap aligned region3 v16+80 +;; @0062 v27 = load.i64 notrap aligned region4 v26+88 +;; @0062 v28 = load.i64 notrap aligned region4 v26+96 +;; @0062 store notrap aligned region5 v27, v25+64 +;; @0062 store notrap aligned region5 v28, v25+72 +;; v150 = iconst.i64 0 +;; @0062 store notrap aligned region3 v150, v16+80 ; v150 = 0 +;; v151 = iconst.i64 2 +;; @0062 store notrap aligned region4 v151, v26+88 ; v151 = 2 +;; @0062 store notrap aligned region4 v16, v26+96 +;; v152 = iconst.i32 1 +;; @0062 store notrap aligned region6 v152, v16+32 ; v152 = 1 +;; v153 = iconst.i32 2 +;; @0062 store notrap aligned region6 v153, v28+32 ; v153 = 2 +;; @0062 v39 = load.i64 notrap aligned region7 v26+72 +;; @0062 v40 = load.i64 notrap aligned region8 v26+64 +;; @0062 v41 = load.i64 notrap aligned region9 v26+80 +;; @0062 store notrap aligned region10 v39, v28+8 +;; @0062 store notrap aligned region11 v40, v28+16 +;; @0062 store notrap aligned region12 v41, v28+24 +;; @0062 v42 = load.i64 notrap aligned region1 v26+24 +;; @0062 store notrap aligned region13 v42, v28 +;; @0062 v45 = load.i64 notrap aligned region13 v16 +;; @0062 store notrap aligned region1 v45, v26+24 +;; @0062 v46 = load.i64 notrap aligned region10 v16+8 +;; @0062 store notrap aligned region7 v46, v26+72 +;; @0062 v47 = load.i64 notrap aligned region11 v16+16 +;; @0062 store notrap aligned region8 v47, v26+64 +;; @0062 v48 = load.i64 notrap aligned region12 v16+24 +;; @0062 store notrap aligned region9 v48, v26+80 +;; v154 = iconst.i64 40 +;; v155 = iadd v28, v154 ; v154 = 40 +;; @0062 store notrap aligned region14 v152, v155+4 ; v152 = 1 +;; @0062 store.i64 notrap aligned region15 v52, v155+8 +;; v156 = iadd.i64 v0, v53 ; v53 = 48 +;; @0062 store notrap aligned region16 v156, v52 +;; @0062 store notrap aligned region17 v152, v155 ; v152 = 1 +;; @0062 store notrap aligned region12 v152, v28+56 ; v152 = 1 +;; v157 = iconst.i64 96 +;; v158 = iadd v25, v157 ; v157 = 96 +;; @0062 v63 = load.i64 notrap aligned region18 v158 +;; v159 = iconst.i64 -24 +;; v160 = iadd v63, v159 ; v159 = -24 +;; v161 = iconst.i64 0x0001_0000_0000 +;; @0062 v66 = stack_switch v160, v160, v161 ; v161 = 0x0001_0000_0000 +;; @0062 v68 = load.i64 notrap aligned region4 v26+88 +;; @0062 v69 = load.i64 notrap aligned region4 v26+96 +;; @0062 store notrap aligned region4 v27, v26+88 +;; @0062 store notrap aligned region4 v28, v26+96 +;; @0062 store notrap aligned region6 v152, v28+32 ; v152 = 1 +;; v162 = iconst.i32 0 +;; @0062 store notrap aligned region17 v162, v155 ; v162 = 0 +;; @0062 store notrap aligned region14 v162, v155+4 ; v162 = 0 +;; @0062 store notrap aligned region15 v150, v155+8 ; v150 = 0 +;; @0062 store notrap aligned region12 v150, v28+56 ; v150 = 0 +;; @0062 brif v66, block9, block6 ;; ;; block9: -;; @0062 v58 = iconst.i64 32 -;; @0062 v75 = ushr.i64 v65, v58 ; v58 = 32 -;; @0062 v76 = iconst.i64 4 -;; @0062 v77 = icmp eq v75, v76 ; v76 = 4 -;; @0062 brif v77, block8, block7 +;; @0062 v59 = iconst.i64 32 +;; @0062 v76 = ushr.i64 v66, v59 ; v59 = 32 +;; @0062 v77 = iconst.i64 4 +;; @0062 v78 = icmp eq v76, v77 ; v77 = 4 +;; @0062 brif v78, block8, block7 ;; ;; block8 cold: -;; @0062 v80 = iconst.i32 5 -;; @0062 store notrap aligned region6 v80, v68+32 ; v80 = 5 -;; @0062 v83 = load.i64 notrap aligned region13 v27 -;; @0062 store notrap aligned region1 v83, v25+24 -;; @0062 v84 = load.i64 notrap aligned region10 v27+8 -;; @0062 store notrap aligned region7 v84, v25+72 -;; @0062 v85 = load.i64 notrap aligned region11 v27+16 -;; @0062 store notrap aligned region8 v85, v25+64 -;; @0062 v86 = load.i64 notrap aligned region12 v27+24 -;; @0062 store notrap aligned region9 v86, v25+80 -;; v167 = iconst.i32 0 -;; v168 = iconst.i64 120 -;; v169 = iadd.i64 v68, v168 ; v168 = 120 -;; @0062 store notrap aligned region17 v167, v169 ; v167 = 0 -;; @0062 store notrap aligned region14 v167, v169+4 ; v167 = 0 -;; v170 = iconst.i64 0 -;; @0062 store notrap aligned region15 v170, v169+8 ; v170 = 0 -;; v171 = iconst.i64 136 -;; v172 = iadd.i64 v68, v171 ; v171 = 136 -;; @0062 store notrap aligned region17 v167, v172 ; v167 = 0 -;; @0062 store notrap aligned region14 v167, v172+4 ; v167 = 0 -;; @0062 store notrap aligned region15 v170, v172+8 ; v170 = 0 +;; @0062 v81 = iconst.i32 5 +;; @0062 store notrap aligned region6 v81, v69+32 ; v81 = 5 +;; @0062 v84 = load.i64 notrap aligned region13 v28 +;; @0062 store notrap aligned region1 v84, v26+24 +;; @0062 v85 = load.i64 notrap aligned region10 v28+8 +;; @0062 store notrap aligned region7 v85, v26+72 +;; @0062 v86 = load.i64 notrap aligned region11 v28+16 +;; @0062 store notrap aligned region8 v86, v26+64 +;; @0062 v87 = load.i64 notrap aligned region12 v28+24 +;; @0062 store notrap aligned region9 v87, v26+80 +;; v168 = iconst.i32 0 +;; v169 = iconst.i64 120 +;; v170 = iadd.i64 v69, v169 ; v169 = 120 +;; @0062 store notrap aligned region17 v168, v170 ; v168 = 0 +;; @0062 store notrap aligned region14 v168, v170+4 ; v168 = 0 +;; v171 = iconst.i64 0 +;; @0062 store notrap aligned region15 v171, v170+8 ; v171 = 0 +;; v172 = iconst.i64 144 +;; v173 = iadd.i64 v69, v172 ; v172 = 144 +;; @0062 store notrap aligned region17 v168, v173 ; v168 = 0 +;; @0062 store notrap aligned region14 v168, v173+4 ; v168 = 0 +;; @0062 store notrap aligned region15 v171, v173+8 ; v171 = 0 ;; @0062 try_call fn2(v0), sig2, block11, [ context v0 ] ;; ;; block11: ;; @0062 trap user12 ;; ;; block7: -;; @0062 v101 = load.i64 notrap aligned region7 v25+72 -;; @0062 v102 = load.i64 notrap aligned region8 v25+64 -;; @0062 v103 = load.i64 notrap aligned region9 v25+80 -;; @0062 store notrap aligned region10 v101, v68+8 -;; @0062 store notrap aligned region11 v102, v68+16 -;; @0062 store notrap aligned region12 v103, v68+24 -;; @0062 v106 = load.i64 notrap aligned region13 v27 -;; @0062 store notrap aligned region1 v106, v25+24 -;; @0062 v107 = load.i64 notrap aligned region10 v27+8 -;; @0062 store notrap aligned region7 v107, v25+72 -;; @0062 v108 = load.i64 notrap aligned region11 v27+16 -;; @0062 store notrap aligned region8 v108, v25+64 -;; @0062 v109 = load.i64 notrap aligned region12 v27+24 -;; @0062 store notrap aligned region9 v109, v25+80 -;; @0062 v111 = load.i64 notrap aligned region2 v68+88 +;; @0062 v102 = load.i64 notrap aligned region7 v26+72 +;; @0062 v103 = load.i64 notrap aligned region8 v26+64 +;; @0062 v104 = load.i64 notrap aligned region9 v26+80 +;; @0062 store notrap aligned region10 v102, v69+8 +;; @0062 store notrap aligned region11 v103, v69+16 +;; @0062 store notrap aligned region12 v104, v69+24 +;; @0062 v107 = load.i64 notrap aligned region13 v28 +;; @0062 store notrap aligned region1 v107, v26+24 +;; @0062 v108 = load.i64 notrap aligned region10 v28+8 +;; @0062 store notrap aligned region7 v108, v26+72 +;; @0062 v109 = load.i64 notrap aligned region11 v28+16 +;; @0062 store notrap aligned region8 v109, v26+64 +;; @0062 v110 = load.i64 notrap aligned region12 v28+24 +;; @0062 store notrap aligned region9 v110, v26+80 +;; @0062 v112 = load.i64 notrap aligned region2 v69+88 ;; @0062 jump block10 ;; ;; block12 cold: ;; @0062 trap user12 ;; ;; block13: -;; @0062 v118 = iconst.i64 136 -;; @0062 v119 = iadd.i64 v68, v118 ; v118 = 136 -;; @0062 v120 = load.i64 notrap aligned region15 v119+8 -;; @0062 v121 = load.i32 notrap aligned region16 v120 -;; v164 = iconst.i32 0 -;; @0062 store notrap aligned region17 v164, v119 ; v164 = 0 +;; @0062 v119 = iconst.i64 144 +;; @0062 v120 = iadd.i64 v69, v119 ; v119 = 144 +;; @0062 v121 = load.i64 notrap aligned region15 v120+8 +;; @0062 v122 = load.i32 notrap aligned region16 v121 +;; v165 = iconst.i32 0 +;; @0062 store notrap aligned region17 v165, v120 ; v165 = 0 ;; @0062 jump block4 ;; ;; block10: -;; @0062 v110 = ireduce.i32 v65 -;; @0062 br_table v110, block12, [block13] +;; @0062 v111 = ireduce.i32 v66 +;; @0062 br_table v111, block12, [block13] ;; ;; block6: -;; @0062 v125 = load.i64 notrap aligned region13 v27 -;; @0062 store notrap aligned region1 v125, v25+24 -;; @0062 v126 = load.i64 notrap aligned region10 v27+8 -;; @0062 store notrap aligned region7 v126, v25+72 -;; @0062 v127 = load.i64 notrap aligned region11 v27+16 -;; @0062 store notrap aligned region8 v127, v25+64 -;; @0062 v128 = load.i64 notrap aligned region12 v27+24 -;; @0062 store notrap aligned region9 v128, v25+80 -;; @0062 v131 = iconst.i32 4 -;; @0062 store notrap aligned region6 v131, v68+32 ; v131 = 4 -;; @0062 v132 = iconst.i64 120 -;; @0062 v133 = iadd.i64 v68, v132 ; v132 = 120 -;; @0062 v134 = load.i64 notrap aligned region15 v133+8 -;; v162 = iconst.i32 0 -;; @0062 store notrap aligned region17 v162, v133 ; v162 = 0 -;; @0062 store notrap aligned region14 v162, v133+4 ; v162 = 0 -;; v163 = iconst.i64 0 -;; @0062 store notrap aligned region15 v163, v133+8 ; v163 = 0 +;; @0062 v126 = load.i64 notrap aligned region13 v28 +;; @0062 store notrap aligned region1 v126, v26+24 +;; @0062 v127 = load.i64 notrap aligned region10 v28+8 +;; @0062 store notrap aligned region7 v127, v26+72 +;; @0062 v128 = load.i64 notrap aligned region11 v28+16 +;; @0062 store notrap aligned region8 v128, v26+64 +;; @0062 v129 = load.i64 notrap aligned region12 v28+24 +;; @0062 store notrap aligned region9 v129, v26+80 +;; @0062 v132 = iconst.i32 4 +;; @0062 store notrap aligned region6 v132, v69+32 ; v132 = 4 +;; @0062 v133 = iconst.i64 120 +;; @0062 v134 = iadd.i64 v69, v133 ; v133 = 120 +;; @0062 v135 = load.i64 notrap aligned region15 v134+8 +;; v163 = iconst.i32 0 +;; @0062 store notrap aligned region17 v163, v134 ; v163 = 0 +;; @0062 store notrap aligned region14 v163, v134+4 ; v163 = 0 +;; v164 = iconst.i64 0 +;; @0062 store notrap aligned region15 v164, v134+8 ; v164 = 0 ;; @0068 return ;; ;; block4: -;; @0062 v113 = uextend.i128 v111 -;; v165 = iconst.i64 64 -;; v166 = ishl v113, v165 ; v165 = 64 -;; @0062 v112 = uextend.i128 v68 -;; @0062 v117 = bor v166, v112 -;; @006d jump block2(v117) +;; @0062 v114 = uextend.i128 v112 +;; v166 = iconst.i64 64 +;; v167 = ishl v114, v166 ; v166 = 64 +;; @0062 v113 = uextend.i128 v69 +;; @0062 v118 = bor v167, v113 +;; @006d jump block2(v118) ;; } diff --git a/tests/disas/stack-switching/resume-suspend.wat b/tests/disas/stack-switching/resume-suspend.wat index bff2ab520ab3..ff5c9192fbe2 100644 --- a/tests/disas/stack-switching/resume-suspend.wat +++ b/tests/disas/stack-switching/resume-suspend.wat @@ -91,8 +91,8 @@ ;; block6: ;; @003b store.i64 notrap aligned region7 v8, v6+80 ;; v82 = iconst.i32 1 -;; @003b v28 = iconst.i64 136 -;; @003b v29 = iadd.i64 v6, v28 ; v28 = 136 +;; @003b v28 = iconst.i64 144 +;; @003b v29 = iadd.i64 v6, v28 ; v28 = 144 ;; @003b store notrap aligned region8 v82, v29+4 ; v82 = 1 ;; @003b v31 = stack_addr.i64 ss0 ;; @003b store notrap aligned region4 v31, v29+8 @@ -117,7 +117,7 @@ ;; @003b brif v51, block8, block9 ;; ;; block8 cold: -;; v87 = iadd.i64 v6, v28 ; v28 = 136 +;; v87 = iadd.i64 v6, v28 ; v28 = 144 ;; @003b v54 = load.i64 notrap aligned region4 v87+8 ;; @003b v55 = load.i32 notrap aligned region6 v54 ;; v88 = iconst.i32 0 @@ -131,7 +131,7 @@ ;; @003b trap user12 ;; ;; block9: -;; v84 = iadd.i64 v6, v28 ; v28 = 136 +;; v84 = iadd.i64 v6, v28 ; v28 = 144 ;; @003b v61 = load.i64 notrap aligned region4 v84+8 ;; v85 = iconst.i32 0 ;; @003b store notrap aligned region11 v85, v84 ; v85 = 0 @@ -169,7 +169,7 @@ ;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 ;; gv2 = load.i64 notrap aligned region1 gv1+24 ;; sig0 = (i64 vmctx, i32) -> i64 tail -;; sig1 = (i64 vmctx, i64, i32, i32) -> i64 tail +;; sig1 = (i64 vmctx, i64, i32, i32, i32) -> i64 tail ;; sig2 = (i64 vmctx) tail ;; fn0 = colocated u805306368:6 sig0 ;; fn1 = colocated u805306368:42 sig1 @@ -180,186 +180,186 @@ ;; @0043 v9 = iconst.i32 0 ;; @0043 v10 = call fn0(v0, v9) ; v9 = 0 ;; @0045 trapz v10, user16 -;; @0045 v13 = call fn1(v0, v10, v9, v9) ; v9 = 0, v9 = 0 -;; @0045 v14 = load.i64 notrap aligned region2 v13+88 +;; @0045 v14 = call fn1(v0, v10, v9, v9, v9) ; v9 = 0, v9 = 0, v9 = 0 +;; @0045 v15 = load.i64 notrap aligned region2 v14+88 ;; @004e jump block3 ;; ;; block3: -;; @0045 v16 = uextend.i128 v14 +;; @0045 v17 = uextend.i128 v15 ;; @0040 v5 = iconst.i64 64 -;; v153 = ishl v16, v5 ; v5 = 64 -;; v155 = ireduce.i64 v153 -;; v157 = bor v155, v13 -;; @004e trapz v157, user16 -;; @004e v26 = load.i64 notrap aligned region2 v157+88 -;; @0045 v15 = uextend.i128 v13 -;; @0045 v20 = bor v153, v15 -;; v159 = ushr v20, v5 ; v5 = 64 -;; @004e v25 = ireduce.i64 v159 -;; @004e v27 = icmp eq v26, v25 -;; @004e trapz v27, user23 -;; @004e v28 = iconst.i64 1 -;; @004e v29 = iadd v26, v28 ; v28 = 1 -;; @004e store notrap aligned region2 v29, v157+88 -;; @004e v30 = load.i64 notrap aligned region3 v157+80 -;; @004e v31 = load.i64 notrap aligned readonly can_move region0 v0+8 -;; @004e v32 = load.i64 notrap aligned region4 v31+88 -;; @004e v33 = load.i64 notrap aligned region4 v31+96 -;; @004e store notrap aligned region5 v32, v30+64 -;; @004e store notrap aligned region5 v33, v30+72 +;; v154 = ishl v17, v5 ; v5 = 64 +;; v156 = ireduce.i64 v154 +;; v158 = bor v156, v14 +;; @004e trapz v158, user16 +;; @004e v27 = load.i64 notrap aligned region2 v158+88 +;; @0045 v16 = uextend.i128 v14 +;; @0045 v21 = bor v154, v16 +;; v160 = ushr v21, v5 ; v5 = 64 +;; @004e v26 = ireduce.i64 v160 +;; @004e v28 = icmp eq v27, v26 +;; @004e trapz v28, user23 +;; @004e v29 = iconst.i64 1 +;; @004e v30 = iadd v27, v29 ; v29 = 1 +;; @004e store notrap aligned region2 v30, v158+88 +;; @004e v31 = load.i64 notrap aligned region3 v158+80 +;; @004e v32 = load.i64 notrap aligned readonly can_move region0 v0+8 +;; @004e v33 = load.i64 notrap aligned region4 v32+88 +;; @004e v34 = load.i64 notrap aligned region4 v32+96 +;; @004e store notrap aligned region5 v33, v31+64 +;; @004e store notrap aligned region5 v34, v31+72 ;; @0040 v2 = iconst.i64 0 -;; @004e store notrap aligned region3 v2, v157+80 ; v2 = 0 -;; @004e v35 = iconst.i64 2 -;; @004e store notrap aligned region4 v35, v31+88 ; v35 = 2 -;; @004e store notrap aligned region4 v157, v31+96 -;; @004e v39 = iconst.i32 1 -;; @004e store notrap aligned region6 v39, v157+32 ; v39 = 1 -;; @004e v40 = iconst.i32 2 -;; @004e store notrap aligned region6 v40, v33+32 ; v40 = 2 -;; @004e v44 = load.i64 notrap aligned region7 v31+72 -;; @004e v45 = load.i64 notrap aligned region8 v31+64 -;; @004e v46 = load.i64 notrap aligned region9 v31+80 -;; @004e store notrap aligned region10 v44, v33+8 -;; @004e store notrap aligned region11 v45, v33+16 -;; @004e store notrap aligned region12 v46, v33+24 -;; @004e v47 = load.i64 notrap aligned region1 v31+24 -;; @004e store notrap aligned region13 v47, v33 -;; @004e v50 = load.i64 notrap aligned region13 v157 -;; @004e store notrap aligned region1 v50, v31+24 -;; @004e v51 = load.i64 notrap aligned region10 v157+8 -;; @004e store notrap aligned region7 v51, v31+72 -;; @004e v52 = load.i64 notrap aligned region11 v157+16 -;; @004e store notrap aligned region8 v52, v31+64 -;; @004e v53 = load.i64 notrap aligned region12 v157+24 -;; @004e store notrap aligned region9 v53, v31+80 -;; @004e v54 = iconst.i64 40 -;; @004e v55 = iadd v33, v54 ; v54 = 40 -;; @004e store notrap aligned region14 v39, v55+4 ; v39 = 1 -;; @004e v57 = stack_addr.i64 ss0 -;; @004e store notrap aligned region15 v57, v55+8 -;; @004e v58 = iconst.i64 48 -;; @004e v59 = iadd.i64 v0, v58 ; v58 = 48 -;; @004e store notrap aligned region16 v59, v57 -;; @004e store notrap aligned region17 v39, v55 ; v39 = 1 -;; @004e store notrap aligned region12 v39, v33+56 ; v39 = 1 -;; @004e v66 = iconst.i64 96 -;; @004e v67 = iadd v30, v66 ; v66 = 96 -;; @004e v68 = load.i64 notrap aligned region18 v67 -;; @004e v69 = iconst.i64 -24 -;; @004e v70 = iadd v68, v69 ; v69 = -24 -;; v161 = iconst.i64 0x0001_0000_0000 -;; @004e v71 = stack_switch v70, v70, v161 ; v161 = 0x0001_0000_0000 -;; @004e v73 = load.i64 notrap aligned region4 v31+88 -;; @004e v74 = load.i64 notrap aligned region4 v31+96 -;; @004e store notrap aligned region4 v32, v31+88 -;; @004e store notrap aligned region4 v33, v31+96 -;; @004e store notrap aligned region6 v39, v33+32 ; v39 = 1 -;; v164 = iconst.i32 0 -;; @004e store notrap aligned region17 v164, v55 ; v164 = 0 -;; @004e store notrap aligned region14 v164, v55+4 ; v164 = 0 -;; @004e store notrap aligned region15 v2, v55+8 ; v2 = 0 -;; @004e store notrap aligned region12 v2, v33+56 ; v2 = 0 -;; @004e brif v71, block7, block4 +;; @004e store notrap aligned region3 v2, v158+80 ; v2 = 0 +;; @004e v36 = iconst.i64 2 +;; @004e store notrap aligned region4 v36, v32+88 ; v36 = 2 +;; @004e store notrap aligned region4 v158, v32+96 +;; @004e v40 = iconst.i32 1 +;; @004e store notrap aligned region6 v40, v158+32 ; v40 = 1 +;; @004e v41 = iconst.i32 2 +;; @004e store notrap aligned region6 v41, v34+32 ; v41 = 2 +;; @004e v45 = load.i64 notrap aligned region7 v32+72 +;; @004e v46 = load.i64 notrap aligned region8 v32+64 +;; @004e v47 = load.i64 notrap aligned region9 v32+80 +;; @004e store notrap aligned region10 v45, v34+8 +;; @004e store notrap aligned region11 v46, v34+16 +;; @004e store notrap aligned region12 v47, v34+24 +;; @004e v48 = load.i64 notrap aligned region1 v32+24 +;; @004e store notrap aligned region13 v48, v34 +;; @004e v51 = load.i64 notrap aligned region13 v158 +;; @004e store notrap aligned region1 v51, v32+24 +;; @004e v52 = load.i64 notrap aligned region10 v158+8 +;; @004e store notrap aligned region7 v52, v32+72 +;; @004e v53 = load.i64 notrap aligned region11 v158+16 +;; @004e store notrap aligned region8 v53, v32+64 +;; @004e v54 = load.i64 notrap aligned region12 v158+24 +;; @004e store notrap aligned region9 v54, v32+80 +;; @004e v55 = iconst.i64 40 +;; @004e v56 = iadd v34, v55 ; v55 = 40 +;; @004e store notrap aligned region14 v40, v56+4 ; v40 = 1 +;; @004e v58 = stack_addr.i64 ss0 +;; @004e store notrap aligned region15 v58, v56+8 +;; @004e v59 = iconst.i64 48 +;; @004e v60 = iadd.i64 v0, v59 ; v59 = 48 +;; @004e store notrap aligned region16 v60, v58 +;; @004e store notrap aligned region17 v40, v56 ; v40 = 1 +;; @004e store notrap aligned region12 v40, v34+56 ; v40 = 1 +;; @004e v67 = iconst.i64 96 +;; @004e v68 = iadd v31, v67 ; v67 = 96 +;; @004e v69 = load.i64 notrap aligned region18 v68 +;; @004e v70 = iconst.i64 -24 +;; @004e v71 = iadd v69, v70 ; v70 = -24 +;; v162 = iconst.i64 0x0001_0000_0000 +;; @004e v72 = stack_switch v71, v71, v162 ; v162 = 0x0001_0000_0000 +;; @004e v74 = load.i64 notrap aligned region4 v32+88 +;; @004e v75 = load.i64 notrap aligned region4 v32+96 +;; @004e store notrap aligned region4 v33, v32+88 +;; @004e store notrap aligned region4 v34, v32+96 +;; @004e store notrap aligned region6 v40, v34+32 ; v40 = 1 +;; v165 = iconst.i32 0 +;; @004e store notrap aligned region17 v165, v56 ; v165 = 0 +;; @004e store notrap aligned region14 v165, v56+4 ; v165 = 0 +;; @004e store notrap aligned region15 v2, v56+8 ; v2 = 0 +;; @004e store notrap aligned region12 v2, v34+56 ; v2 = 0 +;; @004e brif v72, block7, block4 ;; ;; block7: -;; @004e v64 = iconst.i64 32 -;; @004e v81 = ushr.i64 v71, v64 ; v64 = 32 -;; @004e v82 = iconst.i64 4 -;; @004e v83 = icmp eq v81, v82 ; v82 = 4 -;; @004e brif v83, block6, block5 +;; @004e v65 = iconst.i64 32 +;; @004e v82 = ushr.i64 v72, v65 ; v65 = 32 +;; @004e v83 = iconst.i64 4 +;; @004e v84 = icmp eq v82, v83 ; v83 = 4 +;; @004e brif v84, block6, block5 ;; ;; block6 cold: -;; @004e v86 = iconst.i32 5 -;; @004e store notrap aligned region6 v86, v74+32 ; v86 = 5 -;; @004e v89 = load.i64 notrap aligned region13 v33 -;; @004e store notrap aligned region1 v89, v31+24 -;; @004e v90 = load.i64 notrap aligned region10 v33+8 -;; @004e store notrap aligned region7 v90, v31+72 -;; @004e v91 = load.i64 notrap aligned region11 v33+16 -;; @004e store notrap aligned region8 v91, v31+64 -;; @004e v92 = load.i64 notrap aligned region12 v33+24 -;; @004e store notrap aligned region9 v92, v31+80 -;; v173 = iconst.i32 0 -;; v174 = iconst.i64 120 -;; v175 = iadd.i64 v74, v174 ; v174 = 120 -;; @004e store notrap aligned region17 v173, v175 ; v173 = 0 -;; @004e store notrap aligned region14 v173, v175+4 ; v173 = 0 -;; v176 = iconst.i64 0 -;; @004e store notrap aligned region15 v176, v175+8 ; v176 = 0 -;; v177 = iconst.i64 136 -;; v178 = iadd.i64 v74, v177 ; v177 = 136 -;; @004e store notrap aligned region17 v173, v178 ; v173 = 0 -;; @004e store notrap aligned region14 v173, v178+4 ; v173 = 0 -;; @004e store notrap aligned region15 v176, v178+8 ; v176 = 0 +;; @004e v87 = iconst.i32 5 +;; @004e store notrap aligned region6 v87, v75+32 ; v87 = 5 +;; @004e v90 = load.i64 notrap aligned region13 v34 +;; @004e store notrap aligned region1 v90, v32+24 +;; @004e v91 = load.i64 notrap aligned region10 v34+8 +;; @004e store notrap aligned region7 v91, v32+72 +;; @004e v92 = load.i64 notrap aligned region11 v34+16 +;; @004e store notrap aligned region8 v92, v32+64 +;; @004e v93 = load.i64 notrap aligned region12 v34+24 +;; @004e store notrap aligned region9 v93, v32+80 +;; v174 = iconst.i32 0 +;; v175 = iconst.i64 120 +;; v176 = iadd.i64 v75, v175 ; v175 = 120 +;; @004e store notrap aligned region17 v174, v176 ; v174 = 0 +;; @004e store notrap aligned region14 v174, v176+4 ; v174 = 0 +;; v177 = iconst.i64 0 +;; @004e store notrap aligned region15 v177, v176+8 ; v177 = 0 +;; v178 = iconst.i64 144 +;; v179 = iadd.i64 v75, v178 ; v178 = 144 +;; @004e store notrap aligned region17 v174, v179 ; v174 = 0 +;; @004e store notrap aligned region14 v174, v179+4 ; v174 = 0 +;; @004e store notrap aligned region15 v177, v179+8 ; v177 = 0 ;; @004e try_call fn2(v0), sig2, block9, [ context v0 ] ;; ;; block9: ;; @004e trap user12 ;; ;; block5: -;; @004e v107 = load.i64 notrap aligned region7 v31+72 -;; @004e v108 = load.i64 notrap aligned region8 v31+64 -;; @004e v109 = load.i64 notrap aligned region9 v31+80 -;; @004e store notrap aligned region10 v107, v74+8 -;; @004e store notrap aligned region11 v108, v74+16 -;; @004e store notrap aligned region12 v109, v74+24 -;; @004e v112 = load.i64 notrap aligned region13 v33 -;; @004e store notrap aligned region1 v112, v31+24 -;; @004e v113 = load.i64 notrap aligned region10 v33+8 -;; @004e store notrap aligned region7 v113, v31+72 -;; @004e v114 = load.i64 notrap aligned region11 v33+16 -;; @004e store notrap aligned region8 v114, v31+64 -;; @004e v115 = load.i64 notrap aligned region12 v33+24 -;; @004e store notrap aligned region9 v115, v31+80 -;; @004e v117 = load.i64 notrap aligned region2 v74+88 +;; @004e v108 = load.i64 notrap aligned region7 v32+72 +;; @004e v109 = load.i64 notrap aligned region8 v32+64 +;; @004e v110 = load.i64 notrap aligned region9 v32+80 +;; @004e store notrap aligned region10 v108, v75+8 +;; @004e store notrap aligned region11 v109, v75+16 +;; @004e store notrap aligned region12 v110, v75+24 +;; @004e v113 = load.i64 notrap aligned region13 v34 +;; @004e store notrap aligned region1 v113, v32+24 +;; @004e v114 = load.i64 notrap aligned region10 v34+8 +;; @004e store notrap aligned region7 v114, v32+72 +;; @004e v115 = load.i64 notrap aligned region11 v34+16 +;; @004e store notrap aligned region8 v115, v32+64 +;; @004e v116 = load.i64 notrap aligned region12 v34+24 +;; @004e store notrap aligned region9 v116, v32+80 +;; @004e v118 = load.i64 notrap aligned region2 v75+88 ;; @004e jump block8 ;; ;; block10 cold: ;; @004e trap user12 ;; ;; block11: -;; @004e v124 = iconst.i64 136 -;; @004e v125 = iadd.i64 v74, v124 ; v124 = 136 -;; @004e v126 = load.i64 notrap aligned region15 v125+8 -;; v170 = iconst.i32 0 -;; @004e store notrap aligned region17 v170, v125 ; v170 = 0 -;; @004e v119 = uextend.i128 v117 -;; v171 = iconst.i64 64 -;; v172 = ishl v119, v171 ; v171 = 64 -;; @004e v118 = uextend.i128 v74 -;; @004e v123 = bor v172, v118 -;; @004e jump block2(v123) +;; @004e v125 = iconst.i64 144 +;; @004e v126 = iadd.i64 v75, v125 ; v125 = 144 +;; @004e v127 = load.i64 notrap aligned region15 v126+8 +;; v171 = iconst.i32 0 +;; @004e store notrap aligned region17 v171, v126 ; v171 = 0 +;; @004e v120 = uextend.i128 v118 +;; v172 = iconst.i64 64 +;; v173 = ishl v120, v172 ; v172 = 64 +;; @004e v119 = uextend.i128 v75 +;; @004e v124 = bor v173, v119 +;; @004e jump block2(v124) ;; ;; block8: -;; @004e v116 = ireduce.i32 v71 -;; @004e br_table v116, block10, [block11] +;; @004e v117 = ireduce.i32 v72 +;; @004e br_table v117, block10, [block11] ;; ;; block4: -;; @004e v130 = load.i64 notrap aligned region13 v33 -;; @004e store notrap aligned region1 v130, v31+24 -;; @004e v131 = load.i64 notrap aligned region10 v33+8 -;; @004e store notrap aligned region7 v131, v31+72 -;; @004e v132 = load.i64 notrap aligned region11 v33+16 -;; @004e store notrap aligned region8 v132, v31+64 -;; @004e v133 = load.i64 notrap aligned region12 v33+24 -;; @004e store notrap aligned region9 v133, v31+80 -;; @004e v136 = iconst.i32 4 -;; @004e store notrap aligned region6 v136, v74+32 ; v136 = 4 -;; @004e v137 = iconst.i64 120 -;; @004e v138 = iadd.i64 v74, v137 ; v137 = 120 -;; @004e v139 = load.i64 notrap aligned region15 v138+8 -;; v165 = iconst.i32 0 -;; @004e store notrap aligned region17 v165, v138 ; v165 = 0 -;; @004e store notrap aligned region14 v165, v138+4 ; v165 = 0 -;; v166 = iconst.i64 0 -;; @004e store notrap aligned region15 v166, v138+8 ; v166 = 0 -;; v167 = uextend.i128 v166 ; v166 = 0 -;; v168 = iconst.i64 64 -;; v169 = ishl v167, v168 ; v168 = 64 -;; @0040 v8 = bor v169, v167 +;; @004e v131 = load.i64 notrap aligned region13 v34 +;; @004e store notrap aligned region1 v131, v32+24 +;; @004e v132 = load.i64 notrap aligned region10 v34+8 +;; @004e store notrap aligned region7 v132, v32+72 +;; @004e v133 = load.i64 notrap aligned region11 v34+16 +;; @004e store notrap aligned region8 v133, v32+64 +;; @004e v134 = load.i64 notrap aligned region12 v34+24 +;; @004e store notrap aligned region9 v134, v32+80 +;; @004e v137 = iconst.i32 4 +;; @004e store notrap aligned region6 v137, v75+32 ; v137 = 4 +;; @004e v138 = iconst.i64 120 +;; @004e v139 = iadd.i64 v75, v138 ; v138 = 120 +;; @004e v140 = load.i64 notrap aligned region15 v139+8 +;; v166 = iconst.i32 0 +;; @004e store notrap aligned region17 v166, v139 ; v166 = 0 +;; @004e store notrap aligned region14 v166, v139+4 ; v166 = 0 +;; v167 = iconst.i64 0 +;; @004e store notrap aligned region15 v167, v139+8 ; v167 = 0 +;; v168 = uextend.i128 v167 ; v167 = 0 +;; v169 = iconst.i64 64 +;; v170 = ishl v168, v169 ; v169 = 64 +;; @0040 v8 = bor v170, v168 ;; @0056 jump block2(v8) ;; -;; block2(v150: i128): +;; block2(v151: i128): ;; @0058 jump block1 ;; ;; block1: diff --git a/tests/disas/stack-switching/symmetric-switch.wat b/tests/disas/stack-switching/symmetric-switch.wat index 8cc5e60c0814..2318571bda87 100644 --- a/tests/disas/stack-switching/symmetric-switch.wat +++ b/tests/disas/stack-switching/symmetric-switch.wat @@ -52,7 +52,7 @@ ;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 ;; gv2 = load.i64 notrap aligned region1 gv1+24 ;; sig0 = (i64 vmctx, i32) -> i64 tail -;; sig1 = (i64 vmctx, i64, i32, i32) -> i64 tail +;; sig1 = (i64 vmctx, i64, i32, i32, i32) -> i64 tail ;; sig2 = (i64 vmctx, i32) -> i8 tail ;; fn0 = colocated u805306368:6 sig0 ;; fn1 = colocated u805306368:42 sig1 @@ -65,215 +65,216 @@ ;; @003c trapz v3, user16 ;; @003c v4 = iconst.i32 1 ;; @003c v5 = iconst.i32 0 -;; @003c v6 = call fn1(v0, v3, v4, v5) ; v4 = 1, v5 = 0 -;; @003c v7 = load.i64 notrap aligned region2 v6+88 -;; @003c v8 = uextend.i128 v6 +;; @003c v6 = iconst.i32 0 +;; @003c v7 = call fn1(v0, v3, v4, v5, v6) ; v4 = 1, v5 = 0, v6 = 0 +;; @003c v8 = load.i64 notrap aligned region2 v7+88 ;; @003c v9 = uextend.i128 v7 -;; @003c v10 = iconst.i64 64 -;; @003c v11 = uextend.i128 v10 ; v10 = 64 -;; @003c v12 = ishl v9, v11 -;; @003c v13 = bor v12, v8 -;; @003e v14 = ireduce.i64 v13 -;; @003e v15 = iconst.i64 64 -;; @003e v16 = uextend.i128 v15 ; v15 = 64 -;; @003e v17 = ushr v13, v16 -;; @003e v18 = ireduce.i64 v17 -;; @003e trapz v14, user16 -;; @003e v19 = load.i64 notrap aligned region2 v14+88 -;; @003e v20 = icmp eq v19, v18 -;; @003e trapz v20, user23 -;; @003e v21 = iconst.i64 1 -;; @003e v22 = iadd v19, v21 ; v21 = 1 -;; @003e store notrap aligned region2 v22, v14+88 -;; @003e v23 = iconst.i64 48 -;; @003e v24 = iadd v0, v23 ; v23 = 48 -;; @003e v25 = load.i64 notrap aligned readonly can_move region0 v0+8 -;; @003e v26 = load.i64 notrap aligned region3 v25+88 -;; @003e v27 = load.i64 notrap aligned region3 v25+96 -;; @003e jump block2(v26, v27) +;; @003c v10 = uextend.i128 v8 +;; @003c v11 = iconst.i64 64 +;; @003c v12 = uextend.i128 v11 ; v11 = 64 +;; @003c v13 = ishl v10, v12 +;; @003c v14 = bor v13, v9 +;; @003e v15 = ireduce.i64 v14 +;; @003e v16 = iconst.i64 64 +;; @003e v17 = uextend.i128 v16 ; v16 = 64 +;; @003e v18 = ushr v14, v17 +;; @003e v19 = ireduce.i64 v18 +;; @003e trapz v15, user16 +;; @003e v20 = load.i64 notrap aligned region2 v15+88 +;; @003e v21 = icmp eq v20, v19 +;; @003e trapz v21, user23 +;; @003e v22 = iconst.i64 1 +;; @003e v23 = iadd v20, v22 ; v22 = 1 +;; @003e store notrap aligned region2 v23, v15+88 +;; @003e v24 = iconst.i64 48 +;; @003e v25 = iadd v0, v24 ; v24 = 48 +;; @003e v26 = load.i64 notrap aligned readonly can_move region0 v0+8 +;; @003e v27 = load.i64 notrap aligned region3 v26+88 +;; @003e v28 = load.i64 notrap aligned region3 v26+96 +;; @003e jump block2(v27, v28) ;; -;; block2(v28: i64, v29: i64): -;; @003e v30 = iconst.i64 1 -;; @003e v31 = icmp eq v28, v30 ; v30 = 1 -;; @003e brif v31, block7, block3 +;; block2(v29: i64, v30: i64): +;; @003e v31 = iconst.i64 1 +;; @003e v32 = icmp eq v29, v31 ; v31 = 1 +;; @003e brif v32, block7, block3 ;; ;; block3: -;; @003e v32 = load.i64 notrap aligned region4 v29+64 -;; @003e v33 = load.i64 notrap aligned region4 v29+72 -;; @003e v34 = iconst.i64 40 -;; @003e v35 = iadd v33, v34 ; v34 = 40 -;; @003e v36 = load.i64 notrap aligned region5 v35+8 -;; @003e v37 = load.i32 notrap aligned region6 v33+56 -;; @003e v38 = load.i32 notrap aligned region7 v35 -;; @003e jump block4(v37) +;; @003e v33 = load.i64 notrap aligned region4 v30+64 +;; @003e v34 = load.i64 notrap aligned region4 v30+72 +;; @003e v35 = iconst.i64 40 +;; @003e v36 = iadd v34, v35 ; v35 = 40 +;; @003e v37 = load.i64 notrap aligned region5 v36+8 +;; @003e v38 = load.i32 notrap aligned region6 v34+56 +;; @003e v39 = load.i32 notrap aligned region7 v36 +;; @003e jump block4(v38) ;; -;; block4(v39: i32): -;; @003e v40 = icmp ult v39, v38 -;; @003e brif v40, block5, block2(v32, v33) +;; block4(v40: i32): +;; @003e v41 = icmp ult v40, v39 +;; @003e brif v41, block5, block2(v33, v34) ;; ;; block5: -;; @003e v41 = iconst.i32 8 -;; @003e v42 = imul.i32 v39, v41 ; v41 = 8 -;; @003e v43 = uextend.i64 v42 -;; @003e v44 = iadd.i64 v36, v43 -;; @003e v45 = load.i64 notrap aligned region8 v44 -;; @003e v46 = icmp eq v45, v24 -;; @003e v47 = iconst.i32 1 -;; @003e v48 = iadd.i32 v39, v47 ; v47 = 1 -;; @003e brif v46, block6, block4(v48) +;; @003e v42 = iconst.i32 8 +;; @003e v43 = imul.i32 v40, v42 ; v42 = 8 +;; @003e v44 = uextend.i64 v43 +;; @003e v45 = iadd.i64 v37, v44 +;; @003e v46 = load.i64 notrap aligned region8 v45 +;; @003e v47 = icmp eq v46, v25 +;; @003e v48 = iconst.i32 1 +;; @003e v49 = iadd.i32 v40, v48 ; v48 = 1 +;; @003e brif v47, block6, block4(v49) ;; ;; block7 cold: ;; @003e trap user22 ;; ;; block6: -;; @003e store.i64 notrap aligned region9 v29, v27+80 -;; @003e v49 = iconst.i64 136 -;; @003e v50 = iadd.i64 v27, v49 ; v49 = 136 -;; @003e v51 = iconst.i32 1 -;; @003e v52 = stack_addr.i64 ss0 -;; @003e store notrap aligned region10 v51, v50+4 ; v51 = 1 -;; @003e store notrap aligned region5 v52, v50+8 -;; @003e v53 = iconst.i64 0 -;; @003e v54 = iadd.i64 v27, v53 ; v53 = 0 -;; @003e v55 = iconst.i32 3 -;; @003e store notrap aligned region11 v55, v54+32 ; v55 = 3 -;; @003e v56 = iconst.i64 0 +;; @003e store.i64 notrap aligned region9 v30, v28+80 +;; @003e v50 = iconst.i64 144 +;; @003e v51 = iadd.i64 v28, v50 ; v50 = 144 +;; @003e v52 = iconst.i32 1 +;; @003e v53 = stack_addr.i64 ss0 +;; @003e store notrap aligned region10 v52, v51+4 ; v52 = 1 +;; @003e store notrap aligned region5 v53, v51+8 +;; @003e v54 = iconst.i64 0 +;; @003e v55 = iadd.i64 v28, v54 ; v54 = 0 +;; @003e v56 = iconst.i32 3 +;; @003e store notrap aligned region11 v56, v55+32 ; v56 = 3 ;; @003e v57 = iconst.i64 0 -;; @003e store notrap aligned region4 v56, v29+64 ; v56 = 0 -;; @003e store notrap aligned region4 v57, v29+72 ; v57 = 0 -;; @003e v58 = load.i64 notrap aligned readonly can_move region0 v0+8 -;; @003e v59 = iconst.i64 0 -;; @003e v60 = iadd v54, v59 ; v59 = 0 -;; @003e v61 = load.i64 notrap aligned region12 v58+72 -;; @003e v62 = load.i64 notrap aligned region13 v58+64 -;; @003e v63 = load.i64 notrap aligned region14 v58+80 -;; @003e store notrap aligned region15 v61, v60+8 -;; @003e store notrap aligned region16 v62, v60+16 -;; @003e store notrap aligned region6 v63, v60+24 -;; @003e v64 = load.i64 notrap aligned region2 v27+88 -;; @003e v65 = uextend.i128 v27 -;; @003e v66 = uextend.i128 v64 -;; @003e v67 = iconst.i64 64 -;; @003e v68 = uextend.i128 v67 ; v67 = 64 -;; @003e v69 = ishl v66, v68 -;; @003e v70 = bor v69, v65 -;; @003e v72 = iconst.i64 0 -;; @003e v73 = iadd.i64 v14, v72 ; v72 = 0 -;; @003e v74 = load.i32 notrap aligned region11 v73+32 -;; @003e v75 = iconst.i32 0 -;; @003e v76 = icmp ne v74, v75 ; v75 = 0 -;; @003e brif v76, block9, block8 +;; @003e v58 = iconst.i64 0 +;; @003e store notrap aligned region4 v57, v30+64 ; v57 = 0 +;; @003e store notrap aligned region4 v58, v30+72 ; v58 = 0 +;; @003e v59 = load.i64 notrap aligned readonly can_move region0 v0+8 +;; @003e v60 = iconst.i64 0 +;; @003e v61 = iadd v55, v60 ; v60 = 0 +;; @003e v62 = load.i64 notrap aligned region12 v59+72 +;; @003e v63 = load.i64 notrap aligned region13 v59+64 +;; @003e v64 = load.i64 notrap aligned region14 v59+80 +;; @003e store notrap aligned region15 v62, v61+8 +;; @003e store notrap aligned region16 v63, v61+16 +;; @003e store notrap aligned region6 v64, v61+24 +;; @003e v65 = load.i64 notrap aligned region2 v28+88 +;; @003e v66 = uextend.i128 v28 +;; @003e v67 = uextend.i128 v65 +;; @003e v68 = iconst.i64 64 +;; @003e v69 = uextend.i128 v68 ; v68 = 64 +;; @003e v70 = ishl v67, v69 +;; @003e v71 = bor v70, v66 +;; @003e v73 = iconst.i64 0 +;; @003e v74 = iadd.i64 v15, v73 ; v73 = 0 +;; @003e v75 = load.i32 notrap aligned region11 v74+32 +;; @003e v76 = iconst.i32 0 +;; @003e v77 = icmp ne v75, v76 ; v76 = 0 +;; @003e brif v77, block9, block8 ;; ;; block8: -;; @003e v77 = iconst.i64 120 -;; @003e v78 = iadd.i64 v14, v77 ; v77 = 120 -;; @003e v79 = load.i64 notrap aligned region5 v78+8 -;; @003e v80 = load.i32 notrap aligned region7 v78 -;; @003e v81 = iconst.i32 1 -;; @003e v82 = iadd v80, v81 ; v81 = 1 -;; @003e store notrap aligned region7 v82, v78 -;; @003e v83 = uextend.i64 v80 -;; @003e v84 = iconst.i64 16 -;; @003e v85 = imul v83, v84 ; v84 = 16 -;; @003e v86 = iadd v79, v85 -;; @003e jump block10(v86) +;; @003e v78 = iconst.i64 120 +;; @003e v79 = iadd.i64 v15, v78 ; v78 = 120 +;; @003e v80 = load.i64 notrap aligned region5 v79+8 +;; @003e v81 = load.i32 notrap aligned region7 v79 +;; @003e v82 = iconst.i32 1 +;; @003e v83 = iadd v81, v82 ; v82 = 1 +;; @003e store notrap aligned region7 v83, v79 +;; @003e v84 = uextend.i64 v81 +;; @003e v85 = iconst.i64 16 +;; @003e v86 = imul v84, v85 ; v85 = 16 +;; @003e v87 = iadd v80, v86 +;; @003e jump block10(v87) ;; ;; block9: -;; @003e v87 = iconst.i64 136 -;; @003e v88 = iadd.i64 v14, v87 ; v87 = 136 -;; @003e v89 = load.i64 notrap aligned region5 v88+8 -;; @003e v90 = load.i32 notrap aligned region7 v88 -;; @003e v91 = iconst.i32 1 -;; @003e v92 = iadd v90, v91 ; v91 = 1 -;; @003e store notrap aligned region7 v92, v88 -;; @003e v93 = uextend.i64 v90 -;; @003e v94 = iconst.i64 16 -;; @003e v95 = imul v93, v94 ; v94 = 16 -;; @003e v96 = iadd v89, v95 -;; @003e jump block10(v96) +;; @003e v88 = iconst.i64 144 +;; @003e v89 = iadd.i64 v15, v88 ; v88 = 144 +;; @003e v90 = load.i64 notrap aligned region5 v89+8 +;; @003e v91 = load.i32 notrap aligned region7 v89 +;; @003e v92 = iconst.i32 1 +;; @003e v93 = iadd v91, v92 ; v92 = 1 +;; @003e store notrap aligned region7 v93, v89 +;; @003e v94 = uextend.i64 v91 +;; @003e v95 = iconst.i64 16 +;; @003e v96 = imul v94, v95 ; v95 = 16 +;; @003e v97 = iadd v90, v96 +;; @003e jump block10(v97) ;; -;; block10(v71: i64): -;; @003e store.i128 notrap aligned region8 v70, v71 -;; @003e v97 = iconst.i64 0 -;; @003e v98 = iadd.i64 v14, v97 ; v97 = 0 -;; @003e v99 = iconst.i32 1 -;; @003e store notrap aligned region11 v99, v98+32 ; v99 = 1 -;; @003e v100 = load.i64 notrap aligned region9 v14+80 -;; @003e store.i64 notrap aligned region4 v32, v100+64 -;; @003e store.i64 notrap aligned region4 v33, v100+72 -;; @003e v101 = iconst.i64 2 -;; @003e v102 = load.i64 notrap aligned readonly can_move region0 v0+8 -;; @003e store notrap aligned region3 v101, v102+88 ; v101 = 2 -;; @003e store.i64 notrap aligned region3 v14, v102+96 -;; @003e v103 = iconst.i64 0 -;; @003e v104 = iadd v98, v103 ; v103 = 0 -;; @003e v105 = load.i64 notrap aligned region17 v104 -;; @003e store notrap aligned region1 v105, v58+24 -;; @003e v106 = load.i64 notrap aligned region15 v104+8 -;; @003e store notrap aligned region12 v106, v58+72 -;; @003e v107 = load.i64 notrap aligned region16 v104+16 -;; @003e store notrap aligned region13 v107, v58+64 -;; @003e v108 = load.i64 notrap aligned region6 v104+24 -;; @003e store notrap aligned region14 v108, v58+80 -;; @003e v109 = iconst.i64 96 -;; @003e v110 = iadd.i64 v29, v109 ; v109 = 96 -;; @003e v111 = load.i64 notrap aligned region18 v110 -;; @003e v112 = iconst.i64 -24 -;; @003e v113 = iadd v111, v112 ; v112 = -24 -;; @003e v114 = iconst.i64 96 -;; @003e v115 = iadd v100, v114 ; v114 = 96 -;; @003e v116 = load.i64 notrap aligned region18 v115 -;; @003e v117 = iconst.i64 -24 -;; @003e v118 = iadd v116, v117 ; v117 = -24 -;; @003e v119 = stack_addr.i64 ss1 -;; @003e v120 = load.i64 notrap aligned region8 v118 -;; @003e store notrap aligned region19 v120, v119 -;; @003e v121 = load.i64 notrap aligned region8 v113 -;; @003e store notrap aligned region8 v121, v118 -;; @003e v122 = load.i64 notrap aligned region8 v118+8 -;; @003e store notrap aligned region19 v122, v119+8 -;; @003e v123 = load.i64 notrap aligned region8 v113+8 -;; @003e store notrap aligned region8 v123, v118+8 -;; @003e v124 = load.i64 notrap aligned region8 v118+16 -;; @003e store notrap aligned region19 v124, v119+16 -;; @003e v125 = load.i64 notrap aligned region8 v113+16 -;; @003e store notrap aligned region8 v125, v118+16 -;; @003e v126 = iconst.i64 3 -;; @003e v127 = iconst.i64 32 -;; @003e v128 = ishl v126, v127 ; v126 = 3, v127 = 32 -;; @003e v129 = stack_switch v113, v119, v128 -;; @003e v130 = iconst.i64 32 -;; @003e v131 = ushr v129, v130 ; v130 = 32 -;; @003e v132 = iconst.i64 5 -;; @003e v133 = icmp eq v131, v132 ; v132 = 5 -;; @003e brif v133, block11, block12 +;; block10(v72: i64): +;; @003e store.i128 notrap aligned region8 v71, v72 +;; @003e v98 = iconst.i64 0 +;; @003e v99 = iadd.i64 v15, v98 ; v98 = 0 +;; @003e v100 = iconst.i32 1 +;; @003e store notrap aligned region11 v100, v99+32 ; v100 = 1 +;; @003e v101 = load.i64 notrap aligned region9 v15+80 +;; @003e store.i64 notrap aligned region4 v33, v101+64 +;; @003e store.i64 notrap aligned region4 v34, v101+72 +;; @003e v102 = iconst.i64 2 +;; @003e v103 = load.i64 notrap aligned readonly can_move region0 v0+8 +;; @003e store notrap aligned region3 v102, v103+88 ; v102 = 2 +;; @003e store.i64 notrap aligned region3 v15, v103+96 +;; @003e v104 = iconst.i64 0 +;; @003e v105 = iadd v99, v104 ; v104 = 0 +;; @003e v106 = load.i64 notrap aligned region17 v105 +;; @003e store notrap aligned region1 v106, v59+24 +;; @003e v107 = load.i64 notrap aligned region15 v105+8 +;; @003e store notrap aligned region12 v107, v59+72 +;; @003e v108 = load.i64 notrap aligned region16 v105+16 +;; @003e store notrap aligned region13 v108, v59+64 +;; @003e v109 = load.i64 notrap aligned region6 v105+24 +;; @003e store notrap aligned region14 v109, v59+80 +;; @003e v110 = iconst.i64 96 +;; @003e v111 = iadd.i64 v30, v110 ; v110 = 96 +;; @003e v112 = load.i64 notrap aligned region18 v111 +;; @003e v113 = iconst.i64 -24 +;; @003e v114 = iadd v112, v113 ; v113 = -24 +;; @003e v115 = iconst.i64 96 +;; @003e v116 = iadd v101, v115 ; v115 = 96 +;; @003e v117 = load.i64 notrap aligned region18 v116 +;; @003e v118 = iconst.i64 -24 +;; @003e v119 = iadd v117, v118 ; v118 = -24 +;; @003e v120 = stack_addr.i64 ss1 +;; @003e v121 = load.i64 notrap aligned region8 v119 +;; @003e store notrap aligned region19 v121, v120 +;; @003e v122 = load.i64 notrap aligned region8 v114 +;; @003e store notrap aligned region8 v122, v119 +;; @003e v123 = load.i64 notrap aligned region8 v119+8 +;; @003e store notrap aligned region19 v123, v120+8 +;; @003e v124 = load.i64 notrap aligned region8 v114+8 +;; @003e store notrap aligned region8 v124, v119+8 +;; @003e v125 = load.i64 notrap aligned region8 v119+16 +;; @003e store notrap aligned region19 v125, v120+16 +;; @003e v126 = load.i64 notrap aligned region8 v114+16 +;; @003e store notrap aligned region8 v126, v119+16 +;; @003e v127 = iconst.i64 3 +;; @003e v128 = iconst.i64 32 +;; @003e v129 = ishl v127, v128 ; v127 = 3, v128 = 32 +;; @003e v130 = stack_switch v114, v120, v129 +;; @003e v131 = iconst.i64 32 +;; @003e v132 = ushr v130, v131 ; v131 = 32 +;; @003e v133 = iconst.i64 5 +;; @003e v134 = icmp eq v132, v133 ; v133 = 5 +;; @003e brif v134, block11, block12 ;; ;; block11 cold: -;; @003e v134 = iconst.i64 136 -;; @003e v135 = iadd.i64 v27, v134 ; v134 = 136 -;; @003e v136 = load.i64 notrap aligned region5 v135+8 -;; @003e v137 = load.i32 notrap aligned region8 v136 -;; @003e v138 = iconst.i32 0 -;; @003e store notrap aligned region7 v138, v135 ; v138 = 0 +;; @003e v135 = iconst.i64 144 +;; @003e v136 = iadd.i64 v28, v135 ; v135 = 144 +;; @003e v137 = load.i64 notrap aligned region5 v136+8 +;; @003e v138 = load.i32 notrap aligned region8 v137 ;; @003e v139 = iconst.i32 0 -;; @003e store notrap aligned region10 v139, v135+4 ; v139 = 0 -;; @003e v140 = iconst.i64 0 -;; @003e store notrap aligned region5 v140, v135+8 ; v140 = 0 -;; @003e try_call fn2(v0, v137), sig2, block13, [ context v0 ] +;; @003e store notrap aligned region7 v139, v136 ; v139 = 0 +;; @003e v140 = iconst.i32 0 +;; @003e store notrap aligned region10 v140, v136+4 ; v140 = 0 +;; @003e v141 = iconst.i64 0 +;; @003e store notrap aligned region5 v141, v136+8 ; v141 = 0 +;; @003e try_call fn2(v0, v138), sig2, block13, [ context v0 ] ;; ;; block13: ;; @003e trap user12 ;; ;; block12: -;; @003e v141 = iconst.i64 136 -;; @003e v142 = iadd.i64 v27, v141 ; v141 = 136 -;; @003e v143 = load.i64 notrap aligned region5 v142+8 -;; @003e v144 = iconst.i32 0 -;; @003e store notrap aligned region7 v144, v142 ; v144 = 0 +;; @003e v142 = iconst.i64 144 +;; @003e v143 = iadd.i64 v28, v142 ; v142 = 144 +;; @003e v144 = load.i64 notrap aligned region5 v143+8 ;; @003e v145 = iconst.i32 0 -;; @003e store notrap aligned region10 v145, v142+4 ; v145 = 0 -;; @003e v146 = iconst.i64 0 -;; @003e store notrap aligned region5 v146, v142+8 ; v146 = 0 +;; @003e store notrap aligned region7 v145, v143 ; v145 = 0 +;; @003e v146 = iconst.i32 0 +;; @003e store notrap aligned region10 v146, v143+4 ; v146 = 0 +;; @003e v147 = iconst.i64 0 +;; @003e store notrap aligned region5 v147, v143+8 ; v147 = 0 ;; @0041 jump block1 ;; ;; block1: @@ -320,7 +321,7 @@ ;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 ;; gv2 = load.i64 notrap aligned region1 gv1+24 ;; sig0 = (i64 vmctx, i32) -> i64 tail -;; sig1 = (i64 vmctx, i64, i32, i32) -> i64 tail +;; sig1 = (i64 vmctx, i64, i32, i32, i32) -> i64 tail ;; sig2 = (i64 vmctx) tail ;; fn0 = colocated u805306368:6 sig0 ;; fn1 = colocated u805306368:42 sig1 @@ -333,212 +334,213 @@ ;; @0049 trapz v3, user16 ;; @0049 v4 = iconst.i32 0 ;; @0049 v5 = iconst.i32 0 -;; @0049 v6 = call fn1(v0, v3, v4, v5) ; v4 = 0, v5 = 0 -;; @0049 v7 = load.i64 notrap aligned region2 v6+88 -;; @0049 v8 = uextend.i128 v6 +;; @0049 v6 = iconst.i32 0 +;; @0049 v7 = call fn1(v0, v3, v4, v5, v6) ; v4 = 0, v5 = 0, v6 = 0 +;; @0049 v8 = load.i64 notrap aligned region2 v7+88 ;; @0049 v9 = uextend.i128 v7 -;; @0049 v10 = iconst.i64 64 -;; @0049 v11 = uextend.i128 v10 ; v10 = 64 -;; @0049 v12 = ishl v9, v11 -;; @0049 v13 = bor v12, v8 +;; @0049 v10 = uextend.i128 v8 +;; @0049 v11 = iconst.i64 64 +;; @0049 v12 = uextend.i128 v11 ; v11 = 64 +;; @0049 v13 = ishl v10, v12 +;; @0049 v14 = bor v13, v9 ;; @004b jump block2 ;; ;; block2: -;; @004b v14 = ireduce.i64 v13 -;; @004b v15 = iconst.i64 64 -;; @004b v16 = uextend.i128 v15 ; v15 = 64 -;; @004b v17 = ushr.i128 v13, v16 -;; @004b v18 = ireduce.i64 v17 -;; @004b trapz v14, user16 -;; @004b v19 = load.i64 notrap aligned region2 v14+88 -;; @004b v20 = icmp eq v19, v18 -;; @004b trapz v20, user23 -;; @004b v21 = iconst.i64 1 -;; @004b v22 = iadd v19, v21 ; v21 = 1 -;; @004b store notrap aligned region2 v22, v14+88 -;; @004b v23 = load.i64 notrap aligned region3 v14+80 -;; @004b v24 = load.i64 notrap aligned readonly can_move region0 v0+8 -;; @004b v25 = load.i64 notrap aligned region4 v24+88 -;; @004b v26 = load.i64 notrap aligned region4 v24+96 -;; @004b store notrap aligned region5 v25, v23+64 -;; @004b store notrap aligned region5 v26, v23+72 -;; @004b v27 = iconst.i64 0 -;; @004b store notrap aligned region3 v27, v14+80 ; v27 = 0 -;; @004b v28 = iconst.i64 2 -;; @004b v29 = load.i64 notrap aligned readonly can_move region0 v0+8 -;; @004b store notrap aligned region4 v28, v29+88 ; v28 = 2 -;; @004b store notrap aligned region4 v14, v29+96 -;; @004b v30 = iconst.i64 0 -;; @004b v31 = iadd v14, v30 ; v30 = 0 -;; @004b v32 = iconst.i32 1 -;; @004b store notrap aligned region6 v32, v31+32 ; v32 = 1 -;; @004b v33 = iconst.i32 2 -;; @004b store notrap aligned region6 v33, v26+32 ; v33 = 2 -;; @004b v34 = load.i64 notrap aligned readonly can_move region0 v0+8 -;; @004b v35 = iconst.i64 0 -;; @004b v36 = iadd v26, v35 ; v35 = 0 -;; @004b v37 = load.i64 notrap aligned region7 v34+72 -;; @004b v38 = load.i64 notrap aligned region8 v34+64 -;; @004b v39 = load.i64 notrap aligned region9 v34+80 -;; @004b store notrap aligned region10 v37, v36+8 -;; @004b store notrap aligned region11 v38, v36+16 -;; @004b store notrap aligned region12 v39, v36+24 -;; @004b v40 = load.i64 notrap aligned region1 v34+24 -;; @004b store notrap aligned region13 v40, v36 -;; @004b v41 = iconst.i64 0 -;; @004b v42 = iadd v31, v41 ; v41 = 0 -;; @004b v43 = load.i64 notrap aligned region13 v42 -;; @004b store notrap aligned region1 v43, v34+24 -;; @004b v44 = load.i64 notrap aligned region10 v42+8 -;; @004b store notrap aligned region7 v44, v34+72 -;; @004b v45 = load.i64 notrap aligned region11 v42+16 -;; @004b store notrap aligned region8 v45, v34+64 -;; @004b v46 = load.i64 notrap aligned region12 v42+24 -;; @004b store notrap aligned region9 v46, v34+80 -;; @004b v47 = iconst.i64 40 -;; @004b v48 = iadd v26, v47 ; v47 = 40 -;; @004b v49 = iconst.i32 1 -;; @004b v50 = stack_addr.i64 ss0 -;; @004b store notrap aligned region14 v49, v48+4 ; v49 = 1 -;; @004b store notrap aligned region15 v50, v48+8 -;; @004b v51 = iconst.i64 48 -;; @004b v52 = iadd.i64 v0, v51 ; v51 = 48 -;; @004b v53 = iconst.i32 1 -;; @004b v54 = load.i64 notrap aligned region15 v48+8 -;; @004b store notrap aligned region16 v52, v54 -;; @004b store notrap aligned region17 v53, v48 ; v53 = 1 -;; @004b v55 = iconst.i32 0 -;; @004b store notrap aligned region12 v55, v26+56 ; v55 = 0 -;; @004b v56 = iconst.i64 1 -;; @004b v57 = iconst.i64 32 -;; @004b v58 = ishl v56, v57 ; v56 = 1, v57 = 32 -;; @004b v59 = iconst.i64 96 -;; @004b v60 = iadd v23, v59 ; v59 = 96 -;; @004b v61 = load.i64 notrap aligned region18 v60 -;; @004b v62 = iconst.i64 -24 -;; @004b v63 = iadd v61, v62 ; v62 = -24 -;; @004b v64 = stack_switch v63, v63, v58 -;; @004b v65 = load.i64 notrap aligned readonly can_move region0 v0+8 -;; @004b v66 = load.i64 notrap aligned region4 v65+88 -;; @004b v67 = load.i64 notrap aligned region4 v65+96 -;; @004b v68 = load.i64 notrap aligned readonly can_move region0 v0+8 -;; @004b store notrap aligned region4 v25, v68+88 -;; @004b store notrap aligned region4 v26, v68+96 -;; @004b v69 = iconst.i32 1 -;; @004b store notrap aligned region6 v69, v26+32 ; v69 = 1 -;; @004b v70 = iconst.i32 0 -;; @004b store notrap aligned region17 v70, v48 ; v70 = 0 +;; @004b v15 = ireduce.i64 v14 +;; @004b v16 = iconst.i64 64 +;; @004b v17 = uextend.i128 v16 ; v16 = 64 +;; @004b v18 = ushr.i128 v14, v17 +;; @004b v19 = ireduce.i64 v18 +;; @004b trapz v15, user16 +;; @004b v20 = load.i64 notrap aligned region2 v15+88 +;; @004b v21 = icmp eq v20, v19 +;; @004b trapz v21, user23 +;; @004b v22 = iconst.i64 1 +;; @004b v23 = iadd v20, v22 ; v22 = 1 +;; @004b store notrap aligned region2 v23, v15+88 +;; @004b v24 = load.i64 notrap aligned region3 v15+80 +;; @004b v25 = load.i64 notrap aligned readonly can_move region0 v0+8 +;; @004b v26 = load.i64 notrap aligned region4 v25+88 +;; @004b v27 = load.i64 notrap aligned region4 v25+96 +;; @004b store notrap aligned region5 v26, v24+64 +;; @004b store notrap aligned region5 v27, v24+72 +;; @004b v28 = iconst.i64 0 +;; @004b store notrap aligned region3 v28, v15+80 ; v28 = 0 +;; @004b v29 = iconst.i64 2 +;; @004b v30 = load.i64 notrap aligned readonly can_move region0 v0+8 +;; @004b store notrap aligned region4 v29, v30+88 ; v29 = 2 +;; @004b store notrap aligned region4 v15, v30+96 +;; @004b v31 = iconst.i64 0 +;; @004b v32 = iadd v15, v31 ; v31 = 0 +;; @004b v33 = iconst.i32 1 +;; @004b store notrap aligned region6 v33, v32+32 ; v33 = 1 +;; @004b v34 = iconst.i32 2 +;; @004b store notrap aligned region6 v34, v27+32 ; v34 = 2 +;; @004b v35 = load.i64 notrap aligned readonly can_move region0 v0+8 +;; @004b v36 = iconst.i64 0 +;; @004b v37 = iadd v27, v36 ; v36 = 0 +;; @004b v38 = load.i64 notrap aligned region7 v35+72 +;; @004b v39 = load.i64 notrap aligned region8 v35+64 +;; @004b v40 = load.i64 notrap aligned region9 v35+80 +;; @004b store notrap aligned region10 v38, v37+8 +;; @004b store notrap aligned region11 v39, v37+16 +;; @004b store notrap aligned region12 v40, v37+24 +;; @004b v41 = load.i64 notrap aligned region1 v35+24 +;; @004b store notrap aligned region13 v41, v37 +;; @004b v42 = iconst.i64 0 +;; @004b v43 = iadd v32, v42 ; v42 = 0 +;; @004b v44 = load.i64 notrap aligned region13 v43 +;; @004b store notrap aligned region1 v44, v35+24 +;; @004b v45 = load.i64 notrap aligned region10 v43+8 +;; @004b store notrap aligned region7 v45, v35+72 +;; @004b v46 = load.i64 notrap aligned region11 v43+16 +;; @004b store notrap aligned region8 v46, v35+64 +;; @004b v47 = load.i64 notrap aligned region12 v43+24 +;; @004b store notrap aligned region9 v47, v35+80 +;; @004b v48 = iconst.i64 40 +;; @004b v49 = iadd v27, v48 ; v48 = 40 +;; @004b v50 = iconst.i32 1 +;; @004b v51 = stack_addr.i64 ss0 +;; @004b store notrap aligned region14 v50, v49+4 ; v50 = 1 +;; @004b store notrap aligned region15 v51, v49+8 +;; @004b v52 = iconst.i64 48 +;; @004b v53 = iadd.i64 v0, v52 ; v52 = 48 +;; @004b v54 = iconst.i32 1 +;; @004b v55 = load.i64 notrap aligned region15 v49+8 +;; @004b store notrap aligned region16 v53, v55 +;; @004b store notrap aligned region17 v54, v49 ; v54 = 1 +;; @004b v56 = iconst.i32 0 +;; @004b store notrap aligned region12 v56, v27+56 ; v56 = 0 +;; @004b v57 = iconst.i64 1 +;; @004b v58 = iconst.i64 32 +;; @004b v59 = ishl v57, v58 ; v57 = 1, v58 = 32 +;; @004b v60 = iconst.i64 96 +;; @004b v61 = iadd v24, v60 ; v60 = 96 +;; @004b v62 = load.i64 notrap aligned region18 v61 +;; @004b v63 = iconst.i64 -24 +;; @004b v64 = iadd v62, v63 ; v63 = -24 +;; @004b v65 = stack_switch v64, v64, v59 +;; @004b v66 = load.i64 notrap aligned readonly can_move region0 v0+8 +;; @004b v67 = load.i64 notrap aligned region4 v66+88 +;; @004b v68 = load.i64 notrap aligned region4 v66+96 +;; @004b v69 = load.i64 notrap aligned readonly can_move region0 v0+8 +;; @004b store notrap aligned region4 v26, v69+88 +;; @004b store notrap aligned region4 v27, v69+96 +;; @004b v70 = iconst.i32 1 +;; @004b store notrap aligned region6 v70, v27+32 ; v70 = 1 ;; @004b v71 = iconst.i32 0 -;; @004b store notrap aligned region14 v71, v48+4 ; v71 = 0 -;; @004b v72 = iconst.i64 0 -;; @004b store notrap aligned region15 v72, v48+8 ; v72 = 0 -;; @004b store notrap aligned region12 v27, v26+56 ; v27 = 0 -;; @004b brif v64, block6, block3 +;; @004b store notrap aligned region17 v71, v49 ; v71 = 0 +;; @004b v72 = iconst.i32 0 +;; @004b store notrap aligned region14 v72, v49+4 ; v72 = 0 +;; @004b v73 = iconst.i64 0 +;; @004b store notrap aligned region15 v73, v49+8 ; v73 = 0 +;; @004b store notrap aligned region12 v28, v27+56 ; v28 = 0 +;; @004b brif v65, block6, block3 ;; ;; block6: -;; @004b v73 = iconst.i64 32 -;; @004b v74 = ushr.i64 v64, v73 ; v73 = 32 -;; @004b v75 = iconst.i64 4 -;; @004b v76 = icmp eq v74, v75 ; v75 = 4 -;; @004b brif v76, block5, block4 +;; @004b v74 = iconst.i64 32 +;; @004b v75 = ushr.i64 v65, v74 ; v74 = 32 +;; @004b v76 = iconst.i64 4 +;; @004b v77 = icmp eq v75, v76 ; v76 = 4 +;; @004b brif v77, block5, block4 ;; ;; block5 cold: -;; @004b v77 = iconst.i64 0 -;; @004b v78 = iadd.i64 v67, v77 ; v77 = 0 -;; @004b v79 = iconst.i32 5 -;; @004b store notrap aligned region6 v79, v78+32 ; v79 = 5 -;; @004b v80 = iconst.i64 0 -;; @004b v81 = iadd.i64 v26, v80 ; v80 = 0 -;; @004b v82 = load.i64 notrap aligned region13 v81 -;; @004b store notrap aligned region1 v82, v34+24 -;; @004b v83 = load.i64 notrap aligned region10 v81+8 -;; @004b store notrap aligned region7 v83, v34+72 -;; @004b v84 = load.i64 notrap aligned region11 v81+16 -;; @004b store notrap aligned region8 v84, v34+64 -;; @004b v85 = load.i64 notrap aligned region12 v81+24 -;; @004b store notrap aligned region9 v85, v34+80 -;; @004b v86 = iconst.i64 120 -;; @004b v87 = iadd.i64 v67, v86 ; v86 = 120 -;; @004b v88 = iconst.i32 0 -;; @004b store notrap aligned region17 v88, v87 ; v88 = 0 +;; @004b v78 = iconst.i64 0 +;; @004b v79 = iadd.i64 v68, v78 ; v78 = 0 +;; @004b v80 = iconst.i32 5 +;; @004b store notrap aligned region6 v80, v79+32 ; v80 = 5 +;; @004b v81 = iconst.i64 0 +;; @004b v82 = iadd.i64 v27, v81 ; v81 = 0 +;; @004b v83 = load.i64 notrap aligned region13 v82 +;; @004b store notrap aligned region1 v83, v35+24 +;; @004b v84 = load.i64 notrap aligned region10 v82+8 +;; @004b store notrap aligned region7 v84, v35+72 +;; @004b v85 = load.i64 notrap aligned region11 v82+16 +;; @004b store notrap aligned region8 v85, v35+64 +;; @004b v86 = load.i64 notrap aligned region12 v82+24 +;; @004b store notrap aligned region9 v86, v35+80 +;; @004b v87 = iconst.i64 120 +;; @004b v88 = iadd.i64 v68, v87 ; v87 = 120 ;; @004b v89 = iconst.i32 0 -;; @004b store notrap aligned region14 v89, v87+4 ; v89 = 0 -;; @004b v90 = iconst.i64 0 -;; @004b store notrap aligned region15 v90, v87+8 ; v90 = 0 -;; @004b v91 = iconst.i64 136 -;; @004b v92 = iadd.i64 v67, v91 ; v91 = 136 -;; @004b v93 = iconst.i32 0 -;; @004b store notrap aligned region17 v93, v92 ; v93 = 0 +;; @004b store notrap aligned region17 v89, v88 ; v89 = 0 +;; @004b v90 = iconst.i32 0 +;; @004b store notrap aligned region14 v90, v88+4 ; v90 = 0 +;; @004b v91 = iconst.i64 0 +;; @004b store notrap aligned region15 v91, v88+8 ; v91 = 0 +;; @004b v92 = iconst.i64 144 +;; @004b v93 = iadd.i64 v68, v92 ; v92 = 144 ;; @004b v94 = iconst.i32 0 -;; @004b store notrap aligned region14 v94, v92+4 ; v94 = 0 -;; @004b v95 = iconst.i64 0 -;; @004b store notrap aligned region15 v95, v92+8 ; v95 = 0 +;; @004b store notrap aligned region17 v94, v93 ; v94 = 0 +;; @004b v95 = iconst.i32 0 +;; @004b store notrap aligned region14 v95, v93+4 ; v95 = 0 +;; @004b v96 = iconst.i64 0 +;; @004b store notrap aligned region15 v96, v93+8 ; v96 = 0 ;; @004b try_call fn2(v0), sig2, block8, [ context v0 ] ;; ;; block8: ;; @004b trap user12 ;; ;; block4: -;; @004b v96 = iconst.i64 0 -;; @004b v97 = iadd.i64 v67, v96 ; v96 = 0 -;; @004b v98 = iconst.i64 0 -;; @004b v99 = iadd v97, v98 ; v98 = 0 -;; @004b v100 = load.i64 notrap aligned region7 v34+72 -;; @004b v101 = load.i64 notrap aligned region8 v34+64 -;; @004b v102 = load.i64 notrap aligned region9 v34+80 -;; @004b store notrap aligned region10 v100, v99+8 -;; @004b store notrap aligned region11 v101, v99+16 -;; @004b store notrap aligned region12 v102, v99+24 -;; @004b v103 = iconst.i64 0 -;; @004b v104 = iadd.i64 v26, v103 ; v103 = 0 -;; @004b v105 = load.i64 notrap aligned region13 v104 -;; @004b store notrap aligned region1 v105, v34+24 -;; @004b v106 = load.i64 notrap aligned region10 v104+8 -;; @004b store notrap aligned region7 v106, v34+72 -;; @004b v107 = load.i64 notrap aligned region11 v104+16 -;; @004b store notrap aligned region8 v107, v34+64 -;; @004b v108 = load.i64 notrap aligned region12 v104+24 -;; @004b store notrap aligned region9 v108, v34+80 -;; @004b v109 = ireduce.i32 v64 -;; @004b v110 = load.i64 notrap aligned region2 v67+88 -;; @004b v111 = uextend.i128 v67 -;; @004b v112 = uextend.i128 v110 -;; @004b v113 = iconst.i64 64 -;; @004b v114 = uextend.i128 v113 ; v113 = 64 -;; @004b v115 = ishl v112, v114 -;; @004b v116 = bor v115, v111 +;; @004b v97 = iconst.i64 0 +;; @004b v98 = iadd.i64 v68, v97 ; v97 = 0 +;; @004b v99 = iconst.i64 0 +;; @004b v100 = iadd v98, v99 ; v99 = 0 +;; @004b v101 = load.i64 notrap aligned region7 v35+72 +;; @004b v102 = load.i64 notrap aligned region8 v35+64 +;; @004b v103 = load.i64 notrap aligned region9 v35+80 +;; @004b store notrap aligned region10 v101, v100+8 +;; @004b store notrap aligned region11 v102, v100+16 +;; @004b store notrap aligned region12 v103, v100+24 +;; @004b v104 = iconst.i64 0 +;; @004b v105 = iadd.i64 v27, v104 ; v104 = 0 +;; @004b v106 = load.i64 notrap aligned region13 v105 +;; @004b store notrap aligned region1 v106, v35+24 +;; @004b v107 = load.i64 notrap aligned region10 v105+8 +;; @004b store notrap aligned region7 v107, v35+72 +;; @004b v108 = load.i64 notrap aligned region11 v105+16 +;; @004b store notrap aligned region8 v108, v35+64 +;; @004b v109 = load.i64 notrap aligned region12 v105+24 +;; @004b store notrap aligned region9 v109, v35+80 +;; @004b v110 = ireduce.i32 v65 +;; @004b v111 = load.i64 notrap aligned region2 v68+88 +;; @004b v112 = uextend.i128 v68 +;; @004b v113 = uextend.i128 v111 +;; @004b v114 = iconst.i64 64 +;; @004b v115 = uextend.i128 v114 ; v114 = 64 +;; @004b v116 = ishl v113, v115 +;; @004b v117 = bor v116, v112 ;; @004b jump block7 ;; ;; block9 cold: ;; @004b trap user12 ;; ;; block7: -;; @004b br_table v109, block9, [] +;; @004b br_table v110, block9, [] ;; ;; block3: -;; @004b v117 = iconst.i64 0 -;; @004b v118 = iadd.i64 v26, v117 ; v117 = 0 -;; @004b v119 = load.i64 notrap aligned region13 v118 -;; @004b store notrap aligned region1 v119, v34+24 -;; @004b v120 = load.i64 notrap aligned region10 v118+8 -;; @004b store notrap aligned region7 v120, v34+72 -;; @004b v121 = load.i64 notrap aligned region11 v118+16 -;; @004b store notrap aligned region8 v121, v34+64 -;; @004b v122 = load.i64 notrap aligned region12 v118+24 -;; @004b store notrap aligned region9 v122, v34+80 -;; @004b v123 = iconst.i64 0 -;; @004b v124 = iadd.i64 v67, v123 ; v123 = 0 -;; @004b v125 = iconst.i32 4 -;; @004b store notrap aligned region6 v125, v124+32 ; v125 = 4 -;; @004b v126 = iconst.i64 120 -;; @004b v127 = iadd.i64 v67, v126 ; v126 = 120 -;; @004b v128 = load.i64 notrap aligned region15 v127+8 -;; @004b v129 = iconst.i32 0 -;; @004b store notrap aligned region17 v129, v127 ; v129 = 0 +;; @004b v118 = iconst.i64 0 +;; @004b v119 = iadd.i64 v27, v118 ; v118 = 0 +;; @004b v120 = load.i64 notrap aligned region13 v119 +;; @004b store notrap aligned region1 v120, v35+24 +;; @004b v121 = load.i64 notrap aligned region10 v119+8 +;; @004b store notrap aligned region7 v121, v35+72 +;; @004b v122 = load.i64 notrap aligned region11 v119+16 +;; @004b store notrap aligned region8 v122, v35+64 +;; @004b v123 = load.i64 notrap aligned region12 v119+24 +;; @004b store notrap aligned region9 v123, v35+80 +;; @004b v124 = iconst.i64 0 +;; @004b v125 = iadd.i64 v68, v124 ; v124 = 0 +;; @004b v126 = iconst.i32 4 +;; @004b store notrap aligned region6 v126, v125+32 ; v126 = 4 +;; @004b v127 = iconst.i64 120 +;; @004b v128 = iadd.i64 v68, v127 ; v127 = 120 +;; @004b v129 = load.i64 notrap aligned region15 v128+8 ;; @004b v130 = iconst.i32 0 -;; @004b store notrap aligned region14 v130, v127+4 ; v130 = 0 -;; @004b v131 = iconst.i64 0 -;; @004b store notrap aligned region15 v131, v127+8 ; v131 = 0 +;; @004b store notrap aligned region17 v130, v128 ; v130 = 0 +;; @004b v131 = iconst.i32 0 +;; @004b store notrap aligned region14 v131, v128+4 ; v131 = 0 +;; @004b v132 = iconst.i64 0 +;; @004b store notrap aligned region15 v132, v128+8 ; v132 = 0 ;; @0050 jump block1 ;; ;; block1: diff --git a/tests/misc_testsuite/stack-switching-and-gc.wast b/tests/misc_testsuite/stack-switching-and-gc.wast deleted file mode 100644 index 229f2fe16dd1..000000000000 --- a/tests/misc_testsuite/stack-switching-and-gc.wast +++ /dev/null @@ -1,13 +0,0 @@ -;;! stack_switching = true -;;! gc = true - -(assert_invalid - (module - (type $f (func)) - (type $c (cont $f)) - (type $s (struct (field (ref null $c)))) - (func (export "run") - (drop (struct.new_default $s)) - ) - ) - "Stack switching feature not compatible with GC") diff --git a/tests/misc_testsuite/stack-switching/gc.wast b/tests/misc_testsuite/stack-switching/gc.wast new file mode 100644 index 000000000000..5cbe6af86806 --- /dev/null +++ b/tests/misc_testsuite/stack-switching/gc.wast @@ -0,0 +1,363 @@ +;;! stack_switching = true +;;! gc = true +;;! function_references = true +;;! bulk_memory = true + +;; A GC reference in a parent frame must remain live while a child +;; continuation is running. +(module + (type $box (struct (field i32))) + (type $f (func)) + (type $c (cont $f)) + (tag $suspend) + + (import "wasmtime" "gc" (func $gc)) + + (func $child + (call $gc) + (suspend $suspend)) + (elem declare func $child) + + (func (export "parent-frame") (result i32) + (local $box (ref null $box)) + (local.set $box (struct.new $box (i32.const 42))) + + (drop + (block $on-suspend (result (ref $c)) + (resume $c + (on $suspend $on-suspend) + (cont.new $c (ref.func $child))) + (unreachable))) + + (struct.get $box 0 (local.get $box))) +) + +(assert_return (invoke "parent-frame") (i32.const 42)) + +;; Stack-map tracing follows the full parent chain, and execution can +;; subsequently re-enter every parent frame. +(module + (type $box (struct (field i32))) + (type $f (func (result i32))) + (type $c (cont $f)) + (tag $suspend) + + (import "wasmtime" "gc" (func $gc)) + + (func $child (result i32) + (call $gc) + (suspend $suspend) + (i32.const 1)) + (elem declare func $child) + + (func $middle (result i32) + (local $box (ref null $box)) + (local.set $box (struct.new $box (i32.const 20))) + (i32.add + (resume $c (cont.new $c (ref.func $child))) + (struct.get $box 0 (local.get $box)))) + (elem declare func $middle) + + (func (export "parent-chain") (result i32) + (local $box (ref null $box)) + (local $suspended (ref null $c)) + (local.set $box (struct.new $box (i32.const 100))) + + (local.set $suspended + (block $on-suspend (result (ref $c)) + (resume $c + (on $suspend $on-suspend) + (cont.new $c (ref.func $middle))) + (unreachable))) + + (i32.add + (resume $c (local.get $suspended)) + (struct.get $box 0 (local.get $box)))) +) + +(assert_return (invoke "parent-chain") (i32.const 121)) + +;; A bound GC reference is retained only by a fresh continuation's +;; argument buffer when collection runs before the continuation is +;; first resumed. +(module + (type $box (struct (field i32))) + (type $with-arg-f (func (param (ref $box)) (result i32))) + (type $without-arg-f (func (result i32))) + (type $with-arg-c (cont $with-arg-f)) + (type $without-arg-c (cont $without-arg-f)) + + (import "wasmtime" "gc" (func $gc)) + + (func $read-box (param $box (ref $box)) (result i32) + (struct.get $box 0 (local.get $box))) + (elem declare func $read-box) + + (func (export "bound-argument") (result i32) + (local $continuation (ref null $without-arg-c)) + + (local.set $continuation + (cont.bind $with-arg-c $without-arg-c + (struct.new $box (i32.const 73)) + (cont.new $with-arg-c (ref.func $read-box)))) + + (call $gc) + (resume $without-arg-c + (local.get $continuation))) +) + +(assert_return (invoke "bound-argument") (i32.const 73)) + +;; The same root metadata is used for the `values` buffer after a +;; continuation has suspended. +(module + (type $box (struct (field i32))) + (type $initial-f (func (result i32))) + (type $suspended-f (func (param (ref $box)) (result i32))) + (type $initial-c (cont $initial-f)) + (type $suspended-c (cont $suspended-f)) + (tag $yield (result (ref $box))) + + (import "wasmtime" "gc" (func $gc)) + + (func $suspend-then-read (result i32) + (struct.get $box 0 (suspend $yield))) + (elem declare func $suspend-then-read) + + (func (export "bound-suspended-argument") (result i32) + (local $suspended (ref null $suspended-c)) + (local $ready (ref null $initial-c)) + + (local.set $suspended + (block $on-yield (result (ref $suspended-c)) + (resume $initial-c + (on $yield $on-yield) + (cont.new $initial-c (ref.func $suspend-then-read))) + (unreachable))) + + (local.set $ready + (cont.bind $suspended-c $initial-c + (struct.new $box (i32.const 91)) + (local.get $suspended))) + + (call $gc) + (resume $initial-c (ref.as_non_null (local.get $ready)))) +) + +(assert_return (invoke "bound-suspended-argument") (i32.const 91)) + +;; Struct and array fields store continuation references through a +;; store-local side table. Collection may run while the aggregate is +;; the only place that retains the continuation object. +(module + (type $f (func (result i32))) + (type $c (cont $f)) + (type $holder (struct (field (mut (ref null $c))))) + (type $array (array (mut (ref null $c)))) + + (import "wasmtime" "gc" (func $gc)) + + (func $forty-two (result i32) + (i32.const 42)) + (elem declare func $forty-two) + + (func (export "struct-initialize") (result i32) + (local $holder (ref $holder)) + (local.set $holder + (struct.new $holder + (cont.new $c (ref.func $forty-two)))) + (call $gc) + (resume $c + (struct.get $holder 0 (local.get $holder)))) + + (func (export "struct-update") (result i32) + (local $holder (ref $holder)) + (local.set $holder (struct.new_default $holder)) + (struct.set $holder 0 + (local.get $holder) + (cont.new $c (ref.func $forty-two))) + (call $gc) + (resume $c + (struct.get $holder 0 (local.get $holder)))) + + (func (export "array-initialize") (result i32) + (local $array (ref $array)) + (local.set $array + (array.new_fixed $array 1 + (cont.new $c (ref.func $forty-two)))) + (call $gc) + (resume $c + (array.get $array (local.get $array) (i32.const 0)))) + + (func (export "array-update") (result i32) + (local $array (ref $array)) + (local.set $array (array.new_default $array (i32.const 1))) + (array.set $array + (local.get $array) + (i32.const 0) + (cont.new $c (ref.func $forty-two))) + (call $gc) + (resume $c + (array.get $array (local.get $array) (i32.const 0)))) + + (func (export "array-fill-and-copy") (result i32) + (local $source (ref $array)) + (local $destination (ref $array)) + (local.set $source (array.new_default $array (i32.const 1))) + (local.set $destination (array.new_default $array (i32.const 1))) + (array.fill $array + (local.get $source) + (i32.const 0) + (cont.new $c (ref.func $forty-two)) + (i32.const 1)) + (array.copy $array $array + (local.get $destination) + (i32.const 0) + (local.get $source) + (i32.const 0) + (i32.const 1)) + (call $gc) + (resume $c + (array.get $array (local.get $destination) (i32.const 0)))) + + (func (export "null-defaults") (result i32) + (local $holder (ref $holder)) + (local $array (ref $array)) + (local.set $holder (struct.new_default $holder)) + (local.set $array (array.new_default $array (i32.const 1))) + (i32.add + (ref.is_null (struct.get $holder 0 (local.get $holder))) + (ref.is_null + (array.get $array (local.get $array) (i32.const 0))))) + + ;; Loading an alias from a GC object must reproduce the interned + ;; revision witness. Consuming one alias therefore invalidates all + ;; other loads of the same field. + (func (export "consume-struct-field-twice") + (local $holder (ref $holder)) + (local.set $holder + (struct.new $holder + (cont.new $c (ref.func $forty-two)))) + (drop + (resume $c + (struct.get $holder 0 (local.get $holder)))) + (drop + (resume $c + (struct.get $holder 0 (local.get $holder))))) +) + +(assert_return (invoke "struct-initialize") (i32.const 42)) +(assert_return (invoke "struct-update") (i32.const 42)) +(assert_return (invoke "array-initialize") (i32.const 42)) +(assert_return (invoke "array-update") (i32.const 42)) +(assert_return (invoke "array-fill-and-copy") (i32.const 42)) +(assert_return (invoke "null-defaults") (i32.const 2)) +(assert_trap + (invoke "consume-struct-field-twice") + "continuation already consumed") + +;; A GC reference passed to a child continuation must remain live when +;; that child is unwound by `resume_throw`. The parent retains another +;; reference, catches the injected exception, and can still use the +;; object after a second collection. +(module + (type $box (struct (field i32))) + (type $child-f (func (param (ref $box)) (result i32))) + (type $parent-f (func (result i32))) + (type $child-c (cont $child-f)) + (type $parent-c (cont $parent-f)) + (tag $suspend) + (tag $exception) + + (import "wasmtime" "gc" (func $gc)) + + (func $child (param $box (ref $box)) (result i32) + ;; At this collection, both the child and its parent have the box + ;; in live stack slots. Keep the parameter live across the + ;; suspension point. + (call $gc) + (suspend $suspend) + (struct.get $box 0 (local.get $box))) + (elem declare func $child) + + (func $parent (result i32) + (local $box (ref $box)) + (local.set $box (struct.new $box (i32.const 101))) + + (block $caught + (try_table (catch $exception $caught) + (drop + (resume $child-c + (local.get $box) + (cont.new $child-c (ref.func $child))))) + (unreachable)) + + ;; The child continuation is terminal, so only the parent's stack + ;; slot retains the box at this collection. + (call $gc) + (struct.get $box 0 (local.get $box))) + (elem declare func $parent) + + (func (export "gc-ref-survives-resume-throw") (result i32) + (local $suspended (ref null $parent-c)) + + (local.set $suspended + (block $on-suspend (result (ref $parent-c)) + (resume $parent-c + (on $suspend $on-suspend) + (cont.new $parent-c (ref.func $parent))) + (unreachable))) + + (resume_throw $parent-c $exception + (local.get $suspended))) +) + +(assert_return + (invoke "gc-ref-survives-resume-throw") + (i32.const 101)) + +;; A GC reference carried by an exception injected with `resume_throw` +;; must become a live stack root when the exception is caught inside +;; the resumed continuation. +(module + (type $box (struct (field i32))) + (type $f (func (result i32))) + (type $c (cont $f)) + (tag $suspend) + (tag $exception (param (ref $box))) + + (import "wasmtime" "gc" (func $gc)) + + (func $catch-payload (result i32) + (local $box (ref null $box)) + + (local.set $box + (block $caught (result (ref $box)) + (try_table (result (ref $box)) (catch $exception $caught) + (suspend $suspend) + (unreachable)))) + + ;; The caught local is now the only reference to the exception + ;; payload. + (call $gc) + (struct.get $box 0 (ref.as_non_null (local.get $box)))) + (elem declare func $catch-payload) + + (func (export "gc-ref-in-resume-throw-payload") (result i32) + (local $suspended (ref null $c)) + + (local.set $suspended + (block $on-suspend (result (ref $c)) + (resume $c + (on $suspend $on-suspend) + (cont.new $c (ref.func $catch-payload))) + (unreachable))) + + (resume_throw $c $exception + (struct.new $box (i32.const 202)) + (local.get $suspended))) +) + +(assert_return + (invoke "gc-ref-in-resume-throw-payload") + (i32.const 202)) diff --git a/tests/misc_testsuite/stack-switching/issue13021.wast b/tests/misc_testsuite/stack-switching/issue13021.wast new file mode 100644 index 000000000000..949362e0190a --- /dev/null +++ b/tests/misc_testsuite/stack-switching/issue13021.wast @@ -0,0 +1,18 @@ +;;! gc = true +;;! stack_switching = true + +;; Regression test for issue https://github.com/bytecodealliance/wasmtime/issues/13021 +(module + (type $ft (func)) + (type $ct (cont $ft)) + (type $arr (array (mut (ref null $ct)))) + (func (export "boom") + (local $a (ref $arr)) + (local.set $a (array.new_default $arr (i32.const 2))) + (array.copy $arr $arr + (local.get $a) (i32.const 0) + (local.get $a) (i32.const 0) + (i32.const 1)) + ) +) +(assert_return (invoke "boom")) \ No newline at end of file diff --git a/tests/misc_testsuite/stack-switching/issue13022.wast b/tests/misc_testsuite/stack-switching/issue13022.wast new file mode 100644 index 000000000000..1ccf5a7c6991 --- /dev/null +++ b/tests/misc_testsuite/stack-switching/issue13022.wast @@ -0,0 +1,18 @@ +;;! gc = true +;;! stack_switching = true + +;; Regression test for https://github.com/bytecodealliance/wasmtime/issues/13022 +(module + (type $ft (func)) + (type $ct (cont $ft)) + (type $arr (array (mut (ref null $ct)))) + (func (export "boom") + (local $a (ref $arr)) + (local.set $a (array.new_default $arr (i32.const 2))) + (array.copy $arr $arr + (local.get $a) (i32.const 0) + (local.get $a) (i32.const 0) + (i32.const 1)) + ) +) +(assert_return (invoke "boom")) \ No newline at end of file