use std::mem;

use crate::asm::*;
use crate::asm::x86_64::*;
use crate::codegen::split_patch_point;
use crate::stats::{CompileError, trace_compile_phase};
use crate::virtualmem::CodePtr;
use crate::cruby::*;
use crate::backend::lir::*;
use crate::cast::*;
use crate::options::asm_dump;

// Use the x86 register type for this platform
pub type Reg = X86Reg;

/// Convert reg_no for MemBase::Reg into Reg, assuming it's a 64-bit GP register
pub fn mem_base_reg(reg_no: u8) -> Reg {
    Reg { num_bits: 64, reg_type: RegType::GP, reg_no }
}

// Callee-saved registers
pub const CFP: Opnd = Opnd::Reg(R13_REG);
pub const EC: Opnd = Opnd::Reg(R12_REG);
pub const SP: Opnd = Opnd::Reg(RBX_REG);

// C argument registers on this platform
pub const C_ARG_OPNDS: [Opnd; 6] = [
    Opnd::Reg(RDI_REG),
    Opnd::Reg(RSI_REG),
    Opnd::Reg(RDX_REG),
    Opnd::Reg(RCX_REG),
    Opnd::Reg(R8_REG),
    Opnd::Reg(R9_REG)
];
pub const C_ARG_REGREGS: [Reg; 6] = [RDI_REG, RSI_REG, RDX_REG, RCX_REG, R8_REG, R9_REG];

// C return value register on this platform
pub const C_RET_REG: Reg = RAX_REG;
pub const C_RET_OPND: Opnd = Opnd::Reg(RAX_REG);
pub const NATIVE_STACK_PTR: Opnd = Opnd::Reg(RSP_REG);
pub const NATIVE_BASE_PTR: Opnd = Opnd::Reg(RBP_REG);

impl CodeBlock {
    // The number of bytes that are generated by jmp_ptr
    pub fn jmp_ptr_bytes(&self) -> usize { 5 }
}

/// Map Opnd to X86Opnd
impl From<Opnd> for X86Opnd {
    fn from(opnd: Opnd) -> Self {
        match opnd {
            // NOTE: these operand types need to be lowered first
            //Value(VALUE),       // Immediate Ruby value, may be GC'd, movable
            //VReg(usize),     // Output of a preceding instruction in this block

            Opnd::VReg{..} => panic!("VReg operand made it past register allocation"),

            Opnd::UImm(val) => uimm_opnd(val),
            Opnd::Imm(val) => imm_opnd(val),
            Opnd::Value(VALUE(uimm)) => uimm_opnd(uimm as u64),

            // General-purpose register
            Opnd::Reg(reg) => X86Opnd::Reg(reg),

            // Memory operand with displacement
            Opnd::Mem(Mem{ base: MemBase::Reg(reg_no), num_bits, disp }) => {
                let reg = X86Reg {
                    reg_no,
                    num_bits: 64,
                    reg_type: RegType::GP
                };

                mem_opnd(num_bits, X86Opnd::Reg(reg), disp)
            }

            Opnd::None => panic!(
                "Attempted to lower an Opnd::None. This often happens when an out operand was not allocated for an instruction because the output of the instruction was not used. Please ensure you are using the output."
            ),

            _ => panic!("unsupported x86 operand type: {opnd:?}")
        }
    }
}

/// Also implement going from a reference to an operand for convenience.
impl From<&Opnd> for X86Opnd {
    fn from(opnd: &Opnd) -> Self {
        X86Opnd::from(*opnd)
    }
}

/// List of registers that can be used for register allocation.
/// This has the same number of registers for x86_64 and arm64.
/// SCRATCH0_OPND is excluded.
pub const ALLOC_REGS: &[Reg] = &[
    RDI_REG,
    RSI_REG,
    RDX_REG,
    RCX_REG,
    R8_REG,
    R9_REG,
    RAX_REG,
];

/// Special scratch register for intermediate processing. It should be used only by
/// [`Assembler::x86_scratch_split`] or [`Assembler::new_with_scratch_reg`].
const SCRATCH0_OPND: Opnd = Opnd::Reg(R11_REG);
const SCRATCH1_OPND: Opnd = Opnd::Reg(R10_REG);

/// A scratch register available for use by resolve_ssa to break register copy cycles.
/// Must not overlap with ALLOC_REGS or other preserved registers.
pub const SCRATCH_REG: Reg = R11_REG;

impl Assembler {
    // This keeps frame growth below the +/-4096-byte displacement range we rely
    // on for common stack-slot accesses on x86_64.
    const MAX_FRAME_STACK_SLOTS: usize = 2048;

    /// Return an Assembler with scratch registers disabled in the backend, and a scratch register.
    pub fn new_with_scratch_reg() -> (Self, Opnd) {
        (Self::new_with_accept_scratch_reg(true), SCRATCH0_OPND)
    }

    /// Return true if opnd contains a scratch reg
    pub fn has_scratch_reg(opnd: Opnd) -> bool {
        Self::has_reg(opnd, SCRATCH0_OPND.unwrap_reg())
    }

    /// Get the list of registers from which we can allocate on this platform
    pub fn get_alloc_regs() -> Vec<Reg> {
        ALLOC_REGS.to_vec()
    }

    /// Get a list of all of the caller-save registers
    pub fn get_caller_save_regs() -> Vec<Reg> {
        vec![RAX_REG, RCX_REG, RDX_REG, RSI_REG, RDI_REG, R8_REG, R9_REG, R10_REG, R11_REG]
    }

    /// How many bytes a call and a bare bones [Self::frame_setup] would change native SP
    pub fn frame_size() -> i32 {
        0x10
    }

    // These are the callee-saved registers in the x86-64 SysV ABI
    // RBX, RSP, RBP, and R12-R15

    /// Split IR instructions for the x86 platform
    fn x86_split(mut self) -> Assembler
    {
        let mut asm_local = Assembler::new_with_asm(&self);
        let asm = &mut asm_local;
        let mut iterator = self.instruction_iterator();

        while let Some((_index, mut insn)) = iterator.next(asm) {
            let is_load = matches!(insn, Insn::Load { .. } | Insn::LoadInto { .. });
            let is_jump = insn.is_jump();

            if !is_jump {
                insn.for_each_operand_mut(|opnd| {
                    // Lower Opnd::Value to Opnd::VReg or Opnd::UImm
                    if let Opnd::Value(value) = opnd {
                        // If the value is a special constant, and it fits in 32 bits,
                        // then we're going to output this as just an immediate
                        if value.special_const_p() {
                            if imm_num_bits(value.as_i64()) > 32 {
                                *opnd = asm.load(*opnd);
                            } else {
                                *opnd = Opnd::UImm(value.as_u64());
                            }
                            // If we're already loading it, don't load it again
                            // If it's a jump, then we want to let parallel move
                            // take care of the block params (otherwise we end up
                            // with loads between jump instructions)
                        } else if !is_load {
                            *opnd = asm.load(*opnd);
                        }
                    }
                });
            }

            // We are replacing instructions here so we know they are already
            // being used. It is okay not to use their output here.
            #[allow(unused_must_use)]
            match &mut insn {
                Insn::Add { .. } |
                Insn::Sub { .. } |
                Insn::Mul { .. } |
                Insn::And { .. } |
                Insn::Or { .. } |
                Insn::Xor { .. } => {
                    asm.push_insn(insn);
                },
                Insn::Cmp { left, right } => {
                    // Replace `cmp REG, 0` (4 bytes) with `test REG, REG` (3 bytes)
                    // when next IR is `je`, `jne`, `csel_e`, or `csel_ne`
                    match (&left, &right, iterator.peek().map(|(_, insn)| insn)) {
                        (Opnd::VReg { .. },
                         Opnd::UImm(0) | Opnd::Imm(0),
                         Some(Insn::Je(_) | Insn::Jne(_) | Insn::CSelE { .. } | Insn::CSelNE { .. })) => {
                            asm.push_insn(Insn::Test { left: *left, right: *left });
                        }
                        _ => {
                            // Split the instruction if `cmp` can't be encoded with given operands
                            match (&left, &right) {
                                // One of the operands should not be a memory operand
                                (Opnd::Mem(_), Opnd::Mem(_)) => {
                                    *right = asm.load(*right);
                                }
                                // The left operand needs to be either a register or a memory operand
                                (Opnd::UImm(_) | Opnd::Imm(_), _) => {
                                    *left = asm.load(*left);
                                }
                                _ => {},
                            }
                            asm.push_insn(insn);
                        }
                    }
                },
                Insn::Test { left, right } => {
                    match (&left, &right) {
                        (Opnd::Mem(_), Opnd::Mem(_)) => {
                            *right = asm.load(*right);
                        }
                        // The first operand can't be an immediate value
                        (Opnd::UImm(_) | Opnd::Imm(_), _) => {
                            *left = asm.load(*left);
                        }
                        _ => {}
                    }
                    asm.push_insn(insn);
                },
                // These instructions modify their input operand in-place, so we
                // need to load the input value to preserve it
                Insn::LShift { opnd, .. } |
                Insn::RShift { opnd, .. } |
                Insn::URShift { opnd, .. } => {
                    *opnd = asm.load(*opnd);
                    asm.push_insn(insn);
                },
                Insn::CSelZ { truthy, falsy, .. } |
                Insn::CSelNZ { truthy, falsy, .. } |
                Insn::CSelE { truthy, falsy, .. } |
                Insn::CSelNE { truthy, falsy, .. } |
                Insn::CSelL { truthy, falsy, .. } |
                Insn::CSelLE { truthy, falsy, .. } |
                Insn::CSelG { truthy, falsy, .. } |
                Insn::CSelGE { truthy, falsy, .. } => {
                    match *truthy {
                        // If we have an instruction output whose live range
                        // spans beyond this instruction, we have to load it.
                        Opnd::VReg { idx: _, .. } => {
                            if true /* conservatively assume vreg outlives insn */ {
                                *truthy = asm.load(*truthy);
                            }
                        },
                        Opnd::UImm(_) | Opnd::Imm(_) => {
                            *truthy = asm.load(*truthy);
                        },
                        // Opnd::Value could have already been split
                        Opnd::Value(_) if !matches!(truthy, Opnd::VReg { .. }) => {
                            *truthy = asm.load(*truthy);
                        },
                        _ => {}
                    }

                    match falsy {
                        Opnd::UImm(_) | Opnd::Imm(_) => {
                            *falsy = asm.load(*falsy);
                        },
                        _ => {}
                    }

                    asm.push_insn(insn);
                },
                Insn::Mov { dest, src } => {
                    if let Opnd::Mem(_) = dest {
                        asm.store(*dest, *src);
                    } else {
                        asm.mov(*dest, *src);
                    }
                },
                Insn::Not { opnd, .. } => {
                    match *opnd {
                        // If we have an instruction output whose live range
                        // spans beyond this instruction, we have to load it.
                        Opnd::VReg { idx: _, .. } => {
                            if true /* conservatively assume vreg outlives insn */ {
                                *opnd = asm.load(*opnd);
                            }
                        },
                        // We have to load memory and register operands to avoid
                        // corrupting them.
                        Opnd::Mem(_) | Opnd::Reg(_) => {
                            *opnd = asm.load(*opnd);
                        },
                        // Otherwise we can just reuse the existing operand.
                        _ => {},
                    };
                    asm.push_insn(insn);
                },
                Insn::CCall { data } => {
                    assert!(data.opnds.len() <= C_ARG_OPNDS.len());
                    // CCall argument setup is handled by handle_caller_saved_regs.
                    asm.push_insn(insn);
                },
                Insn::Lea { .. } => {
                    asm.push_insn(insn);
                },
                _ => {
                    asm.push_insn(insn);
                }
            }
        }

        asm_local
    }

    /// Split instructions using scratch registers. To maximize the use of the register pool
    /// for VRegs, most splits should happen in [`Self::x86_split`]. However, some instructions
    /// need to be split with registers after `alloc_regs`, e.g. for `compile_exits`, so
    /// this splits them and uses scratch registers for it.
    pub fn x86_scratch_split(self) -> Assembler {
        /// For some instructions, we want to be able to lower a 64-bit operand
        /// without requiring more registers to be available in the register
        /// allocator. So we just use the SCRATCH0_OPND register temporarily to hold
        /// the value before we immediately use it.
        fn split_64bit_immediate(asm: &mut Assembler, opnd: Opnd, scratch_opnd: Opnd) -> Opnd {
            match opnd {
                Opnd::Imm(value) => {
                    // 32-bit values will be sign-extended
                    if imm_num_bits(value) > 32 {
                        asm.mov(scratch_opnd, opnd);
                        scratch_opnd
                    } else {
                        opnd
                    }
                },
                Opnd::UImm(value) => {
                    // 32-bit values will be sign-extended
                    if imm_num_bits(value as i64) > 32 {
                        asm.mov(scratch_opnd, opnd);
                        scratch_opnd
                    } else {
                        Opnd::Imm(value as i64)
                    }
                },
                _ => opnd
            }
        }

        /// If a given operand is Opnd::Mem and it uses MemBase::Stack, lower it to MemBase::Reg(NATIVE_BASE_PTR).
        /// In general, `out` operand is a `VReg`, so it may use MemBase::Stack, but not MemBase::StackIndirect.
        fn lower_stack_membase(asm: &Assembler, opnd: Opnd) -> Opnd {
            match opnd {
                Opnd::Mem(Mem { base: stack_membase @ MemBase::Stack { .. }, disp: opnd_disp, num_bits: opnd_num_bits }) => {
                    // Convert MemBase::Stack to MemBase::Reg(NATIVE_BASE_PTR) with the
                    // correct stack displacement. The stack slot value lives directly at
                    // [NATIVE_BASE_PTR + stack_disp], so we just adjust the base and
                    // combine displacements -- no indirection needed.
                    let Mem { base, disp: stack_disp, .. } = asm.stack_state.stack_membase_to_mem(stack_membase);
                    Opnd::Mem(Mem { base, disp: stack_disp + opnd_disp, num_bits: opnd_num_bits })
                }
                Opnd::Mem(Mem { base: MemBase::StackIndirect { .. }, .. }) => {
                    unreachable!("lower_stack_membase expects {opnd:?} to not have MemBase::StackIndirect")
                }
                _ => opnd,
            }
        }

        /// If a given operand is Opnd::Mem and it uses MemBase::Stack, lower it to MemBase::Reg(NATIVE_BASE_PTR).
        /// For MemBase::StackIndirect, load the pointer from the stack slot into a scratch register.
        fn split_stack_membase(asm: &mut Assembler, opnd: Opnd, scratch_opnd: Opnd) -> Opnd {
            match opnd {
                Opnd::Mem(Mem { base: MemBase::Stack { .. }, .. }) => lower_stack_membase(asm, opnd),
                Opnd::Mem(Mem { base: MemBase::StackIndirect { stack_idx }, disp: opnd_disp, num_bits: opnd_num_bits }) => {
                    // The spilled value (a pointer) lives at a stack slot. Load it
                    // into a scratch register, then use the register as the base.
                    let stack_mem = asm.stack_state.stack_membase_to_mem(MemBase::Stack { stack_idx, num_bits: 64 });
                    asm.load_into(scratch_opnd, Opnd::Mem(stack_mem));
                    Opnd::Mem(Mem {
                        base: MemBase::Reg(scratch_opnd.unwrap_reg().reg_no),
                        disp: opnd_disp,
                        num_bits: opnd_num_bits,
                    })
                }
                _ => opnd,
            }
        }

        /// If opnd is Opnd::Mem, set scratch_reg to *opnd. Return Some(Opnd::Mem) if it needs to be written back from scratch_reg.
        fn split_memory_write(opnd: &mut Opnd, scratch_opnd: Opnd) -> Option<Opnd> {
            if let Opnd::Mem(_) = opnd {
                let mem_opnd = opnd.clone();
                *opnd = opnd.num_bits().map(|num_bits| scratch_opnd.with_num_bits(num_bits)).unwrap_or(scratch_opnd);
                Some(mem_opnd)
            } else {
                None
            }
        }

        fn assert_out_is_phys_reg_or_stack_mem(out: Opnd) {
            assert!(
                matches!(out, Opnd::Reg(_) | Opnd::Mem(Mem { base: MemBase::Stack { .. }, .. })),
                "x86_scratch_split expects out to be a physical register or stack memory, got {out:?}"
            );
        }

        /// If both opnd and other are Opnd::Mem, split opnd with scratch_opnd.
        fn split_if_both_memory(asm: &mut Assembler, opnd: Opnd, other: Opnd, scratch_opnd: Opnd) -> Opnd {
            if let (Opnd::Mem(_), Opnd::Mem(_)) = (opnd, other) {
                asm.load_into(scratch_opnd.with_num_bits(opnd.rm_num_bits()), opnd);
                scratch_opnd.with_num_bits(opnd.rm_num_bits())
            } else {
                opnd
            }
        }

        /// Move src to dst, splitting it with scratch_opnd if it's a Mem-to-Mem move. Skip it if dst == src.
        fn asm_mov(asm: &mut Assembler, dst: Opnd, src: Opnd, scratch_opnd: Opnd) {
            if dst != src {
                if let (Opnd::Mem(_), Opnd::Mem(_)) = (dst, src) {
                    asm.mov(scratch_opnd, src);
                    asm.mov(dst, scratch_opnd);
                } else if let (Opnd::Mem(_), Opnd::Value(value)) = (dst, src) {
                    if imm_num_bits(value.as_i64()) > 32 {
                        asm.mov(scratch_opnd, src);
                        asm.mov(dst, scratch_opnd);
                    } else {
                        asm.mov(dst, src);
                    }
                } else {
                    asm.mov(dst, src);
                }
            }
        }

        let mut asm_local = Assembler::new_with_asm_without_blocks(&self);
        asm_local.accept_scratch_reg = true;

        // Create one giant block to linearize everything into
        asm_local.new_block_without_id("linearized");

        let asm = &mut asm_local;

        // Get linearized instructions with branch parameters expanded into ParallelMov
        let linearized_insns = self.linearize_instructions();

        for insn in linearized_insns.iter() {
            let mut insn = insn.clone();
            match &mut insn {
                Insn::Add { left, right, out } |
                Insn::Sub { left, right, out } |
                Insn::And { left, right, out } |
                Insn::Or  { left, right, out } |
                Insn::Xor { left, right, out } => {
                    assert_out_is_phys_reg_or_stack_mem(*out);

                    // Sequential pipeline to lower binops into x86 two-operand form.
                    // Uses SCRATCH0 for left, SCRATCH1 for right to avoid conflicts.

                    // Phase 1: Protect right if it occupies the same location as out,
                    // since we'll overwrite out when moving left into it.
                    // Compare before lowering (Stack membases change during lowering).
                    let right_eq_out = out == right;
                    *out = lower_stack_membase(asm, *out);
                    if right_eq_out {
                        asm.mov(SCRATCH1_OPND, *out);
                        *right = SCRATCH1_OPND;
                    }

                    // Phase 2: Lower stack memory bases
                    *left = split_stack_membase(asm, *left, SCRATCH0_OPND);
                    if !right_eq_out {
                        *right = split_stack_membase(asm, *right, SCRATCH1_OPND);
                    }

                    // Phase 3: If right is a Mem whose base register equals the out
                    // register, materialize it before we clobber out.
                    if let (Opnd::Reg(out_reg), Opnd::Mem(Mem { base: MemBase::Reg(base_reg_no), .. })) = (*out, *right) {
                        if out_reg.reg_no == base_reg_no {
                            asm.mov(SCRATCH1_OPND, *right);
                            *right = SCRATCH1_OPND;
                        }
                    }

                    // Phase 4: x86 can't encode two memory operands.
                    if let (Opnd::Mem(_), Opnd::Mem(_)) = (*out, *right) {
                        asm.mov(SCRATCH1_OPND, *right);
                        *right = SCRATCH1_OPND;
                    }

                    // Phase 5: Move left into out (x86 two-operand form: OP clobbers dst).
                    // For Mem out, split large left immediates first (mov [mem], imm64 is invalid).
                    if let Opnd::Mem(_) = *out {
                        *left = split_64bit_immediate(asm, *left, SCRATCH0_OPND);
                    }
                    asm_mov(asm, *out, *left, SCRATCH0_OPND);
                    *left = *out;

                    // Phase 6: Split large right immediates (>32-bit) into a register.
                    *right = split_64bit_immediate(asm, *right, SCRATCH1_OPND);

                    // Phase 7: Emit the instruction.
                    asm.push_insn(insn);
                }
                Insn::Mul { left, right, out } => {
                    assert_out_is_phys_reg_or_stack_mem(*out);

                    *left = split_stack_membase(asm, *left, SCRATCH0_OPND);
                    *left = split_if_both_memory(asm, *left, *right, SCRATCH0_OPND);
                    *right = split_stack_membase(asm, *right, SCRATCH1_OPND);
                    *right = split_64bit_immediate(asm, *right, SCRATCH1_OPND);
                    *out = lower_stack_membase(asm, *out);

                    // imul doesn't have (Mem, Reg) encoding. Swap left and right in that case.
                    if let (Opnd::Mem(_), Opnd::Reg(_)) = (&left, &right) {
                        mem::swap(left, right);
                    }

                    let (out, left) = (*out, *left);
                    asm.push_insn(insn);
                    asm_mov(asm, out, left, SCRATCH0_OPND);
                }
                &mut Insn::Not { opnd, out } |
                &mut Insn::LShift { opnd, out, .. } |
                &mut Insn::RShift { opnd, out, .. } |
                &mut Insn::URShift { opnd, out, .. } => {
                    asm.push_insn(insn);
                    asm_mov(asm, out, opnd, SCRATCH0_OPND);
                }
                Insn::Test { left, right } |
                Insn::Cmp { left, right } => {
                    *left = split_stack_membase(asm, *left, SCRATCH1_OPND);
                    *right = split_stack_membase(asm, *right, SCRATCH0_OPND);
                    *right = split_if_both_memory(asm, *right, *left, SCRATCH0_OPND);

                    let num_bits = match right {
                        Opnd::Imm(value) => Some(imm_num_bits(*value)),
                        Opnd::UImm(value) => Some(uimm_num_bits(*value)),
                        _ => None
                    };

                    // If the immediate is less than 64 bits (like 32, 16, 8), and the operand
                    // sizes match, then we can represent it as an immediate in the instruction
                    // without moving it to a register first.
                    // IOW, 64 bit immediates must always be moved to a register
                    // before comparisons, where other sizes may be encoded
                    // directly in the instruction.
                    let use_imm = num_bits.is_some() && left.num_bits() == num_bits && num_bits.unwrap() < 64;
                    if !use_imm {
                        *right = split_64bit_immediate(asm, *right, SCRATCH0_OPND);
                    }
                    asm.push_insn(insn);
                }
                // For compile_exits, support splitting simple C arguments here
                Insn::CCall { data } if !data.opnds.is_empty() => {
                    for (i, opnd) in data.opnds.iter().enumerate() {
                        asm.load_into(C_ARG_OPNDS[i], *opnd);
                    }
                    data.opnds = vec![];
                    asm.push_insn(insn);
                }
                Insn::CSelZ { truthy: left, falsy: right, out } |
                Insn::CSelNZ { truthy: left, falsy: right, out } |
                Insn::CSelE { truthy: left, falsy: right, out } |
                Insn::CSelNE { truthy: left, falsy: right, out } |
                Insn::CSelL { truthy: left, falsy: right, out } |
                Insn::CSelLE { truthy: left, falsy: right, out } |
                Insn::CSelG { truthy: left, falsy: right, out } |
                Insn::CSelGE { truthy: left, falsy: right, out } => {
                    *left = split_stack_membase(asm, *left, SCRATCH1_OPND);
                    *right = split_stack_membase(asm, *right, SCRATCH0_OPND);
                    *right = split_if_both_memory(asm, *right, *left, SCRATCH0_OPND);
                    *out = lower_stack_membase(asm, *out);
                    let mem_out = split_memory_write(out, SCRATCH0_OPND);
                    asm.push_insn(insn);
                    if let Some(mem_out) = mem_out {
                        asm.store(mem_out, SCRATCH0_OPND);
                    }
                }
                Insn::Lea { opnd, out } => {
                    *opnd = split_stack_membase(asm, *opnd, SCRATCH0_OPND);
                    *out = lower_stack_membase(asm, *out);
                    let mem_out = split_memory_write(out, SCRATCH0_OPND);
                    asm.push_insn(insn);
                    if let Some(mem_out) = mem_out {
                        asm.store(mem_out, SCRATCH0_OPND);
                    }
                }
                Insn::LeaJumpTarget { target, out } => {
                    if let Target::Label(_) = target {
                        asm.push_insn(Insn::LeaJumpTarget { out: SCRATCH0_OPND, target: target.clone() });
                        asm.mov(*out, SCRATCH0_OPND);
                    }
                }
                Insn::Load { out, opnd } |
                Insn::LoadInto { dest: out, opnd } => {
                    *opnd = split_stack_membase(asm, *opnd, SCRATCH0_OPND);
                    // Split stack membase on out before checking for memory write
                    *out = lower_stack_membase(asm, *out);
                    let mem_out = split_memory_write(out, SCRATCH0_OPND);
                    asm.push_insn(insn);
                    if let Some(mem_out) = mem_out {
                        asm.store(mem_out, SCRATCH0_OPND.with_num_bits(mem_out.rm_num_bits()));
                    }
                }
                // Convert Opnd::const_ptr into Opnd::Mem. This split is done here to give
                // a register for compile_exits.
                &mut Insn::IncrCounter { mem, value } => {
                    assert!(matches!(mem, Opnd::UImm(_)));
                    asm.load_into(SCRATCH0_OPND, mem);
                    asm.incr_counter(Opnd::mem(64, SCRATCH0_OPND, 0), value);
                }
                &mut Insn::Mov { dest, src } => {
                    let dest = split_stack_membase(asm, dest, SCRATCH1_OPND);
                    let src = split_stack_membase(asm, src, SCRATCH0_OPND);
                    asm_mov(asm, dest, src, SCRATCH0_OPND);
                }
                // Handle various operand combinations for spills on compile_exits.
                &mut Insn::Store { dest, src } => {
                    let num_bits = dest.rm_num_bits();
                    let src = split_stack_membase(asm, src, SCRATCH0_OPND);
                    let dest = split_stack_membase(asm, dest, SCRATCH1_OPND);

                    let src = match src {
                        Opnd::Reg(_) => src,
                        Opnd::Mem(_) => {
                            asm.mov(SCRATCH0_OPND, src);
                            SCRATCH0_OPND
                        }
                        Opnd::Imm(imm) => {
                            // For 64 bit destinations, 32-bit values will be sign-extended
                            if num_bits == 64 && imm_num_bits(imm) > 32 {
                                asm.mov(SCRATCH0_OPND, src);
                                SCRATCH0_OPND
                            } else if uimm_num_bits(imm as u64) <= num_bits {
                                // If the bit string is short enough for the destination, use the unsigned representation.
                                // Note that 64-bit and negative values are ruled out.
                                Opnd::UImm(imm as u64)
                            } else {
                                src
                            }
                        }
                        Opnd::UImm(imm) => {
                            // For 64 bit destinations, 32-bit values will be sign-extended
                            if num_bits == 64 && imm_num_bits(imm as i64) > 32 {
                                asm.mov(SCRATCH0_OPND, src);
                                SCRATCH0_OPND
                            } else {
                                src.into()
                            }
                        }
                        Opnd::Value(_) => {
                            asm.load_into(SCRATCH0_OPND, src);
                            SCRATCH0_OPND
                        }
                        src @ (Opnd::None | Opnd::VReg { .. }) => panic!("Unexpected source operand during x86_scratch_split: {src:?}"),
                    };
                    asm.store(dest, src);
                }
                &mut Insn::PatchPoint(ref data) => {
                    split_patch_point(asm, &data.target, data.invariant, data.version);
                }
                _ => {
                    asm.push_insn(insn);
                }
            }
        }

        asm_local
    }

    /// Emit platform-specific machine code
    pub fn x86_emit(&mut self, cb: &mut CodeBlock) -> Result<Vec<CodePtr>, CompileError> {
        fn emit_csel(
            cb: &mut CodeBlock,
            truthy: Opnd,
            falsy: Opnd,
            out: Opnd,
            cmov_fn: fn(&mut CodeBlock, X86Opnd, X86Opnd),
            cmov_neg: fn(&mut CodeBlock, X86Opnd, X86Opnd)){

            // Assert that output is a register
            let out_reg = out.unwrap_reg();

            /// Check if a memory operand uses the given register as its base
            fn mem_uses_reg(opnd: &Opnd, reg: Reg) -> bool {
                if let Opnd::Mem(Mem { base: MemBase::Reg(reg_no), .. }) = opnd {
                    *reg_no == reg.reg_no
                } else {
                    false
                }
            }

            /// Check if an operand aliases the given register (either as a
            /// register operand or as a memory base).
            fn aliases_reg(opnd: &Opnd, reg: Reg) -> bool {
                match opnd {
                    Opnd::Reg(r) => r.reg_no == reg.reg_no,
                    Opnd::Mem(Mem { base: MemBase::Reg(reg_no), .. }) => *reg_no == reg.reg_no,
                    _ => false,
                }
            }

            // If the truthy value is a memory operand
            if let Opnd::Mem(_) = truthy {
                // If out aliases truthy, we must load truthy into the scratch
                // register first to avoid clobbering it with the falsy mov.
                if aliases_reg(&truthy, out_reg) {
                    mov(cb, SCRATCH0_OPND.into(), truthy.into());
                    if out != falsy {
                        mov(cb, out.into(), falsy.into());
                    }
                    cmov_fn(cb, out.into(), SCRATCH0_OPND.into());
                } else {
                    if out != falsy {
                        mov(cb, out.into(), falsy.into());
                    }
                    cmov_fn(cb, out.into(), truthy.into());
                }
            } else {
                // If out aliases falsy, we must load falsy into the scratch
                // register first to avoid clobbering it with the truthy mov.
                if aliases_reg(&falsy, out_reg) {
                    mov(cb, SCRATCH0_OPND.into(), falsy.into());
                    if out != truthy {
                        mov(cb, out.into(), truthy.into());
                    }
                    cmov_neg(cb, out.into(), SCRATCH0_OPND.into());
                } else {
                    if out != truthy {
                        mov(cb, out.into(), truthy.into());
                    }
                    cmov_neg(cb, out.into(), falsy.into());
                }
            }
        }

        fn emit_load_gc_value(cb: &mut CodeBlock, gc_offsets: &mut Vec<CodePtr>, dest_reg: X86Opnd, value: VALUE) {
            // Using movabs because mov might write value in 32 bits
            movabs(cb, dest_reg, value.0 as _);
            // The pointer immediate is encoded as the last part of the mov written out
            let ptr_offset = cb.get_write_ptr().sub_bytes(SIZEOF_VALUE);
            gc_offsets.push(ptr_offset);
        }

        /// Fill nops until a jump written at `last_patch_pos` can no longer reach
        /// past the current write position.
        fn emit_pad_after_patch_point(cb: &mut CodeBlock, last_patch_pos: Option<usize>) {
            let Some(last_patch_pos) = last_patch_pos else { return };
            let code_size = cb.get_write_pos().saturating_sub(last_patch_pos);
            if code_size < cb.jmp_ptr_bytes() {
                nop(cb, (cb.jmp_ptr_bytes() - code_size) as u32);
            }
        }

        // List of GC offsets
        let mut gc_offsets: Vec<CodePtr> = Vec::new();

        // Buffered list of PosMarker callbacks to fire if codegen is successful
        let mut pos_markers: Vec<(usize, CodePtr)> = vec![];

        // The write_pos for the last Insn::PatchPoint, if any
        let mut last_patch_pos: Option<usize> = None;

        // For each instruction
        let mut insn_idx: usize = 0;
        assert_eq!(self.basic_blocks.len(), 1, "Assembler should be linearized into a single block before arm64_emit");
        let insns = &self.basic_blocks[0].insns;

        while let Some(insn) = insns.get(insn_idx) {
            match insn {
                Insn::Comment(text) => {
                    cb.add_comment(text);
                },

                // Write the label at the current position
                Insn::Label(target) => {
                    cb.write_label(target.unwrap_label());
                },

                // Report back the current position in the generated code
                Insn::PosMarker(..) => {
                    pos_markers.push((insn_idx, cb.get_write_ptr()));
                },
                Insn::PosMarkerAtBlockEnd(..) => {
                    unreachable!("PosMarkerAtBlockEnd should have been lowered by linearize_instructions");
                },

                Insn::BakeString(text) => {
                    for byte in text.as_bytes() {
                        cb.write_byte(*byte);
                    }

                    // Add a null-terminator byte for safety (in case we pass
                    // this to C code)
                    cb.write_byte(0);
                },

                // Set up RBP as frame pointer work with unwinding
                // (e.g. with Linux `perf record --call-graph fp`)
                // and to allow push and pops in the function.
                &Insn::FrameSetup { preserved, mut slot_count } => {
                    // Bump slot_count for alignment if necessary
                    const { assert!(SIZEOF_VALUE == 8, "alignment logic relies on SIZEOF_VALUE == 8"); }
                    let total_slots = 2 /* rbp and return address*/ + slot_count + preserved.len();
                    if total_slots % 2 == 1 {
                        slot_count += 1;
                    }
                    push(cb, RBP);
                    mov(cb, RBP, RSP);
                    for reg in preserved {
                        push(cb, reg.into());
                    }
                    if slot_count > 0 {
                        sub(cb, RSP, uimm_opnd((slot_count * SIZEOF_VALUE) as u64));
                    }
                }
                &Insn::FrameTeardown { preserved } => {
                    let mut preserved_offset = -8;
                    for reg in preserved {
                        mov(cb, reg.into(), mem_opnd(64, RBP, preserved_offset));
                        preserved_offset -= 8;
                    }
                    mov(cb, RSP, RBP);
                    pop(cb, RBP);
                }

                Insn::Add { left, right, .. } => {
                    add(cb, left.into(), right.into());
                },

                Insn::Sub { left, right, .. } => {
                    sub(cb, left.into(), right.into());
                },

                Insn::Mul { left, right, .. } => {
                    imul(cb, left.into(), right.into());
                },

                Insn::And { left, right, .. } => {
                    and(cb, left.into(), right.into());
                },

                Insn::Or { left, right, .. } => {
                    or(cb, left.into(), right.into());
                },

                Insn::Xor { left, right, .. } => {
                    xor(cb, left.into(), right.into());
                },

                Insn::Not { opnd, .. } => {
                    not(cb, opnd.into());
                },

                Insn::LShift { opnd, shift , ..} => {
                    shl(cb, opnd.into(), shift.into())
                },

                Insn::RShift { opnd, shift , ..} => {
                    sar(cb, opnd.into(), shift.into())
                },

                Insn::URShift { opnd, shift, .. } => {
                    shr(cb, opnd.into(), shift.into())
                },

                // This assumes only load instructions can contain references to GC'd Value operands
                Insn::Load { opnd, out } |
                Insn::LoadInto { dest: out, opnd } => {
                    match opnd {
                        Opnd::Value(val) if val.heap_object_p() => {
                            emit_load_gc_value(cb, &mut gc_offsets, out.into(), *val);
                        }
                        _ => mov(cb, out.into(), opnd.into())
                    }
                },

                Insn::LoadSExt { opnd, out } => {
                    movsx(cb, out.into(), opnd.into());
                },

                Insn::Store { dest, src } |
                Insn::Mov { dest, src } => {
                    mov(cb, dest.into(), src.into());
                },

                // Load effective address
                Insn::Lea { opnd, out } => {
                    lea(cb, out.into(), opnd.into());
                },

                // Load address of jump target
                Insn::LeaJumpTarget { target, out } => {
                    if let Target::Label(label) = target {
                        let out = *out;
                        // Set output to the raw address of the label
                        cb.label_ref(*label, 7, move |cb, src_addr, dst_addr| {
                            let disp = dst_addr - src_addr;
                            lea(cb, out.into(), mem_opnd(8, RIP, disp.try_into().unwrap()));
                            Ok(())
                        });
                    } else {
                        // Set output to the jump target's raw address
                        let target_code = target.unwrap_code_ptr();
                        let target_addr = target_code.raw_addr(cb).as_u64();
                        // Constant encoded length important for patching
                        movabs(cb, out.into(), target_addr);
                    }
                },

                // Push and pop to/from the C stack
                Insn::CPush(opnd) => {
                    push(cb, opnd.into());
                },
                Insn::CPushPair(opnd0, opnd1) => {
                    push(cb, opnd0.into());
                    push(cb, opnd1.into());
                },
                Insn::CPop { out } => {
                    pop(cb, out.into());
                },
                Insn::CPopInto(opnd) => {
                    pop(cb, opnd.into());
                },
                Insn::CPopPairInto(opnd0, opnd1) => {
                    pop(cb, opnd0.into());
                    pop(cb, opnd1.into());
                },

                // C function call
                Insn::CCall { data, .. } => {
                    let fptr = &data.fptr;
                    match fptr {
                        Opnd::UImm(fptr) => {
                            call_ptr(cb, RAX, *fptr as *const u8);
                        }
                        Opnd::Reg(_) => {
                            call(cb, fptr.into());
                        }
                        _ => unreachable!("unsupported ccall fptr: {fptr:?}")
                    }
                },

                Insn::CRet(opnd) => {
                    // TODO: bias allocation towards return register
                    if *opnd != Opnd::Reg(C_RET_REG) {
                        mov(cb, RAX, opnd.into());
                    }

                    ret(cb);
                },

                // Compare
                Insn::Cmp { left, right } => {
                    cmp(cb, left.into(), right.into());
                }

                // Test and set flags
                Insn::Test { left, right } => {
                    test(cb, left.into(), right.into());
                }

                Insn::JmpOpnd(opnd) => {
                    jmp_rm(cb, opnd.into());
                }

                // Conditional jump to a label
                Insn::Jmp(target) => {
                    match *target {
                        Target::CodePtr(code_ptr) => jmp_ptr(cb, code_ptr),
                        Target::Label(label) => jmp_label(cb, label),
                        Target::Block(ref edge) => jmp_label(cb, self.block_label(edge.target)),
                        Target::SideExit(..) => unreachable!("Target::SideExit should have been compiled by compile_exits"),
                    }
                }

                Insn::Je(target) => {
                    match *target {
                        Target::CodePtr(code_ptr) => je_ptr(cb, code_ptr),
                        Target::Label(label) => je_label(cb, label),
                        Target::Block(ref edge) => je_label(cb, self.block_label(edge.target)),
                        Target::SideExit(..) => unreachable!("Target::SideExit should have been compiled by compile_exits"),
                    }
                }

                Insn::Jne(target) => {
                    match *target {
                        Target::CodePtr(code_ptr) => jne_ptr(cb, code_ptr),
                        Target::Label(label) => jne_label(cb, label),
                        Target::Block(ref edge) => jne_label(cb, self.block_label(edge.target)),
                        Target::SideExit(..) => unreachable!("Target::SideExit should have been compiled by compile_exits"),
                    }
                }

                Insn::Jl(target) => {
                    match *target {
                        Target::CodePtr(code_ptr) => jl_ptr(cb, code_ptr),
                        Target::Label(label) => jl_label(cb, label),
                        Target::Block(ref edge) => jl_label(cb, self.block_label(edge.target)),
                        Target::SideExit(..) => unreachable!("Target::SideExit should have been compiled by compile_exits"),
                    }
                },

                Insn::Jg(target) => {
                    match *target {
                        Target::CodePtr(code_ptr) => jg_ptr(cb, code_ptr),
                        Target::Label(label) => jg_label(cb, label),
                        Target::Block(ref edge) => jg_label(cb, self.block_label(edge.target)),
                        Target::SideExit(..) => unreachable!("Target::SideExit should have been compiled by compile_exits"),
                    }
                },

                Insn::Jge(target) => {
                    match *target {
                        Target::CodePtr(code_ptr) => jge_ptr(cb, code_ptr),
                        Target::Label(label) => jge_label(cb, label),
                        Target::Block(ref edge) => jge_label(cb, self.block_label(edge.target)),
                        Target::SideExit(..) => unreachable!("Target::SideExit should have been compiled by compile_exits"),
                    }
                },

                Insn::Jbe(target) => {
                    match *target {
                        Target::CodePtr(code_ptr) => jbe_ptr(cb, code_ptr),
                        Target::Label(label) => jbe_label(cb, label),
                        Target::Block(ref edge) => jbe_label(cb, self.block_label(edge.target)),
                        Target::SideExit(..) => unreachable!("Target::SideExit should have been compiled by compile_exits"),
                    }
                },

                Insn::Jb(target) => {
                    match *target {
                        Target::CodePtr(code_ptr) => jb_ptr(cb, code_ptr),
                        Target::Label(label) => jb_label(cb, label),
                        Target::Block(ref edge) => jb_label(cb, self.block_label(edge.target)),
                        Target::SideExit(..) => unreachable!("Target::SideExit should have been compiled by compile_exits"),
                    }
                },

                Insn::Jz(target) => {
                    match *target {
                        Target::CodePtr(code_ptr) => jz_ptr(cb, code_ptr),
                        Target::Label(label) => jz_label(cb, label),
                        Target::Block(ref edge) => jz_label(cb, self.block_label(edge.target)),
                        Target::SideExit(..) => unreachable!("Target::SideExit should have been compiled by compile_exits"),
                    }
                }

                Insn::Jnz(target) => {
                    match *target {
                        Target::CodePtr(code_ptr) => jnz_ptr(cb, code_ptr),
                        Target::Label(label) => jnz_label(cb, label),
                        Target::Block(ref edge) => jnz_label(cb, self.block_label(edge.target)),
                        Target::SideExit(..) => unreachable!("Target::SideExit should have been compiled by compile_exits"),
                    }
                }

                Insn::Jo(target) |
                Insn::JoMul(target) => {
                    match *target {
                        Target::CodePtr(code_ptr) => jo_ptr(cb, code_ptr),
                        Target::Label(label) => jo_label(cb, label),
                        Target::Block(ref edge) => jo_label(cb, self.block_label(edge.target)),
                        Target::SideExit(..) => unreachable!("Target::SideExit should have been compiled by compile_exits"),
                    }
                }

                Insn::Joz(..) | Insn::Jonz(..) => unreachable!("Joz/Jonz should be unused for now"),

                Insn::PatchPoint(..) => unreachable!("PatchPoint should have been lowered to PatchPointPad in x86_scratch_split"),
                Insn::PatchPointPad => {
                    emit_pad_after_patch_point(cb, last_patch_pos);
                    // This position is itself where a jump gets written on invalidation, so it
                    // becomes what following code has to keep its distance from.
                    last_patch_pos = Some(cb.get_write_pos());
                },
                Insn::BoundaryPad => {
                    // A boundary is never patched, so it doesn't become a position to keep away
                    // from. The last patch point stays that, and the pad just gave it its room.
                    emit_pad_after_patch_point(cb, last_patch_pos);
                },

                // Atomically increment a counter at a given memory location
                Insn::IncrCounter { mem, value } => {
                    assert!(matches!(mem, Opnd::Mem(_)));
                    assert!(matches!(value, Opnd::UImm(_) | Opnd::Imm(_) ) );
                    write_lock_prefix(cb);
                    add(cb, mem.into(), value.into());
                },

                Insn::Breakpoint => int3(cb),
                Insn::Abort => ud2(cb),

                Insn::CSelZ { truthy, falsy, out } => {
                    emit_csel(cb, *truthy, *falsy, *out, cmovz, cmovnz);
                },
                Insn::CSelNZ { truthy, falsy, out } => {
                    emit_csel(cb, *truthy, *falsy, *out, cmovnz, cmovz);
                },
                Insn::CSelE { truthy, falsy, out } => {
                    emit_csel(cb, *truthy, *falsy, *out, cmove, cmovne);
                },
                Insn::CSelNE { truthy, falsy, out } => {
                    emit_csel(cb, *truthy, *falsy, *out, cmovne, cmove);
                },
                Insn::CSelL { truthy, falsy, out } => {
                    emit_csel(cb, *truthy, *falsy, *out, cmovl, cmovge);
                },
                Insn::CSelLE { truthy, falsy, out } => {
                    emit_csel(cb, *truthy, *falsy, *out, cmovle, cmovg);
                },
                Insn::CSelG { truthy, falsy, out } => {
                    emit_csel(cb, *truthy, *falsy, *out, cmovg, cmovle);
                },
                Insn::CSelGE { truthy, falsy, out } => {
                    emit_csel(cb, *truthy, *falsy, *out, cmovge, cmovl);
                }
            };

            insn_idx += 1;
        }

        // Error if we couldn't write out everything
        if cb.has_dropped_bytes() {
            Err(CompileError::OutOfMemory)
        } else {
            // No bytes dropped, so the pos markers point to valid code
            for (insn_idx, pos) in pos_markers {
                if let Insn::PosMarker(callback) = insns.get(insn_idx).unwrap() {
                    callback(pos, cb);
                } else {
                    panic!("non-PosMarker in pos_markers insn_idx={insn_idx} {self:?}");
                }
            }

            Ok(gc_offsets)
        }
    }

    /// Optimize and compile the stored instructions
    pub fn compile_with_regs(self, cb: &mut CodeBlock, regs: Vec<Reg>) -> Result<(CodePtr, Vec<CodePtr>), CompileError> {
        // The backend is allowed to use scratch registers only if it has not accepted them so far.
        let use_scratch_regs = !self.accept_scratch_reg;
        asm_dump!(self, init);

        let mut asm = trace_compile_phase("split", || self.x86_split());

        asm_dump!(asm, split);

        trace_compile_phase("regalloc", || {
            trace_compile_phase("number_instructions", || asm.number_instructions(0));

            let live_in = trace_compile_phase("analyze_liveness", || asm.analyze_liveness());
            let intervals = trace_compile_phase("build_intervals", || asm.build_intervals(live_in));

            // Dump live intervals if requested
            if let Some(crate::options::Options { dump_lir: Some(dump_lirs), .. }) = unsafe { crate::options::OPTIONS.as_ref() } {
                if dump_lirs.contains(&crate::options::DumpLIR::live_intervals) {
                    println!("LIR live_intervals:\n{}", crate::backend::lir::debug_intervals(&asm, &intervals));
                }
            }

            let preferred_registers = trace_compile_phase("preferred_registers", || asm.preferred_register_assignments(&intervals));
            let (assignments, num_stack_slots) = trace_compile_phase("linear_scan", || asm.linear_scan(intervals.clone(), regs.len(), &preferred_registers));

            asm.stack_state.num_spill_slots = num_stack_slots;
            asm.stack_state.num_side_exit_stack_map_slots = asm.side_exit_stack_map_slots(&assignments);
            let stack_slot_count = asm.stack_state.stack_slot_count();
            if stack_slot_count > Self::MAX_FRAME_STACK_SLOTS {
                return Err(CompileError::NativeStackTooLarge);
            }

            // Dump vreg-to-physical-register mapping if requested
            if let Some(crate::options::Options { dump_lir: Some(dump_lirs), .. }) = unsafe { crate::options::OPTIONS.as_ref() } {
                if dump_lirs.contains(&crate::options::DumpLIR::alloc_regs) {
                    println!("LIR live_intervals:\n{}", crate::backend::lir::debug_intervals(&asm, &intervals));

                    println!("VReg assignments:");
                    for (i, alloc) in assignments.iter().enumerate() {
                        if let Some(alloc) = alloc {
                            let range = &intervals[i].range;
                            let alloc_str = match alloc {
                                Allocation::Reg(n) => format!("{}", regs[*n]),
                                Allocation::Fixed(reg) => format!("{}", reg),
                                Allocation::Stack(n) => format!("Stack[{}]", n),
                            };
                            println!("  v{} => {} (range: {:?}..{:?})", i, alloc_str, range.start, range.end);
                        }
                    }
                }
            }

            // Update FrameSetup slot_count now that StackState knows the
            // register allocator spill and side-exit capture counts.
            trace_compile_phase("count_stack_slots", || {
                for block in asm.basic_blocks.iter_mut() {
                    for insn in block.insns.iter_mut() {
                        if let Insn::FrameSetup { slot_count, .. } = insn {
                            *slot_count = stack_slot_count;
                        }
                    }
                }
            });

            trace_compile_phase("resolve_ssa", || {
                asm.handle_caller_saved_regs(&intervals, &assignments, &C_ARG_REGREGS);
                asm.resolve_ssa(&intervals, &assignments);
            });

            Ok(())
        })?;
        asm_dump!(asm, alloc_regs);

        // We are moved out of SSA after resolve_ssa

        // We put compile_exits after alloc_regs to avoid extending live ranges for VRegs spilled on side exits.
        // Exit code is compiled into a separate list of instructions that we append
        // to the last reachable block before scratch_split, so it gets linearized and split.
        trace_compile_phase("compile_exits", || {
            let exit_insns = asm.compile_exits();

            // Append exit instructions to the last reachable block so they are
            // included in linearize_instructions and processed by scratch_split.
            if let Some(&last_block) = asm.block_order().last() {
                for insn in exit_insns {
                    asm.basic_blocks[last_block.0].insns.push(insn);
                    asm.basic_blocks[last_block.0].insn_ids.push(None);
                }
            }
        });
        asm_dump!(asm, compile_exits);

        if use_scratch_regs {
            asm = trace_compile_phase("scratch_split", || asm.x86_scratch_split());
            asm_dump!(asm, scratch_split);
        } else {
            // For trampolines that use scratch registers, resolve ParallelMov without scratch_reg.
            asm = trace_compile_phase("resolve_parallel_mov", || asm.resolve_parallel_mov_pass());
            asm_dump!(asm, resolve_parallel_mov);
        }

        trace_compile_phase("emit", || {
            // Create label instances in the code block
            for (idx, name) in asm.label_names.iter().enumerate() {
                let label = cb.new_label(name.to_string());
                assert_eq!(label, Label(idx));
            }

            let start_ptr = cb.get_write_ptr();
            let gc_offsets = asm.x86_emit(cb).inspect_err(|_| cb.clear_labels())?;
            assert!(!cb.has_dropped_bytes(), "emit should not drop bytes without error");

            cb.link_labels().or(Err(CompileError::LabelLinkingFailure))?;
            Ok((start_ptr, gc_offsets))
        })
    }
}

#[cfg(test)]
mod tests {
    use insta::assert_snapshot;
    use crate::assert_disasm_snapshot;
    use crate::options::rb_zjit_prepare_options;
    use super::*;

    const BOLD_BEGIN: &str = "\x1b[1m";
    const BOLD_END: &str = "\x1b[22m";

    fn setup_asm() -> (Assembler, CodeBlock) {
        rb_zjit_prepare_options(); // for get_option! on asm.compile
        let mut asm = Assembler::new();
        asm.new_block_without_id("test");
        (asm, CodeBlock::new_dummy())
    }

    fn stack_mem(stack_idx: StackIdx) -> Opnd {
        Opnd::Mem(Mem {
            base: MemBase::Stack { stack_idx, num_bits: 64 },
            disp: 0,
            num_bits: 64,
        })
    }

    fn stack_indirect_mem(stack_idx: StackIdx) -> Opnd {
        Opnd::Mem(Mem {
            base: MemBase::StackIndirect { stack_idx },
            disp: 0,
            num_bits: 64,
        })
    }

    #[derive(Clone, Copy, Debug)]
    enum BinOpKind {
        Add,
        Sub,
        And,
        Or,
        Xor,
    }

    fn split_binop(kind: BinOpKind, left: Opnd, right: Opnd, out: Opnd) -> Assembler {
        let (mut asm, _) = setup_asm();
        match kind {
            BinOpKind::Add => asm.push_insn(Insn::Add { left, right, out }),
            BinOpKind::Sub => asm.push_insn(Insn::Sub { left, right, out }),
            BinOpKind::And => asm.push_insn(Insn::And { left, right, out }),
            BinOpKind::Or => asm.push_insn(Insn::Or { left, right, out }),
            BinOpKind::Xor => asm.push_insn(Insn::Xor { left, right, out }),
        }
        asm.x86_scratch_split()
    }

    fn binop_mnemonic(kind: BinOpKind) -> &'static str {
        match kind {
            BinOpKind::Add => "add",
            BinOpKind::Sub => "sub",
            BinOpKind::And => "and",
            BinOpKind::Or => "or",
            BinOpKind::Xor => "xor",
        }
    }

    fn split_binop_disasm_lines(kind: BinOpKind, left: Opnd, right: Opnd, out: Opnd) -> Vec<String> {
        let mut asm = split_binop(kind, left, right, out);
        let mut cb = CodeBlock::new_dummy();
        for name in &asm.label_names {
            cb.new_label(name.to_string());
        }
        assert!(asm.x86_emit(&mut cb).is_ok(), "{kind:?}: x86_emit failed");

        cb.disasm()
            .lines()
            .map(str::trim)
            .filter(|line| !line.is_empty())
            .map(|line| {
                line.split_once(": ")
                    .map(|(_, rest)| rest.to_string())
                    .unwrap_or_else(|| line.to_string())
            })
            .collect()
    }

    fn assert_split_binop_case(kind: BinOpKind, left: Opnd, right: Opnd, out: Opnd, case: &str) {
        fn reg_names(reg: Reg) -> (&'static str, &'static str) {
            const RAX_NO: u8 = RAX_REG.reg_no;
            const RDI_NO: u8 = RDI_REG.reg_no;
            const R13_NO: u8 = R13_REG.reg_no;
            match reg.reg_no {
                RAX_NO => ("rax", "eax"),
                RDI_NO => ("rdi", "edi"),
                R13_NO => ("r13", "r13d"),
                _ => panic!("unexpected register in test helper: {reg:?}"),
            }
        }

        let lines = split_binop_disasm_lines(kind, left, right, out);
        let mnemonic = binop_mnemonic(kind);
        let matching: Vec<(usize, &String)> = lines.iter()
            .enumerate()
            .filter(|(_, line)| line.starts_with(&format!("{mnemonic} ")))
            .collect();

        assert_eq!(matching.len(), 1, "{kind:?} {case}: expected exactly one `{mnemonic}` in disasm, got {lines:?}");
        assert!(
            !matching[0].1.contains("], qword ptr ["),
            "{kind:?} {case}: emitted mem/mem `{mnemonic}` in disasm {lines:?}"
        );

        if let (Opnd::Reg(out_reg), Opnd::Imm(_) | Opnd::UImm(_)) = (out, left) {
            let (out64, out32) = reg_names(out_reg);
            let prelude = &lines[..matching[0].0];
            assert!(
                prelude.iter().any(|line|
                    line.starts_with(&format!("mov {out64}, ")) ||
                    line.starts_with(&format!("mov {out32}, ")) ||
                    line.starts_with(&format!("movabs {out64}, "))
                ),
                "{kind:?} {case}: expected left immediate to be materialized into output register before `{mnemonic}`, got {lines:?}"
            );
        }

        if matches!(right, Opnd::Imm(value) if imm_num_bits(value) > 32)
            || matches!(right, Opnd::UImm(value) if imm_num_bits(value as i64) > 32)
        {
            assert!(lines.iter().any(|line| line.starts_with("movabs ")), "{kind:?} {case}: expected movabs materialization for 64-bit immediate, got {lines:?}");
        }
    }

    #[test]
    fn test_fallthrough_to_a_patchpoint() {
        // At one point, this generated unnecessary nop padding
        use crate::cruby::test_utils::{compile_to_iseq, with_rubyvm};
        use crate::hir::Invariant;
        use crate::payload::IseqVersion;

        let version = IseqVersion::new(compile_to_iseq("nil"));

        // The PosMarker that split_patch_point() installs registers the patch point
        // with ZJITState's invariant table while emitting, so the VM has to be booted.
        let cb = with_rubyvm(|| {
            crate::options::rb_zjit_prepare_options(); // Allow `get_option!` in Assembler
            let mut asm = Assembler::new();
            let mut cb = CodeBlock::new_dummy();

            let bb0 = asm.new_block(crate::hir::BlockId(0), true, 0);
            let bb1 = asm.new_block(crate::hir::BlockId(1), false, 1);

            // The patch point's target only has to resolve to some address for the
            // PosMarker that records it, so bb0 stands in for the side exit code.
            let side_exit = asm.new_label("side_exit");

            // bb0 falls through to bb1
            asm.set_current_block(bb0);
            let label_bb0 = asm.new_label("bb0");
            asm.write_label(label_bb0);
            asm.write_label(side_exit.clone());
            asm.mov(C_ARG_OPNDS[0], Opnd::UImm(1));
            asm.push_insn(Insn::Jmp(Target::Block(Box::new(BranchEdge { target: bb1, args: vec![] }))));

            asm.set_current_block(bb1);
            let label_bb1 = asm.new_label("bb1");
            asm.write_label(label_bb1);
            asm.patch_point(side_exit.clone(), Invariant::SingleRactorMode, version);
            asm.cret(C_ARG_OPNDS[0]);

            asm.compile_with_num_regs(&mut cb, 0);
            cb
        });

        assert_disasm_snapshot!(cb.disasm(), @"
        0x0: mov edi, 1
        0x5: mov rax, rdi
        0x8: ret
        0x9: nop
        ");
        assert_snapshot!(cb.hexdump(), @"bf010000004889f8c390");
    }

    #[test]
    fn test_lir_string() {
        use crate::hir::SideExitReason;

        let mut asm = Assembler::new();
        asm.new_block_without_id("test");
        asm.stack_state.stack_base_idx = 1;

        let label = asm.new_label("bb0");
        asm.write_label(label.clone());
        asm.push_insn(Insn::Comment("bb0(): foo@/tmp/a.rb:1".into()));
        asm.frame_setup(JIT_PRESERVED_REGS);

        let val64 = asm.add(CFP, Opnd::UImm(64));
        asm.store(Opnd::mem(64, SP, 0x10), val64);
        let side_exit = Target::SideExit(Box::new(SideExitTarget { reason: SideExitReason::Interrupt, exit: SideExit { pc: 0.into(), iseq: std::ptr::null(), stack: vec![], locals: vec![], stack_map: None, recompile: None } }));
        asm.push_insn(Insn::Joz(val64, side_exit));
        asm.mov(C_ARG_OPNDS[0], C_RET_OPND.with_num_bits(32));
        asm.mov(C_ARG_OPNDS[1], Opnd::mem(64, SP, -8));

        let val32 = asm.sub(Opnd::Value(Qtrue), Opnd::Imm(1));
        asm.store(Opnd::mem(64, EC, 0x10).with_num_bits(32), val32.with_num_bits(32));
        asm.push_insn(Insn::Je(label));
        asm.cret(val64);

        asm.frame_teardown(JIT_PRESERVED_REGS);
        assert_disasm_snapshot!(lir_string(&mut asm), @"
        test():
        bb0():
          # bb0(): foo@/tmp/a.rb:1
          FrameSetup 1, r13, rbx, r12
          v0 = Add r13, 0x40
          Store [rbx + 0x10], v0
          Joz Exit(Interrupt), v0
          Mov rdi, eax
          Mov rsi, [rbx - 8]
          v1 = Sub Value(0x14), Imm(1)
          Store Mem32[r12 + 0x10], VReg32(v1)
          Je bb0
          CRet v0
          FrameTeardown r13, rbx, r12
        ");
    }

    #[test]
    #[ignore]
    fn test_emit_add_lt_32_bits() {
        let (mut asm, mut cb) = setup_asm();

        let _ = asm.add(Opnd::Reg(RAX_REG), Opnd::UImm(0xFF));
        asm.compile_with_num_regs(&mut cb, 1);

        assert_disasm_snapshot!(cb.disasm(), @"
        0x0: mov rax, rax
        0x3: add rax, 0xff
        ");
        assert_snapshot!(cb.hexdump(), @"4889c04881c0ff000000");
    }

    #[test]
    #[ignore]
    fn test_emit_add_gt_32_bits() {
        let (mut asm, mut cb) = setup_asm();

        let _ = asm.add(Opnd::Reg(RAX_REG), Opnd::UImm(0xFFFF_FFFF_FFFF));
        asm.compile_with_num_regs(&mut cb, 1);

        assert_disasm_snapshot!(cb.disasm(), @"
        0x0: mov rax, rax
        0x3: movabs r11, 0xffffffffffff
        0xd: add rax, r11
        ");
        assert_snapshot!(cb.hexdump(), @"4889c049bbffffffffffff00004c01d8");
    }

    #[test]
    #[ignore]
    fn test_emit_and_lt_32_bits() {
        let (mut asm, mut cb) = setup_asm();

        let _ = asm.and(Opnd::Reg(RAX_REG), Opnd::UImm(0xFF));
        asm.compile_with_num_regs(&mut cb, 1);

        assert_disasm_snapshot!(cb.disasm(), @"
        0x0: mov rax, rax
        0x3: and rax, 0xff
        ");
        assert_snapshot!(cb.hexdump(), @"4889c04881e0ff000000");
    }

    #[test]
    #[ignore]
    fn test_emit_and_gt_32_bits() {
        let (mut asm, mut cb) = setup_asm();

        let _ = asm.and(Opnd::Reg(RAX_REG), Opnd::UImm(0xFFFF_FFFF_FFFF));
        asm.compile_with_num_regs(&mut cb, 1);

        assert_disasm_snapshot!(cb.disasm(), @"
        0x0: mov rax, rax
        0x3: movabs r11, 0xffffffffffff
        0xd: and rax, r11
        ");
        assert_snapshot!(cb.hexdump(), @"4889c049bbffffffffffff00004c21d8");
    }

    #[test]
    fn test_emit_cmp_lt_32_bits() {
        let (mut asm, mut cb) = setup_asm();

        asm.cmp(Opnd::Reg(RAX_REG), Opnd::UImm(0xFF));
        asm.compile_with_num_regs(&mut cb, 0);

        assert_disasm_snapshot!(cb.disasm(), @"  0x0: cmp rax, 0xff");
        assert_snapshot!(cb.hexdump(), @"4881f8ff000000");
    }

    #[test]
    fn test_emit_cmp_gt_32_bits() {
        let (mut asm, mut cb) = setup_asm();

        asm.cmp(Opnd::Reg(RAX_REG), Opnd::UImm(0xFFFF_FFFF_FFFF));
        asm.compile_with_num_regs(&mut cb, 0);

        assert_disasm_snapshot!(cb.disasm(), @"
        0x0: movabs r11, 0xffffffffffff
        0xa: cmp rax, r11
        ");
        assert_snapshot!(cb.hexdump(), @"49bbffffffffffff00004c39d8");
    }

    #[test]
    fn test_emit_cmp_64_bits() {
        let (mut asm, mut cb) = setup_asm();

        asm.cmp(Opnd::Reg(RAX_REG), Opnd::UImm(0xFFFF_FFFF_FFFF_FFFF));
        asm.compile_with_num_regs(&mut cb, 0);

        assert_disasm_snapshot!(cb.disasm(), @"  0x0: cmp rax, -1");
        assert_snapshot!(cb.hexdump(), @"4883f8ff");
    }

    #[test]
    fn test_emit_cmp_mem_16_bits_with_imm_16() {
        let (mut asm, mut cb) = setup_asm();

        let shape_opnd = Opnd::mem(16, Opnd::Reg(RAX_REG), 6);

        asm.cmp(shape_opnd, Opnd::UImm(0xF000));
        asm.compile_with_num_regs(&mut cb, 0);

        assert_disasm_snapshot!(cb.disasm(), @"  0x0: cmp word ptr [rax + 6], 0xf000");
        assert_snapshot!(cb.hexdump(), @"6681780600f0");
    }

    #[test]
    fn test_emit_cmp_mem_32_bits_with_imm_32() {
        let (mut asm, mut cb) = setup_asm();

        let shape_opnd = Opnd::mem(32, Opnd::Reg(RAX_REG), 4);

        asm.cmp(shape_opnd, Opnd::UImm(0xF000_0000));
        asm.compile_with_num_regs(&mut cb, 0);

        assert_disasm_snapshot!(cb.disasm(), @"  0x0: cmp dword ptr [rax + 4], 0xf0000000");
        assert_snapshot!(cb.hexdump(), @"817804000000f0");
    }

    #[test]
    #[ignore]
    fn test_emit_or_lt_32_bits() {
        let (mut asm, mut cb) = setup_asm();

        let _ = asm.or(Opnd::Reg(RAX_REG), Opnd::UImm(0xFF));
        asm.compile_with_num_regs(&mut cb, 1);

        assert_disasm_snapshot!(cb.disasm(), @"
        0x0: mov rax, rax
        0x3: or rax, 0xff
        ");
        assert_snapshot!(cb.hexdump(), @"4889c04881c8ff000000");
    }

    #[test]
    #[ignore]
    fn test_emit_or_gt_32_bits() {
        let (mut asm, mut cb) = setup_asm();

        let _ = asm.or(Opnd::Reg(RAX_REG), Opnd::UImm(0xFFFF_FFFF_FFFF));
        asm.compile_with_num_regs(&mut cb, 1);

        assert_disasm_snapshot!(cb.disasm(), @"
        0x0: mov rax, rax
        0x3: movabs r11, 0xffffffffffff
        0xd: or rax, r11
        ");
        assert_snapshot!(cb.hexdump(), @"4889c049bbffffffffffff00004c09d8");
    }

    #[test]
    #[ignore]
    fn test_emit_sub_lt_32_bits() {
        let (mut asm, mut cb) = setup_asm();

        let _ = asm.sub(Opnd::Reg(RAX_REG), Opnd::UImm(0xFF));
        asm.compile_with_num_regs(&mut cb, 1);

        assert_disasm_snapshot!(cb.disasm(), @"
        0x0: mov rax, rax
        0x3: sub rax, 0xff
        ");
        assert_snapshot!(cb.hexdump(), @"4889c04881e8ff000000");
    }

    #[test]
    #[ignore]
    fn test_emit_sub_gt_32_bits() {
        let (mut asm, mut cb) = setup_asm();

        let _ = asm.sub(Opnd::Reg(RAX_REG), Opnd::UImm(0xFFFF_FFFF_FFFF));
        asm.compile_with_num_regs(&mut cb, 1);

        assert_disasm_snapshot!(cb.disasm(), @"
        0x0: mov rax, rax
        0x3: movabs r11, 0xffffffffffff
        0xd: sub rax, r11
        ");
        assert_snapshot!(cb.hexdump(), @"4889c049bbffffffffffff00004c29d8");
    }

    #[test]
    fn test_emit_test_lt_32_bits() {
        let (mut asm, mut cb) = setup_asm();

        asm.test(Opnd::Reg(RAX_REG), Opnd::UImm(0xFF));
        asm.compile_with_num_regs(&mut cb, 0);

        assert_disasm_snapshot!(cb.disasm(), @"  0x0: test rax, 0xff");
        assert_snapshot!(cb.hexdump(), @"48f7c0ff000000");
    }

    #[test]
    fn test_emit_test_gt_32_bits() {
        let (mut asm, mut cb) = setup_asm();

        asm.test(Opnd::Reg(RAX_REG), Opnd::UImm(0xFFFF_FFFF_FFFF));
        asm.compile_with_num_regs(&mut cb, 0);

        assert_disasm_snapshot!(cb.disasm(), @"
        0x0: movabs r11, 0xffffffffffff
        0xa: test rax, r11
        ");
        assert_snapshot!(cb.hexdump(), @"49bbffffffffffff00004c85d8");
    }

    #[test]
    #[ignore]
    fn test_emit_xor_lt_32_bits() {
        let (mut asm, mut cb) = setup_asm();

        let _ = asm.xor(Opnd::Reg(RAX_REG), Opnd::UImm(0xFF));
        asm.compile_with_num_regs(&mut cb, 1);

        assert_disasm_snapshot!(cb.disasm(), @"
        0x0: mov rax, rax
        0x3: xor rax, 0xff
        ");
        assert_snapshot!(cb.hexdump(), @"4889c04881f0ff000000");
    }

    #[test]
    #[ignore]
    fn test_emit_xor_gt_32_bits() {
        let (mut asm, mut cb) = setup_asm();

        let _ = asm.xor(Opnd::Reg(RAX_REG), Opnd::UImm(0xFFFF_FFFF_FFFF));
        asm.compile_with_num_regs(&mut cb, 1);

        assert_disasm_snapshot!(cb.disasm(), @"
        0x0: mov rax, rax
        0x3: movabs r11, 0xffffffffffff
        0xd: xor rax, r11
        ");
        assert_snapshot!(cb.hexdump(), @"4889c049bbffffffffffff00004c31d8");
    }

    #[test]
    fn test_merge_lea_reg() {
        let (mut asm, mut cb) = setup_asm();

        let sp = asm.lea(Opnd::mem(64, SP, 8));
        asm.mov(SP, sp); // should be merged to lea
        asm.compile_with_num_regs(&mut cb, 1);

        assert_disasm_snapshot!(cb.disasm(), @"  0x0: lea rbx, [rbx + 8]");
        assert_snapshot!(cb.hexdump(), @"488d5b08");
    }

    #[test]
    #[ignore]
    fn test_merge_lea_mem() {
        let (mut asm, mut cb) = setup_asm();

        let sp = asm.lea(Opnd::mem(64, SP, 8));
        asm.mov(Opnd::mem(64, SP, 0), sp); // should NOT be merged to lea
        asm.compile_with_num_regs(&mut cb, 1);

        assert_disasm_snapshot!(cb.disasm(), @"
        0x0: movabs r11, 0xffffffffffff
        0xa: cmp rax, r11
        ");
        assert_snapshot!(cb.hexdump(), @"49bbffffffffffff00004c39d8");
    }

    #[test]
    #[ignore]
    fn test_replace_cmp_0() {
        let (mut asm, mut cb) = setup_asm();

        let val = asm.load(Opnd::mem(64, SP, 8));
        asm.cmp(val, 0.into());
        let result = asm.csel_e(Qtrue.into(), Qfalse.into());
        asm.mov(Opnd::Reg(RAX_REG), result);
        asm.compile_with_num_regs(&mut cb, 2);

        assert_disasm_snapshot!(cb.disasm(), @"
        0x0: mov rax, qword ptr [rbx + 8]
        0x4: test rax, rax
        0x7: mov eax, 0x14
        0xc: mov ecx, 0
        0x11: cmovne rax, rcx
        0x15: mov rax, rax
        ");
        assert_snapshot!(cb.hexdump(), @"488b43084885c0b814000000b900000000480f45c14889c0");
    }

    #[test]
    fn test_merge_add_mov() {
        let (mut asm, mut cb) = setup_asm();

        let sp = asm.add(CFP, Opnd::UImm(0x40));
        asm.mov(CFP, sp); // should be merged to add
        asm.compile_with_num_regs(&mut cb, 1);

        assert_disasm_snapshot!(cb.disasm(), @"  0x0: add r13, 0x40");
        assert_snapshot!(cb.hexdump(), @"4983c540");
    }

    #[test]
    fn test_add_into() {
        let (mut asm, mut cb) = setup_asm();

        asm.add_into(CFP, Opnd::UImm(0x40));
        asm.compile_with_num_regs(&mut cb, 1);

        assert_disasm_snapshot!(cb.disasm(), @"  0x0: add r13, 0x40");
        assert_snapshot!(cb.hexdump(), @"4983c540");
    }

    #[test]
    fn test_merge_sub_mov() {
        let (mut asm, mut cb) = setup_asm();

        let sp = asm.sub(CFP, Opnd::UImm(0x40));
        asm.mov(CFP, sp); // should be merged to add
        asm.compile_with_num_regs(&mut cb, 1);

        assert_disasm_snapshot!(cb.disasm(), @"  0x0: sub r13, 0x40");
        assert_snapshot!(cb.hexdump(), @"4983ed40");
    }

    #[test]
    fn test_sub_into() {
        let (mut asm, mut cb) = setup_asm();

        asm.sub_into(CFP, Opnd::UImm(0x40));
        asm.compile_with_num_regs(&mut cb, 1);

        assert_disasm_snapshot!(cb.disasm(), @"  0x0: sub r13, 0x40");
        assert_snapshot!(cb.hexdump(), @"4983ed40");
    }

    #[test]
    fn test_merge_and_mov() {
        let (mut asm, mut cb) = setup_asm();

        let sp = asm.and(CFP, Opnd::UImm(0x40));
        asm.mov(CFP, sp); // should be merged to add
        asm.compile_with_num_regs(&mut cb, 1);

        assert_disasm_snapshot!(cb.disasm(), @"  0x0: and r13, 0x40");
        assert_snapshot!(cb.hexdump(), @"4983e540");
    }

    #[test]
    fn test_merge_or_mov() {
        let (mut asm, mut cb) = setup_asm();

        let sp = asm.or(CFP, Opnd::UImm(0x40));
        asm.mov(CFP, sp); // should be merged to add
        asm.compile_with_num_regs(&mut cb, 1);

        assert_disasm_snapshot!(cb.disasm(), @"  0x0: or r13, 0x40");
        assert_snapshot!(cb.hexdump(), @"4983cd40");
    }

    #[test]
    fn test_merge_xor_mov() {
        let (mut asm, mut cb) = setup_asm();

        let sp = asm.xor(CFP, Opnd::UImm(0x40));
        asm.mov(CFP, sp); // should be merged to add
        asm.compile_with_num_regs(&mut cb, 1);

        assert_disasm_snapshot!(cb.disasm(), @"  0x0: xor r13, 0x40");
        assert_snapshot!(cb.hexdump(), @"4983f540");
    }

    #[test]
    fn test_ccall_resolve_parallel_moves_no_cycle() {
        crate::options::rb_zjit_prepare_options();
        let (mut asm, mut cb) = setup_asm();

        asm.ccall(0 as _, vec![
            C_ARG_OPNDS[0], // mov rdi, rdi (optimized away)
            C_ARG_OPNDS[1], // mov rsi, rsi (optimized away)
        ]);
        asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len());

        assert_disasm_snapshot!(cb.disasm(), @"
        0x0: mov eax, 0
        0x5: call rax
        ");
        assert_snapshot!(cb.hexdump(), @"b800000000ffd0");
    }

    #[test]
    fn test_ccall_resolve_parallel_moves_single_cycle() {
        crate::options::rb_zjit_prepare_options();
        let (mut asm, mut cb) = setup_asm();

        // rdi and rsi form a cycle
        asm.ccall(0 as _, vec![
            C_ARG_OPNDS[1], // mov rdi, rsi
            C_ARG_OPNDS[0], // mov rsi, rdi
            C_ARG_OPNDS[2], // mov rdx, rdx (optimized away)
        ]);
        asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len());

        assert_disasm_snapshot!(cb.disasm(), @"
        0x0: mov r11, rsi
        0x3: mov rsi, rdi
        0x6: mov rdi, r11
        0x9: mov eax, 0
        0xe: call rax
        ");
        assert_snapshot!(cb.hexdump(), @"4989f34889fe4c89dfb800000000ffd0");
    }

    #[test]
    fn test_ccall_resolve_parallel_moves_two_cycles() {
        crate::options::rb_zjit_prepare_options();
        let (mut asm, mut cb) = setup_asm();

        // rdi and rsi form a cycle, and rdx and rcx form another cycle
        asm.ccall(0 as _, vec![
            C_ARG_OPNDS[1], // mov rdi, rsi
            C_ARG_OPNDS[0], // mov rsi, rdi
            C_ARG_OPNDS[3], // mov rdx, rcx
            C_ARG_OPNDS[2], // mov rcx, rdx
        ]);
        asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len());

        assert_disasm_snapshot!(cb.disasm(), @"
        0x0: mov r11, rcx
        0x3: mov rcx, rdx
        0x6: mov rdx, r11
        0x9: mov r11, rsi
        0xc: mov rsi, rdi
        0xf: mov rdi, r11
        0x12: mov eax, 0
        0x17: call rax
        ");
        assert_snapshot!(cb.hexdump(), @"4989cb4889d14c89da4989f34889fe4c89dfb800000000ffd0");
    }

    #[test]
    fn test_ccall_resolve_parallel_moves_large_cycle() {
        crate::options::rb_zjit_prepare_options();
        let (mut asm, mut cb) = setup_asm();

        // rdi, rsi, and rdx form a cycle
        asm.ccall(0 as _, vec![
            C_ARG_OPNDS[1], // mov rdi, rsi
            C_ARG_OPNDS[2], // mov rsi, rdx
            C_ARG_OPNDS[0], // mov rdx, rdi
        ]);
        asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len());

        assert_disasm_snapshot!(cb.disasm(), @"
        0x0: mov r11, rdx
        0x3: mov rdx, rdi
        0x6: mov rdi, rsi
        0x9: mov rsi, r11
        0xc: mov eax, 0
        0x11: call rax
        ");
        assert_snapshot!(cb.hexdump(), @"4989d34889fa4889f74c89deb800000000ffd0");
    }

    #[test]
    #[ignore]
    fn test_ccall_resolve_parallel_moves_with_insn_out() {
        let (mut asm, mut cb) = setup_asm();

        let rax = asm.load(Opnd::UImm(1));
        let rcx = asm.load(Opnd::UImm(2));
        let rdx = asm.load(Opnd::UImm(3));
        // rcx and rdx form a cycle
        asm.ccall(0 as _, vec![
            rax, // mov rdi, rax
            rcx, // mov rsi, rcx
            rcx, // mov rdx, rcx
            rdx, // mov rcx, rdx
        ]);
        asm.compile_with_num_regs(&mut cb, 3);

        assert_disasm_snapshot!(cb.disasm(), @"
            0x0: mov eax, 1
            0x5: mov ecx, 2
            0xa: mov edx, 3
            0xf: mov rdi, rax
            0x12: mov rsi, rcx
            0x15: mov r11, rcx
            0x18: mov rcx, rdx
            0x1b: mov rdx, r11
            0x1e: mov eax, 0
            0x23: call rax
        ");
        assert_snapshot!(cb.hexdump(), @"b801000000b902000000ba030000004889c74889ce4989cb4889d14c89dab800000000ffd0");
    }

    #[test]
    fn test_ccall_register_preservation_even() {
        let (mut asm, mut cb) = setup_asm();

        let v0 = asm.load(1.into());
        let v1 = asm.load(2.into());
        let v2 = asm.load(3.into());
        let v3 = asm.load(4.into());
        asm.ccall(0 as _, vec![]);
        _ = asm.add(v0, v1);
        _ = asm.add(v2, v3);

        asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len());

        assert_disasm_snapshot!(cb.disasm(), @"
        0x0: mov edi, 1
        0x5: mov esi, 2
        0xa: mov edx, 3
        0xf: mov ecx, 4
        0x14: push rdi
        0x15: push rsi
        0x16: push rdx
        0x17: push rcx
        0x18: mov eax, 0
        0x1d: call rax
        0x1f: pop rcx
        0x20: pop rdx
        0x21: pop rsi
        0x22: pop rdi
        0x23: add rdi, rsi
        0x26: mov rdi, rdx
        0x29: add rdi, rcx
        ");
        assert_snapshot!(cb.hexdump(), @"bf01000000be02000000ba03000000b90400000057565251b800000000ffd0595a5e5f4801f74889d74801cf");
    }

    #[test]
    fn test_ccall_register_preservation_odd() {
        let (mut asm, mut cb) = setup_asm();

        let v0 = asm.load(1.into());
        let v1 = asm.load(2.into());
        let v2 = asm.load(3.into());
        let v3 = asm.load(4.into());
        let v4 = asm.load(5.into());
        asm.ccall(0 as _, vec![]);
        _ = asm.add(v0, v1);
        _ = asm.add(v2, v3);
        _ = asm.add(v2, v4);

        asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len());

        assert_disasm_snapshot!(cb.disasm(), @"
        0x0: mov edi, 1
        0x5: mov esi, 2
        0xa: mov edx, 3
        0xf: mov ecx, 4
        0x14: mov r8d, 5
        0x1a: push rdi
        0x1b: push rsi
        0x1c: push rdx
        0x1d: push rcx
        0x1e: push r8
        0x20: push 0
        0x22: mov eax, 0
        0x27: call rax
        0x29: pop r8
        0x2b: pop r8
        0x2d: pop rcx
        0x2e: pop rdx
        0x2f: pop rsi
        0x30: pop rdi
        0x31: add rdi, rsi
        0x34: mov rdi, rdx
        0x37: add rdi, rcx
        0x3a: mov rdi, rdx
        0x3d: add rdi, r8
        ");
        assert_snapshot!(cb.hexdump(), @"bf01000000be02000000ba03000000b90400000041b8050000005756525141506a00b800000000ffd041584158595a5e5f4801f74889d74801cf4889d74c01c7");
    }

    #[test]
    fn test_cpush_pair() {
        let (mut asm, mut cb) = setup_asm();
        let v0 = asm.load(1.into());
        let v1 = asm.load(2.into());
        asm.cpush_pair(v0, v1);
        asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len());

        assert_disasm_snapshot!(cb.disasm(), @"
        0x0: mov edi, 1
        0x5: mov esi, 2
        0xa: push rdi
        0xb: push rsi
        ");
        assert_snapshot!(cb.hexdump(), @"bf01000000be020000005756");
    }

    #[test]
    fn test_cpop_pair_into() {
        let (mut asm, mut cb) = setup_asm();
        let v0 = asm.load(1.into());
        let v1 = asm.load(2.into());
        asm.cpop_pair_into(v0, v1);
        asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len());

        assert_disasm_snapshot!(cb.disasm(), @"
        0x0: mov edi, 1
        0x5: mov esi, 2
        0xa: pop rdi
        0xb: pop rsi
        ");
        assert_snapshot!(cb.hexdump(), @"bf01000000be020000005f5e");
    }

    #[test]
    fn test_cmov_mem() {
        let (mut asm, mut cb) = setup_asm();

        let top = Opnd::mem(64, SP, 0);
        let ary_opnd = SP;
        let array_len_opnd = Opnd::mem(64, SP, 16);

        asm.cmp(array_len_opnd, 1.into());
        let elem_opnd = asm.csel_g(Opnd::mem(64, ary_opnd, 0), Qnil.into());
        asm.mov(top, elem_opnd);

        asm.compile_with_num_regs(&mut cb, 1);

        assert_disasm_snapshot!(cb.disasm(), @"
            0x0: cmp qword ptr [rbx + 0x10], 1
            0x5: mov edi, 4
            0xa: cmovg rdi, qword ptr [rbx]
            0xe: mov qword ptr [rbx], rdi
        ");
        assert_snapshot!(cb.hexdump(), @"48837b1001bf04000000480f4f3b48893b");
    }

    #[test]
    #[ignore]
    fn test_csel_split() {
        let (mut asm, mut cb) = setup_asm();

        let stack_top = Opnd::mem(64, SP, 0);
        let elem_opnd = asm.csel_ne(VALUE(0x7f22c88d1930).into(), Qnil.into());
        asm.mov(stack_top, elem_opnd);

        asm.compile_with_num_regs(&mut cb, 3);

        assert_disasm_snapshot!(cb.disasm(), @"
            0x0: movabs rax, 0x7f22c88d1930
            0xa: mov ecx, 4
            0xf: cmove rax, rcx
            0x13: mov qword ptr [rbx], rax
        ");
        assert_snapshot!(cb.hexdump(), @"48b830198dc8227f0000b904000000480f44c1488903");
    }

    #[test]
    fn test_mov_m32_imm32() {
        let (mut asm, mut cb) = setup_asm();

        let shape_opnd = Opnd::mem(32, C_RET_OPND, 0);
        asm.mov(shape_opnd, Opnd::UImm(0x8000_0001));
        asm.mov(shape_opnd, Opnd::Imm(0x8000_0001));

        asm.compile_with_num_regs(&mut cb, 0);

        assert_disasm_snapshot!(cb.disasm(), @"
            0x0: mov dword ptr [rax], 0x80000001
            0x6: mov dword ptr [rax], 0x80000001
        ");
        assert_snapshot!(cb.hexdump(), @"c70001000080c70001000080");
    }

    #[test]
    fn frame_setup_teardown_preserved_regs() {
        let (mut asm, mut cb) = setup_asm();
        asm.frame_setup(JIT_PRESERVED_REGS);
        asm.frame_teardown(JIT_PRESERVED_REGS);
        asm.cret(C_RET_OPND);
        asm.compile_with_num_regs(&mut cb, 0);

        assert_disasm_snapshot!(cb.disasm(), @"
        0x0: push rbp
        0x1: mov rbp, rsp
        0x4: push r13
        0x6: push rbx
        0x7: push r12
        0x9: sub rsp, 8
        0xd: mov r13, qword ptr [rbp - 8]
        0x11: mov rbx, qword ptr [rbp - 0x10]
        0x15: mov r12, qword ptr [rbp - 0x18]
        0x19: mov rsp, rbp
        0x1c: pop rbp
        0x1d: ret
        ");
        assert_snapshot!(cb.hexdump(), @"554889e541555341544883ec084c8b6df8488b5df04c8b65e84889ec5dc3");
    }

    #[test]
    fn frame_setup_teardown_stack_base_idx() {
        let (mut asm, mut cb) = setup_asm();
        asm.stack_state.stack_base_idx = 5;
        asm.frame_setup(&[]);
        asm.frame_teardown(&[]);
        asm.compile_with_num_regs(&mut cb, 0);

        assert_disasm_snapshot!(cb.disasm(), @"
        0x0: push rbp
        0x1: mov rbp, rsp
        0x4: sub rsp, 0x30
        0x8: mov rsp, rbp
        0xb: pop rbp
        ");
        assert_snapshot!(cb.hexdump(), @"554889e54883ec304889ec5d");
    }

    #[test]
    fn test_store_value_without_split() {
        let (mut asm, mut cb) = setup_asm();

        let imitation_heap_value = VALUE(0x1000);
        assert!(imitation_heap_value.heap_object_p());
        asm.store(Opnd::mem(VALUE_BITS, SP, 0), imitation_heap_value.into());

        asm = asm.x86_scratch_split();
        for name in &asm.label_names {
            cb.new_label(name.to_string());
        }
        let gc_offsets = asm.x86_emit(&mut cb).unwrap();
        assert_eq!(1, gc_offsets.len(), "VALUE source operand should be reported as gc offset");

        assert_disasm_snapshot!(cb.disasm(), @"
            0x0: movabs r11, 0x1000
            0xa: mov qword ptr [rbx], r11
        ");
        assert_snapshot!(cb.hexdump(), @"49bb00100000000000004c891b");
    }

    #[test]
    fn test_csel_split_memory_read() {
        let (mut asm, mut cb) = setup_asm();

        let left = Opnd::Mem(Mem { base: MemBase::Stack { stack_idx: 0, num_bits: 64 }, disp: 0, num_bits: 64 });
        let right = Opnd::Mem(Mem { base: MemBase::Stack { stack_idx: 1, num_bits: 64 }, disp: 2, num_bits: 64 });
        let _ = asm.csel_e(left, right);
        asm.compile_with_num_regs(&mut cb, 0);

        assert_disasm_snapshot!(cb.disasm(), @"
        0x0: mov r11, qword ptr [rbp - 0xe]
        0x4: cmove r11, qword ptr [rbp - 8]
        0x9: mov qword ptr [rbp - 8], r11
        ");
        assert_snapshot!(cb.hexdump(), @"4c8b5df24c0f445df84c895df8");
    }

    #[test]
    fn test_lea_split_memory_read() {
        let (mut asm, mut cb) = setup_asm();

        let opnd = Opnd::Mem(Mem { base: MemBase::Stack { stack_idx: 0, num_bits: 64 }, disp: 0, num_bits: 64 });
        let _ = asm.lea(opnd);
        asm.compile_with_num_regs(&mut cb, 0);

        assert_disasm_snapshot!(cb.disasm(), @"
        0x0: lea r11, [rbp - 8]
        0x4: mov qword ptr [rbp - 8], r11
        ");
        assert_snapshot!(cb.hexdump(), @"4c8d5df84c895df8");
    }

    #[test]
    fn test_add_split_direct_mem() {
        // RAX is safe to be clobbered because it's an output
        // c_ret <- add stack[0], stack[1]
        let lines = split_binop_disasm_lines(BinOpKind::Add, stack_mem(0), stack_mem(1), C_RET_OPND);

        // load scratch0, [stack[0]]
        // add  scratch0, [stack[1]]
        // mov  c_ret, scratch0
        assert_snapshot!(lines.join("\n"), @"
        mov rax, qword ptr [rbp - 8]
        add rax, qword ptr [rbp - 0x10]
        ");
    }

    #[test]
    fn test_add_split_stack_indirect_left_uses_scratch0_for_base_and_result() {
        // stack[1] <- add stack[mem[0]], cfp
        let lines = split_binop_disasm_lines(BinOpKind::Add, stack_indirect_mem(0), CFP, stack_mem(1));

        // load scratch0, [stack[left_base]]
        // load scratch0, [scratch0]
        // seed [stack[out]] from scratch0, then add right in place
        assert_snapshot!(lines.join("\n"), @"
            mov r11, qword ptr [rbp - 8]
            mov r11, qword ptr [r11]
            mov qword ptr [rbp - 0x10], r11
            add qword ptr [rbp - 0x10], r13
        ");
    }

    #[test]
    fn test_add_split_stack_indirect_right_uses_separate_base_scratch() {
        // mem[1] <- add cfp, mem[stack[0]]
        let lines = split_binop_disasm_lines(BinOpKind::Add, CFP, stack_indirect_mem(0), stack_mem(1));

        // load scratch1, [stack[right_base]]
        // load scratch1, [scratch1]
        // seed [stack[out]] from left, then add scratch1 in place
        assert_snapshot!(lines.join("\n"), @"
            mov r10, qword ptr [rbp - 8]
            mov r10, qword ptr [r10]
            mov qword ptr [rbp - 0x10], r13
            add qword ptr [rbp - 0x10], r10
        ");
    }

    #[test]
    fn test_add_split_two_stack_indirect_inputs_need_two_scratch_regs() {
        // stack[2] <- add [stack[0]], [stack[1]]
        let lines = split_binop_disasm_lines(BinOpKind::Add,
            stack_indirect_mem(0), stack_indirect_mem(1), stack_mem(2));

        // load scratch0, [stack[left_base]]
        // load scratch0, [scratch0]
        // load scratch1, [stack[right_base]]
        // load scratch1, [scratch1]
        // mov  [stack[out]], scratch0; add [stack[out]], scratch1
        assert_snapshot!(lines.join("\n"), @"
        mov r11, qword ptr [rbp - 8]
        mov r10, qword ptr [rbp - 0x10]
        mov r10, qword ptr [r10]
        mov r11, qword ptr [r11]
        mov qword ptr [rbp - 0x18], r11
        add qword ptr [rbp - 0x18], r10
        ");
    }

    #[test]
    fn test_add_split_memory_output_can_compute_in_place() {
        // stack[1] <- add cfp, stack[0]
        let lines = split_binop_disasm_lines(BinOpKind::Add, CFP, stack_mem(0), stack_mem(1));

        // Seed the output slot with the left register, then perform the add in place.
        assert_snapshot!(lines.join("\n"), @"
            mov r10, qword ptr [rbp - 8]
            mov qword ptr [rbp - 0x10], r13
            add qword ptr [rbp - 0x10], r10
        ");
    }

    #[test]
    fn test_add_split_reg_mem_mem_when_right_equals_out() {
        // stack[1] <- add cfp, stack[1]
        //
        // Preserve the original RHS/output value first, then seed the output
        // slot with the left register and finish the add in place.
        let lines = split_binop_disasm_lines(BinOpKind::Add, CFP, stack_mem(1), stack_mem(1));

        assert_snapshot!(lines.join("\n"), @"
            mov r10, qword ptr [rbp - 0x10]
            mov qword ptr [rbp - 0x10], r13
            add qword ptr [rbp - 0x10], r10
        ");
    }

    #[test]
    #[should_panic(expected = "x86_scratch_split expects out to be a physical register or stack memory")]
    fn test_binop_split_rejects_non_stack_memory_output() {
        let _ = split_binop(BinOpKind::Add, CFP, C_RET_OPND, Opnd::mem(64, CFP, 8));
    }

    #[test]
    fn test_add_split_mem_mem_mem_when_right_equals_out() {
        // stack[1] <- add stack[0], stack[1]
        //
        // Preserve the original RHS/output value first, then reload the LHS and
        // write the result back to the output stack slot.
        let mut asm = split_binop(BinOpKind::Add, stack_mem(0), stack_mem(1), stack_mem(1));
        let mut cb = CodeBlock::new_dummy();
        for name in &asm.label_names {
            cb.new_label(name.to_string());
        }
        assert!(asm.x86_emit(&mut cb).is_ok());

        assert_disasm_snapshot!(cb.disasm(), @"
        0x0: mov r10, qword ptr [rbp - 0x10]
        0x4: mov r11, qword ptr [rbp - 8]
        0x8: mov qword ptr [rbp - 0x10], r11
        0xc: add qword ptr [rbp - 0x10], r10
        ");
    }

    #[test]
    fn test_add_split_output_reg_reused_as_input_memory_base() {
        // cfp <- add [cfp + 8], [cfp + 16]
        //
        // Preserve the RHS first because reusing `out` for the LHS would otherwise
        // clobber the base register needed to address the RHS memory operand.
        let mut asm = split_binop(
            BinOpKind::Add,
            Opnd::mem(64, CFP, 8),
            Opnd::mem(64, CFP, 16),
            CFP,
        );
        let mut cb = CodeBlock::new_dummy();
        for name in &asm.label_names {
            cb.new_label(name.to_string());
        }
        assert!(asm.x86_emit(&mut cb).is_ok());

        assert_disasm_snapshot!(cb.disasm(), @"
            0x0: mov r10, qword ptr [r13 + 0x10]
            0x4: mov r13, qword ptr [r13 + 8]
            0x8: add r13, r10
        ");
        assert_snapshot!(cb.hexdump(), @"4d8b55104d8b6d084d01d5");
    }

    #[test]
    fn test_add_split_output_reg_reused_as_stack_indirect_memory_base() {
        // cfp <- add mem[stack[0]], mem[stack[1]]
        //
        // The LHS lowering reuses `out` as the temporary base register for the
        // stack-indirect address, then the RHS uses the normal scratch register.
        let mut asm = split_binop(
            BinOpKind::Add,
            stack_indirect_mem(0),
            stack_indirect_mem(1),
            CFP,
        );
        let mut cb = CodeBlock::new_dummy();
        for name in &asm.label_names {
            cb.new_label(name.to_string());
        }
        assert!(asm.x86_emit(&mut cb).is_ok());

        assert_disasm_snapshot!(cb.disasm(), @"
        0x0: mov r11, qword ptr [rbp - 8]
        0x4: mov r10, qword ptr [rbp - 0x10]
        0x8: mov r13, qword ptr [r11]
        0xb: add r13, qword ptr [r10]
        ");
    }

    #[test]
    fn test_add_split_output_reg_reused_as_input_memory_base_with_imm() {
        // cfp <- add 7, [cfp + 16]
        //
        // Preserve the RHS first because reusing `out` for the immediate
        // materialization would otherwise clobber the base register needed for
        // the memory operand.
        let lines = split_binop_disasm_lines(
            BinOpKind::Add,
            Opnd::Imm(7),
            Opnd::mem(64, CFP, 16),
            CFP,
        );

        assert_snapshot!(lines.join("\n"), @"
            mov r10, qword ptr [r13 + 0x10]
            mov r13d, 7
            add r13, r10
        ");
    }

    #[test]
    fn test_binop_split_matrix() {
        let left_cases = [
            ("reg", CFP),
            ("mem_reg", Opnd::mem(64, C_RET_OPND, 8)),
            ("mem_stack", stack_mem(0)),
            ("mem_stack_indirect", stack_indirect_mem(0)),
            ("imm32", Opnd::Imm(7)),
            ("imm64", Opnd::UImm(0x1_0000_0000)),
        ];
        let right_cases = [
            ("reg", C_RET_OPND),
            ("mem_reg", Opnd::mem(64, CFP, 16)),
            ("mem_stack", stack_mem(1)),
            ("mem_stack_indirect", stack_indirect_mem(1)),
            ("imm32", Opnd::Imm(9)),
            ("imm64", Opnd::UImm(0x2_0000_0000)),
        ];
        let out_cases = [
            ("out_reg", C_ARG_OPNDS[0]),
            ("out_mem_stack", stack_mem(2)),
        ];
        let alias_out_cases = [
            ("alias_out_cfp", CFP),
            ("alias_out_reg", C_RET_OPND),
            ("alias_out_mem_stack", stack_mem(1)),
        ];
        let alias_mem_base_cases = [
            (
                "out_reused_as_left_mem_base",
                Opnd::mem(64, CFP, 8),
                C_RET_OPND,
                CFP,
            ),
            (
                "out_reused_as_right_mem_base",
                C_RET_OPND,
                Opnd::mem(64, CFP, 16),
                CFP,
            ),
            (
                "out_reused_as_both_mem_bases",
                Opnd::mem(64, CFP, 8),
                Opnd::mem(64, CFP, 16),
                CFP,
            ),
        ];

        for kind in [BinOpKind::Add, BinOpKind::Sub, BinOpKind::And, BinOpKind::Or, BinOpKind::Xor] {
            for (left_name, left) in left_cases {
                for (right_name, right) in right_cases {
                    for (out_name, out) in out_cases {
                        let case = format!("{left_name}/{right_name}/{out_name}");
                        assert_split_binop_case(kind, left, right, out, &case);
                    }
                }
            }

            for (left_name, left) in left_cases {
                for (alias_name, out) in alias_out_cases {
                    let case = format!("{left_name}/right_eq_out/{alias_name}");
                    assert_split_binop_case(kind, left, out, out, &case);
                }
            }

            for (case, left, right, out) in alias_mem_base_cases {
                assert_split_binop_case(kind, left, right, out, case);
            }
        }
    }
}
