// SPDX-License-Identifier: LGPL-2.1-or-later
// Derived from chibicc by Rui Ueyama.
// Binary codegen: asm_* wrappers emit bytes via secbuf_emit*() to ObjFile.
#include "rcc.h"
#include "asm.h"
#include "codegen_asm.h"
#include "bitint_rt.h"

AsmFixupNode *asm_fixup_htab[CG_HT_SIZE];
int asm_fixup_count = 0;

#include <ctype.h>
#include <stdarg.h>
#include <time.h>

SecBuf *cg_sec;
ObjFile *cg_obj;
static bool cg_discard_result;
bool cg_dry_run; // pass 1: track regs only (extern for codegen_asm.h)
CgAsciiStr *cg_ascii_strings;

static void cg_set_section(int sec) {
    if (!cg_obj) return;
    switch (sec) {
    case SEC_TEXT: cg_sec = &cg_obj->text; break;
    case SEC_DATA: cg_sec = &cg_obj->data; break;
    case SEC_RODATA: cg_sec = &cg_obj->rodata; break;
    case SEC_INIT_ARRAY: cg_sec = &cg_obj->init_array; break;
    case SEC_FINI_ARRAY: cg_sec = &cg_obj->fini_array; break;
    case SEC_TDATA: cg_sec = &cg_obj->data_tls; break;
    case SEC_THREAD_VARS: cg_sec = &cg_obj->thread_vars; break;
    default: {
        // A custom section id (>= SEC_NUM, e.g. a global with
        // __attribute__((section("name")))) has its own growable SecBuf
        // in obj->extra_secs rather than one of the fixed fields above.
        SecBuf *custom = objfile_section_buf(cg_obj, sec);
        cg_sec = custom ? custom : &cg_obj->text;
        break;
    }
    }
}

#ifdef _WIN32
// Emit the MinGW emulated TLS control block and template for a thread-local
// global variable.  The control block lives in .data and is what
// __emutls_get_address receives; the template lives in .rdata and holds the
// per-thread initial value.
static void cg_emit_emutls_data(LVar *var) {
    char *label = var->asm_name ? var->asm_name : var->name;
    int align = var->ty->align > 1 ? var->ty->align : 1;

    cg_set_section(SEC_DATA);
    secbuf_align(cg_sec, 32);
    size_t ctrl_off = cg_sec->len;
    char *ctrl_name = format("__emutls_v.%s", label);
    SymBind bind = var->is_static ? SB_LOCAL : SB_GLOBAL;
    objfile_add_sym(cg_obj, ctrl_name, SEC_DATA, ctrl_off, 32, bind, ST_OBJECT);
    secbuf_emit64le(cg_sec, var->ty->size);
    secbuf_emit64le(cg_sec, align);
    secbuf_emit64le(cg_sec, 0);
    size_t ptr_off = cg_sec->len;
    secbuf_emit64le(cg_sec, 0);

    char *tmpl_name = format("__emutls_t.%s", label);
    int tidx = objfile_find_sym(cg_obj, tmpl_name);
    if (tidx < 0)
        tidx = objfile_add_sym(cg_obj, tmpl_name, SEC_UNDEF, 0, 0, SB_LOCAL, ST_NOTYPE);
    objfile_add_reloc(cg_obj, SEC_DATA, ptr_off, tidx, R_X86_64_64, 0);

    cg_set_section(SEC_RODATA);
    secbuf_align(cg_sec, align);
    size_t tmpl_off = cg_sec->len;
    objfile_add_sym(cg_obj, tmpl_name, SEC_RODATA, tmpl_off, var->ty->size, SB_LOCAL, ST_OBJECT);
    if (var->init_data || var->relocs) {
        int pos = 0;
        for (Reloc *rel = var->relocs; rel; rel = rel->next) {
            for (; pos < rel->offset; pos++)
                secbuf_emit8(cg_sec, (uint8_t)var->init_data[pos]);
            size_t rel_off = cg_sec->len;
            secbuf_emit64le(cg_sec, 0);
            int sidx = objfile_find_sym(cg_obj, rel->label);
            if (sidx < 0)
                sidx = objfile_add_sym(cg_obj, rel->label, SEC_UNDEF, 0, 0, SB_GLOBAL, ST_NOTYPE);
            objfile_add_reloc(cg_obj, SEC_RODATA, rel_off, sidx, R_X86_64_64, (int64_t)rel->addend);
            pos += 8;
        }
        for (; pos < var->init_size; pos++)
            secbuf_emit8(cg_sec, (uint8_t)var->init_data[pos]);
        if (var->ty->size > var->init_size) {
            size_t pad = var->ty->size - var->init_size;
            secbuf_reserve(cg_sec, pad);
            memset(cg_sec->data + cg_sec->len, 0, pad);
            cg_sec->len += pad;
        }
    } else if (var->has_init) {
        if (var->ty->size == 1)
            secbuf_emit8(cg_sec, (uint8_t)var->init_val);
        else if (var->ty->size == 2)
            secbuf_emit16le(cg_sec, (uint16_t)var->init_val);
        else if (var->ty->size == 4)
            secbuf_emit32le(cg_sec, (uint32_t)var->init_val);
        else
            secbuf_emit64le(cg_sec, (uint64_t)var->init_val);
    } else if (var->ty->size > 0) {
        size_t pad = var->ty->size;
        secbuf_reserve(cg_sec, pad);
        memset(cg_sec->data + cg_sec->len, 0, pad);
        cg_sec->len += pad;
    }
}
#endif

// Define (finalize) a label/symbol at the current section offset. Forward
// references via emit_adrp_add/emit_adrp_got may have already created this
// symbol as a SEC_RODATA/ST_NOTYPE placeholder at offset 0; objfile_add_sym
// only updates existing symbols when their section is SEC_UNDEF, so such a
// placeholder would otherwise keep its bogus offset forever. Force-overwrite
// it here since this call site holds the authoritative offset.
static void cg_def_sym(const char *name, int sec, SymBind bind, SymType type) {
    int idx = objfile_find_sym(cg_obj, name);
    if (idx >= 0) {
        cg_obj->syms[idx].section = sec;
        cg_obj->syms[idx].offset = cg_sec->len;
        cg_obj->syms[idx].bind = bind;
        cg_obj->syms[idx].type = type;
    } else {
        objfile_add_sym(cg_obj, name, sec, cg_sec->len, 0, bind, type);
    }
}

// cg_def_label: internal branch-target / loop / switch-case labels (the
// overwhelming majority of .L-prefixed symbols a function emits). These are
// NOT function entry points — GAS never types a local branch label as
// STT_FUNC either — so they get ST_NOTYPE. They still need a symbol table
// entry (not just an internal fixup) because computed-goto (&&label) values
// need something for the linker's ARM64_RELOC_UNSIGNED/R_X86_64_64 to point
// at. See cg_def_fn_label below for the actual-function-symbol case.
void cg_def_label(const char *name) {
    if (cg_dry_run) return;
    // .L. labels referenced by relocations (e.g. &&label values) must be in the
    // symbol table so the linker can resolve ARM64_RELOC_UNSIGNED relocations.
    // We use SB_LOCAL to avoid creating subsection boundaries that the linker
    // might dead-strip — local symbols are not subsection boundaries.
    cg_def_sym(name, SEC_TEXT, SB_LOCAL, ST_NOTYPE);
    cg_label_ht_add(name, cg_sec->len);
    asm_fixup_resolve(cg_sec, name, cg_sec->len);
}

void cg_def_label_sec(const char *name, int sec) {
    if (cg_dry_run) return;
    cg_def_sym(name, sec, SB_LOCAL, ST_OBJECT);
    cg_label_ht_add(name, cg_sec->len);
    asm_fixup_resolve(cg_sec, name, cg_sec->len);
}

static void cg_global_label(const char *name) {
    if (cg_dry_run) return;
    objfile_add_sym(cg_obj, name, SEC_TEXT, cg_sec->len, 0, SB_GLOBAL, ST_FUNC);
    cg_label_ht_add(name, cg_sec->len);
    asm_fixup_resolve(cg_sec, name, cg_sec->len);
}

static void cg_weak_label(const char *name) {
    if (cg_dry_run) return;
    objfile_add_sym(cg_obj, name, SEC_TEXT, cg_sec->len, 0, SB_WEAK, ST_FUNC);
}

// cg_def_fn_label: the static/non-exported-function counterpart to
// cg_global_label/cg_weak_label above — a REAL function entry point (unlike
// cg_def_label's internal branch labels), so it keeps ST_FUNC. Pair with
// cg_patch_fn_size once the function's epilogue is fully emitted so the
// ELF/COFF symbol carries a correct size instead of the 0 every ST_FUNC
// symbol here used to get (objtool: "%s() is missing an ELF size
// annotation" for literally every function in the object file).
static void cg_def_fn_label(const char *name) {
    if (cg_dry_run) return;
    cg_def_sym(name, SEC_TEXT, SB_LOCAL, ST_FUNC);
    cg_label_ht_add(name, cg_sec->len);
    asm_fixup_resolve(cg_sec, name, cg_sec->len);
}

// Close the loop started by cg_global_label/cg_weak_label/cg_def_fn_label:
// patch the function symbol's size now that its full extent is known. GAS
// emits ".size name, .-name" for exactly this reason — the size can't be
// known at the label-definition site, only once the epilogue's last
// instruction has been emitted.
static void cg_patch_fn_size(const char *name, size_t start_off) {
    if (cg_dry_run) return;
    int idx = objfile_find_sym(cg_obj, name);
    if (idx >= 0)
        cg_obj->syms[idx].size = cg_sec->len - start_off;
}

static void cg_weak_declare(const char *name) {
    if (cg_dry_run) return;
    int sidx = objfile_find_sym(cg_obj, name);
    if (sidx < 0)
        objfile_add_sym(cg_obj, name, SEC_UNDEF, 0, 0, SB_WEAK, ST_NOTYPE);
    else if (cg_obj->syms[sidx].section == SEC_UNDEF)
        cg_obj->syms[sidx].bind = SB_WEAK;
}

#ifdef _WIN32
// Win64 SEH unwind recording. Mirrors the .seh_* directives the golden
// codegen.c.main emitted (.seh_proc/.seh_pushreg/.seh_stackalloc/
// .seh_endprologue/.seh_endproc); coff_write turns the captured entries into
// .pdata/.xdata. The X86Reg enum value equals the Win64 unwind register
// number (RBP=5, RBX=3, R12..R15=12..15, RSI=6), so it is used directly.
static UnwindEntry *cg_uw; // current function's unwind entry (NULL when inactive)
static uint32_t cg_uw_start; // .text offset of the current function start

static void uw_begin(void) { // .seh_proc
    if (cg_dry_run || !cg_obj) {
        cg_uw = NULL;
        return;
    }
    cg_uw = objfile_add_unwind(cg_obj);
    cg_uw_start = (uint32_t)cg_sec->len;
    cg_uw->func_start = cg_uw_start;
}
static void uw_pushreg(int reg) { // .seh_pushreg
    if (!cg_uw) return;
    UnwindCode *c = &cg_uw->codes[cg_uw->code_count++];
    c->code_offset = (uint8_t)((uint32_t)cg_sec->len - cg_uw_start);
    c->op = UWOP_PUSH_NONVOL;
    c->info = (uint8_t)reg;
}
static void uw_set_fpreg(int reg) { // .seh_setframe
    if (!cg_uw) return;
    cg_uw->frame_register = (uint8_t)reg;
    UnwindCode *c = &cg_uw->codes[cg_uw->code_count++];
    c->code_offset = (uint8_t)((uint32_t)cg_sec->len - cg_uw_start);
    c->op = UWOP_SET_FPREG;
    c->info = (uint8_t)reg;
}
static void uw_stackalloc(int size) { // .seh_stackalloc
    if (!cg_uw) return;
    UnwindCode *c = &cg_uw->codes[cg_uw->code_count++];
    c->code_offset = (uint8_t)((uint32_t)cg_sec->len - cg_uw_start);
    if (size <= 128) {
        c->op = UWOP_ALLOC_SMALL;
        c->info = (uint8_t)(size / 8 - 1);
    } else if (size <= 8 * 0xffff) {
        c->op = UWOP_ALLOC_LARGE;
        c->info = 0;
        c->extra[0] = (uint16_t)(size / 8);
        c->extra_count = 1;
    } else {
        c->op = UWOP_ALLOC_LARGE;
        c->info = 1;
        c->extra[0] = (uint16_t)(size & 0xffff);
        c->extra[1] = (uint16_t)((uint32_t)size >> 16);
        c->extra_count = 2;
    }
}
static void uw_endprologue(void) { // .seh_endprologue
    if (!cg_uw) return;
    cg_uw->prolog_size = (uint8_t)((uint32_t)cg_sec->len - cg_uw_start);
}
static void uw_endproc(void) { // .seh_endproc
    if (!cg_uw) return;
    cg_uw->func_end = (uint32_t)cg_sec->len;
    cg_uw = NULL;
}
#endif


#ifndef ARCH_ARM64
size_t asm_lea_rip_reg(SecBuf *s, int r, const char *label) {
    size_t off = s->len;
    X86Mem m = {X86_RIP, X86_NOREG, 1, 0};
    x86_lea(s, 8, REG(r), m);
    if (!cg_dry_run) {
        int sidx = objfile_find_sym(cg_obj, label);
        if (sidx < 0) {
            bool is_local_label = label[0] == '.';
            sidx = objfile_add_sym(cg_obj, label, SEC_UNDEF, 0, 0, is_local_label ? SB_LOCAL : SB_GLOBAL, ST_NOTYPE);
        }
        objfile_add_reloc(cg_obj, SEC_TEXT, off + 3, sidx, R_X86_64_PC32, -4);
    }
    asm_record(ASM_LEA_FP, off, s->len - off, r, -1, -1, 8, 0, 0, label, 0, -1, false);
    return s->len - off;
}
// movsd .Llabel(%rip), %xmm0 — 8 bytes: f2 0f 10 05 <disp32>
static size_t asm_movsd_rip_xmm(SecBuf *s, const char *label) {
    size_t off = s->len;
    secbuf_emit32le(s, 0x05100ff2); // f2 0f 10 05
    secbuf_emit32le(s, 0); // placeholder disp32
    if (!cg_dry_run) {
        int sidx = objfile_find_sym(cg_obj, label);
        if (sidx < 0) {
            bool il = label[0] == '.';
            sidx = objfile_add_sym(cg_obj, label, SEC_UNDEF, 0, 0, il ? SB_LOCAL : SB_GLOBAL, ST_NOTYPE);
        }
        objfile_add_reloc(cg_obj, SEC_TEXT, off + 4, sidx, R_X86_64_PC32, -4);
    }
    return s->len - off;
}
// mov sym@GOTPCREL(%rip), reg  — load GOT entry pointer (x86 only)
// On COFF/PE there is no separate GOT cell for our weak-symbol scheme: an
// unresolved weak external resolves directly to the absolute address 0 (see
// coff_write.c), so the address must be computed with lea, not loaded with
// mov (which would dereference whatever lands at that resolved address).
static size_t asm_mov_got_rip_reg(SecBuf *s, int r, const char *label) {
    size_t off = s->len;
    X86Mem m = {X86_RIP, X86_NOREG, 1, 0};
#ifdef _WIN32
    x86_lea(s, 8, REG(r), m); // lea sym(%rip), reg
#else
    x86_mov_rm(s, 8, REG(r), m); // mov sym@GOTPCREL(%rip), reg
#endif
    if (!cg_dry_run) {
        int sidx = objfile_find_sym(cg_obj, label);
        if (sidx < 0)
            sidx = objfile_add_sym(cg_obj, label, SEC_UNDEF, 0, 0, SB_GLOBAL, ST_NOTYPE);
        objfile_add_reloc(cg_obj, SEC_TEXT, off + 3, sidx, R_X86_64_GOTPCREL, -4);
    }
    asm_record(ASM_MOV_RRBP, off, s->len - off, r, -1, -1, 8, 0, 0, label, 0, -1, false);
    return s->len - off;
}
__attribute__((unused)) static size_t asm_mov_fs0_reg(SecBuf *s, VReg r) {
    X86Reg reg = REG(r);
    EMIT_GUARD;
    size_t off = s->len;
    secbuf_emit8(s, 0x64);
    uint8_t rex = 0x48;
    if (reg & 8) rex |= 0x04;
    secbuf_emit8(s, rex);
    secbuf_emit8(s, 0x8B);
    secbuf_emit8(s, (uint8_t)(((reg & 7) << 3) | 0x04));
    secbuf_emit8(s, 0x25);
    secbuf_emit32le(s, 0);
    return s->len - off;
}
__attribute__((unused)) static size_t asm_lea_tpoff_base_reg(SecBuf *s, VReg dst, VReg base, const char *label) {
    X86Reg rd = REG(dst);
    X86Reg rb = REG(base);
    EMIT_GUARD;
    if (opt_pic) {
        // Initial-exec TLS model for shared objects:
        //   mov var@GOTTPOFF(%rip), dst ; add base, dst
        // (base holds %fs:0). Local-exec TPOFF32 relocs are rejected by
        // the linker when building a .so.
        size_t off = s->len;
        X86Mem m = {X86_RIP, X86_NOREG, 1, 0};
        x86_mov_rm(s, 8, rd, m); // mov var@GOTTPOFF(%rip), dst
        if (!cg_dry_run) {
            int sidx = objfile_find_sym(cg_obj, label);
            if (sidx < 0) {
                bool il = label[0] == '.';
                sidx = objfile_add_sym(cg_obj, label, SEC_UNDEF, 0, 0, il ? SB_LOCAL : SB_GLOBAL, ST_TLS);
            }
            objfile_add_reloc(cg_obj, SEC_TEXT, off + 3, sidx, R_X86_64_GOTTPOFF, -4);
        }
        x86_add_rr(s, 8, rd, rb); // add base, dst
        return s->len - off;
    }
    size_t off = s->len;
    uint8_t rex = 0x48;
    if (rd & 8) rex |= 0x04;
    if (rb & 8) rex |= 0x01;
    secbuf_emit8(s, rex);
    secbuf_emit8(s, 0x8D);
    if ((rb & 7) == X86_RSP) {
        secbuf_emit8(s, (uint8_t)(0x80 | ((rd & 7) << 3) | 0x04));
        secbuf_emit8(s, 0x24);
    } else {
        secbuf_emit8(s, (uint8_t)(0x80 | ((rd & 7) << 3) | (rb & 7)));
    }
    size_t disp_off = s->len;
    secbuf_emit32le(s, 0);
    if (!cg_dry_run) {
        int sidx = objfile_find_sym(cg_obj, label);
        if (sidx < 0) {
            bool il = label[0] == '.';
            sidx = objfile_add_sym(cg_obj, label, SEC_UNDEF, 0, 0, il ? SB_LOCAL : SB_GLOBAL, ST_NOTYPE);
        }
        objfile_add_reloc(cg_obj, SEC_TEXT, disp_off, sidx, R_X86_64_TPOFF32, 0);
    }
    return s->len - off;
}

#endif

// Forward-fixup callback for assemble_inline: registers unresolved labels
// into the codegen fixup table so cg_def_label can patch them later.
// patch_off is the offset of the 4-byte REL32 operand within section
// `section` — not always cg_obj->text: a %l[label] branch inside a
// .pushsection'd non-.text region (e.g. the kernel's ALTERNATIVE() macro's
// .altinstr_replacement) can forward-reference a C label that's defined
// elsewhere (always in .text, via cg_def_label), so the pending fixup must
// remember which buffer to patch once that label shows up — cg_def_label
// itself always resolves against cg_sec, which is the *label's* section,
// not necessarily the branch's.
static void cg_inline_fixup_cb(size_t patch_off, int section, const char *label, void *ctx) {
    (void)ctx;
    char *dup = arena_alloc(strlen(label) + 1);
    strcpy(dup, label);
    SecBuf *sbuf = objfile_section_buf(cg_obj, section);
    // instr_off = patch_off - 1 (type=0 patches at instr_off+1); delta formula is identical.
    asm_fixup_ht_add_sec2(patch_off - 1, dup, 0, sbuf, section);
}

// Bridge for the remaining printf("  <asm line>\n", ...)-based codegen (x86 only):
// assembles the formatted line directly into cg_obj->text via assemble_inline().
// ARM64 uses direct asm_*() wrappers exclusively.
#ifndef ARCH_ARM64
static void cg_emit(const char *fmt, ...) {
    if (cg_dry_run) return;
    char buf[512];
    va_list ap;
    va_start(ap, fmt);
    int n = vsnprintf(buf, sizeof(buf), fmt, ap);
    va_end(ap);
    if (n <= 0) return;
    while (n > 0 && (buf[n - 1] == '\n' || buf[n - 1] == '\r')) buf[--n] = '\0';
    if (n == 0) return;
    char *p = buf;
    while (*p == ' ' || *p == '\t') p++;
    size_t len = strlen(p);
    if (len > 1 && p[len - 1] == ':') {
        bool is_label = true;
        for (size_t i = 0; i + 1 < len; i++)
            if (isspace((unsigned char)p[i])) {
                is_label = false;
                break;
            }
        if (is_label) {
            p[len - 1] = '\0';
            cg_def_label(p);
            return;
        }
    }
    assemble_inline(cg_obj, p, cg_inline_fixup_cb, NULL);
}
#define printf(...) cg_emit(__VA_ARGS__)
#endif /* !ARCH_ARM64 */

Function *current_fn_def;
static TLItem *all_items;
static StrLit *all_strs;

#define FUNC_HASH_SIZE 8192
static TLItem *func_htab[FUNC_HASH_SIZE];

static uint32_t func_hash_name(const char *s) {
    uint32_t h = 2166136261u;
    for (int i = 0; s[i]; i++) {
        h ^= (unsigned char)s[i];
        h *= 16777619;
    }
    return h;
}

static char *current_fn;
const char *cg_dbg_fn = "?";
int cg_dbg_line = 0;
int current_fn_stack_size = 0; // fn->stack_size of the function being generated

// Debug info (DWARF .loc directives, enabled by -g)
static int last_debug_file = 0;
static int last_debug_line = 0;

static int get_debug_file_idx(char *filename) {
    if (!filename) return 1;
    int idx = objfile_add_debug_file(cg_obj, filename);
    return idx;
}

static void emit_loc(Node *node) {
    if (node->tok && node->tok->filename && node->tok->lineno > 0)
        cg_dbg_line = node->tok->lineno;
    if (cg_dry_run) return; // Pass 1 uses a throwaway buffer; offsets are not real
    if (!node->tok || !node->tok->filename)
        return;
    int line = node->tok->lineno;
    if (line <= 0) return;
    int fidx = get_debug_file_idx(node->tok->filename);
    if (fidx == last_debug_file && line == last_debug_line)
        return;
    objfile_add_debug_line(cg_obj, cg_sec->len, fidx, line);
    last_debug_file = fidx;
    last_debug_line = line;
}
int fn_struct_ret_off = 0; // next free offset within the struct-ret-buf area
int fn_struct_ret_total = 0; // high-water mark of struct-ret-buf space used
// Same two-pass-stable allocation scheme as fn_struct_ret_off/total, for
// GNU nested-function trampoline slots (see rcc.h's TRAMPOLINE_SIZE);
// this region starts right after fn_struct_ret_total's (see the `off`
// computation at each allocation site and `need`'s frame-size folding).
static int fn_trampoline_off = 0;
static int fn_trampoline_total = 0;
// next_spill_slot right after per-fn spill init (8 ARM64, 24 x86-64).
// Growth past this during Pass 1 is real spill pressure and gets
// re-anchored on top of `need` before Pass 2 (see both prologue sites).
static int spill_reserved_base;
int rcc_label_count = 0;
static int va_gp_start;
static int va_fp_start;
static int va_st_start;
#ifndef ARCH_ARM64
static int va_reg_save_ofs;
#endif
static int break_stack[128];
static int continue_stack[128];
static int ctrl_depth = 0;
static int float_lit_count = 0;

typedef struct FloatLit FloatLit;
struct FloatLit {
    FloatLit *next;
    int id;
    double val;
    int size; // 4=float, 8=double, TODO 12|16=long double
};
static FloatLit *float_lits;
static bool alloca_needed;
static bool fn_uses_alloca;

#ifdef ARCH_ARM64
// Arm64: 12 allocatable GP registers
#define NUM_REGS 12
// x16-x18: reserved, x19-x28: callee-saved, x29: fp, x30: lr, sp/xzr
// We use x10-x15 (6 caller-saved) and x19-x24 (6 callee-saved) = 12 allocatable
static char *reg64[] = {"x10", "x11", "x12", "x13", "x14", "x15", "x19", "x20", "x21", "x22", "x23", "x24"};
static char *reg32[] = {"w10", "w11", "w12", "w13", "w14", "w15", "w19", "w20", "w21", "w22", "w23", "w24"};
// Arm64 has no dedicated 8/16-bit registers; use w regs, mask after load/store
#define reg16 reg32
#define reg8  reg32
#define NUM_REGS 12
#define FRAME_PTR CG_ARM_FP
// codeql[cpp/commented-out-code]: names the AAPCS64 link-reg/sp roles for reference; unused since FRAME_PTR is the only one needed
//#define LINK_REG  "x30"
//#define STACK_REG "sp"

#else
// x86_64: 8 allocatable GP registers
static char *reg64[] = {"%r10", "%r11", "%rbx", "%r12", "%r13", "%r14", "%r15", "%rsi"};
static char *reg32[] = {"%r10d", "%r11d", "%ebx", "%r12d", "%r13d", "%r14d", "%r15d", "%esi"};
static char *reg16[] = {"%r10w", "%r11w", "%bx", "%r12w", "%r13w", "%r14w", "%r15w", "%si"};
static char *reg8[] = {"%r10b", "%r11b", "%bl", "%r12b", "%r13b", "%r14b", "%r15b", "%sil"};
#define NUM_REGS 8
#define FRAME_PTR CG_X86_FP
// codeql[cpp/commented-out-code]: names the x86-64 stack-pointer register for reference; unused since FRAME_PTR is the only one needed
//#define STACK_REG "rsp"
#endif

static int used_regs = 0;
static int ever_used_regs = 0;

#ifdef ARCH_ARM64
void emit_mov_imm64(Arm64Reg reg, uint64_t val);

// Arm64: spill to [x29, #-N]; offsets grown from frame base.
//
// A register index can have MULTIPLE outstanding spills at once: under
// heavy register pressure, alloc_reg()'s spill-victim search can pick a
// register that is already mid-spill (protecting some outer, still-live
// VReg) as the victim for a second, unrelated borrower -- e.g. evaluating
// a bitfield read-modify-write nested inside a ternary that is itself the
// last argument of a 5+-argument call. Tracking only ONE slot per index
// (the original scheme) makes the second spill overwrite the first
// spill's bytes: when the first (outer) value is finally restored, it
// reads back the SECOND spill's data instead of its own, silently
// corrupting whichever VReg the physical register was standing in for at
// the time. Found via a qbe-style `(Con){.type=CBits, .bits.i=val}`
// nested-designator compound literal as the last of 5 call arguments: an
// inner `or %reg,%reg` (base ORed with itself instead of the shifted new
// bits) silently dropped the just-computed bitfield write and clobbered
// an already-spilled sibling temporary.
//
// spill_offset(r) keeps its original "lazily create, then always return
// the SAME (topmost/current) slot" contract for the many read-only call
// sites elsewhere in this file (comparisons/arithmetic operating directly
// on a still-outstanding spill's backing memory) -- those never push or
// pop. Only alloc_reg()'s spill-victim selection (push_spill_slot, a
// FRESH slot every call) and free_reg()/materialize_reg()'s restore
// (pop_spill_slot, consuming the current top) touch stack depth.
#define MAX_SPILL_DEPTH 32
static int spill_slot[NUM_REGS][MAX_SPILL_DEPTH];
static int spill_depth[NUM_REGS];
static int next_spill_slot;

static int push_spill_slot(int r) {
    next_spill_slot += 8;
    if (spill_depth[r] < MAX_SPILL_DEPTH)
        spill_slot[r][spill_depth[r]++] = next_spill_slot;
    return next_spill_slot;
}

static int pop_spill_slot(int r) {
    if (spill_depth[r] > 0)
        return spill_slot[r][--spill_depth[r]];
    return spill_slot[r][0]; // shouldn't happen; degrade rather than crash
}

static int spill_offset(int r) {
    if (spill_depth[r] == 0)
        return push_spill_slot(r);
    return spill_slot[r][spill_depth[r] - 1];
}
#else
// Spill slot offsets from rbp for register spilling (dynamic, grows from 8).
// See the ARM64 branch above for why this is a per-register STACK, not a
// single slot: a physical register can be re-spilled (borrowed a second
// time) while an earlier spill of the same index is still outstanding,
// under heavy register pressure (e.g. a bitfield read-modify-write nested
// inside a ternary that is the last of 5+ call arguments).
#define MAX_SPILL_DEPTH 32
static int spill_slot[NUM_REGS][MAX_SPILL_DEPTH];
static int spill_depth[NUM_REGS];
static int next_spill_slot;
static int spill_logand, spill_atomic_old;
#define ALL_REGS_MASK ((1 << NUM_REGS) - 1)

static int alloc_spill_slot(void);

static void init_spill_slots(void) {
    memset(spill_slot, 0, sizeof(spill_slot));
    memset(spill_depth, 0, sizeof(spill_depth));
    next_spill_slot = 8;
    spill_logand = alloc_spill_slot();
    spill_atomic_old = alloc_spill_slot();
}

static int alloc_spill_slot(void) {
    int slot = next_spill_slot;
    next_spill_slot += 8;
    return slot;
}

static int push_spill_slot(int r) {
    int slot = alloc_spill_slot();
    if (spill_depth[r] < MAX_SPILL_DEPTH)
        spill_slot[r][spill_depth[r]++] = slot;
    return slot;
}

static int pop_spill_slot(int r) {
    if (spill_depth[r] > 0)
        return spill_slot[r][--spill_depth[r]];
    return spill_slot[r][0]; // shouldn't happen; degrade rather than crash
}

static int spill_offset(int r) {
    if (spill_depth[r] == 0)
        return push_spill_slot(r);
    return spill_slot[r][spill_depth[r] - 1];
}
#endif

static int spilled_regs = 0;
static int spill_count = 0;
static const char *reg_owner[NUM_REGS];

// Builtin name pointers that remain in codegen.c (others moved to cg_builtins.c)
#define _BI_LOCAL(s) keyword_interned(s, sizeof(s) - 1)
char *bi_unreachable;
static char *kw_retbuf;

static void init_local_builtins(void) {
    static bool done = false;
    if (done) return;
    done = true;
    kw_retbuf = _BI_LOCAL("__retbuf");
    bi_unreachable = str_intern("__builtin_unreachable", 21);
}

// True if a block statement is a bare __builtin_unreachable() call.
static bool stmt_is_unreachable(Node *n) {
    if (!n)
        return false;
    Node *e = (n->kind == ND_EXPR_STMT) ? n->lhs : n;
    return e && e->kind == ND_FUNCALL && e->funcname &&
        e->funcname == bi_unreachable;
}
static size_t emit_jcc_fixup(SecBuf *s, int cond, const char *label) {
    size_t off = asm_jcc_label(s, cond);
    asm_fixup_add(s, off, label, 1);
    return off;
}
static void emit_jmp_fixup(SecBuf *s, const char *label) {
    size_t off = asm_jmp_label(s);
    asm_fixup_add(s, off, label, 0);
}


static char *reg(VReg r, int size);
static void materialize_reg(VReg r);
static int free_reg_count(void);
VReg gen(Node *node);
VReg gen_addr(Node *node);
static bool is_asm_reserved(const char *name);
static void sign_extend_to(VReg r, int from_size, int to_size);
static void zero_extend_to(VReg r, int from_size, int to_size);
VReg alloc_int128_addr(void);
int alloc_int128_slot(void);
int alloc_wide_slot(int size);
VReg alloc_wide_addr(int size);
static VReg gen_to_int128(Node *operand);
static VReg gen_int128(Node *node);
static VReg gen_bitint(Node *node);
static void gen_flonum_branch_if_zero(VReg r, size_t *fwd_off, const char *shared_label);
static void gen_flonum_branch_if_nonzero(VReg r, const char *label);
static void gen_flonum_truthiness(VReg r);

static void emit_bitint_bits_arg(int bits);
static void emit_bitint_addr_arg(VReg r, int n);
static void emit_bitint_imm_arg(int imm, int n);
static void emit_bitint_slot_arg(int slot, int n);
static void emit_bitint_amount_arg(VReg r, int n);
static void emit_bitint_copy_bytes(VReg dst, VReg src, int size);
static void emit_bitint_const_fill(VReg dst, long long val, int size, bool is_unsigned);
static VReg widen_to_bitint(VReg val, Type *ty, bool src_is_unsigned);
static VReg gen_to_bitint(Node *operand);

// A 16-byte value passed in two GP registers: __int128 and _BitInt(65..128)
// share the SysV/Win64/AAPCS64 convention (2 consecutive GP arg regs,
// RAX:RDX / X0:X1 return, 16-byte stack slot). _BitInt(N) with N > 128
// (size > 16) instead follows the large-struct convention (hidden return
// buffer + by-pointer/stack arg marshalling).
static bool is_int128_like(Type *ty) {
    return ty && (ty->kind == TY_INT128 || ty->kind == TY_DECIMAL128 || (ty->kind == TY_BITINT && ty->size == 16));
}

// A wide _BitInt (N > 64, size > 8): routed through gen_bitint().
static bool is_wide_bitint(Type *ty) {
    return ty && ty->kind == TY_BITINT && ty->size > 8;
}

// ---------------------------------------------------------------------------
// Decimal floating point (_Decimal32/64/128), IEEE 754-2008 BID encoding.
//
// Values are plain BID bit patterns: _Decimal32/64 fit a GP register,
// _Decimal128 is a 16-byte slot (like __int128). ALL arithmetic,
// comparisons and conversions go through the __bid_* runtime helpers from
// the bundled libdfp.a (lib/libdfp/rcc_dec_rt.c wrapper layer + the
// vendored libbid core); rcc's linker auto-links libdfp.a for decimal TUs
// (see main.c). The helpers use the kefir/gcc-compatible symbol names and
// a plain bit-pattern ABI: decimal32/64 in one GP reg, decimal128 as a
// two-GP-reg UINT128 struct.
// ---------------------------------------------------------------------------

static const char *dec_sfx(Type *ty) {
    return ty->kind == TY_DECIMAL32 ? "sd" : ty->kind == TY_DECIMAL64 ? "dd"
                                                                      : "td";
}

static const char *dec_op_name(int kind) {
    switch (kind) {
    case ND_ADD: return "__bid_add";
    case ND_SUB: return "__bid_sub";
    case ND_MUL: return "__bid_mul";
    default: return "__bid_div";
    }
}

static const char *dec_cmp_name(int kind) {
    switch (kind) {
    case ND_EQ: return "__bid_eq";
    case ND_NE: return "__bid_ne";
    case ND_LE: return "__bid_le";
    default: return "__bid_lt";
    }
}

// Move VReg `r` (a decimal32/64 BID value) into integer arg register N.
static void emit_dec_arg(VReg r, int n) {
#ifdef ARCH_ARM64
    static const Arm64Reg a64[] = {ARM64_X0, ARM64_X1, ARM64_X2, ARM64_X3, ARM64_X4};
    asm_mov_phy_reg(cg_sec, a64[n], r, 1); // mov xN, x{r}
#elif defined(_WIN32)
    static const X86Reg aw[] = {X86_RCX, X86_RDX, X86_R8, X86_R9};
    x86_mov_rr(cg_sec, 8, aw[n], cg_x86_reg[r]); // movq %r, %arg
#else
    static const X86Reg al[] = {X86_RDI, X86_RSI, X86_RDX, X86_RCX, X86_R8};
    x86_mov_rr(cg_sec, 8, al[n], cg_x86_reg[r]); // movq %r, %arg
#endif
}

// Load low/high 64-bit words of the 16-byte UINT128 at slot `addr` into
// integer arg registers n and n+1 (the two-GP-reg UINT128 convention).
static void emit_dec128_arg(VReg addr, int n) {
#ifdef ARCH_ARM64
    static const Arm64Reg a64[] = {ARM64_X0, ARM64_X1, ARM64_X2, ARM64_X3, ARM64_X4};
    arm64_ldr_uoff(cg_sec, 3, ARM64_X16, REG(addr), 0); // ldr x16, [addr]
    arm64_ldr_uoff(cg_sec, 3, ARM64_X17, REG(addr), 1); // ldr x17, [addr, #8]
    asm_mov_phy_phy(cg_sec, a64[n], ARM64_X16, 1); // mov xN, x16
    asm_mov_phy_phy(cg_sec, a64[n + 1], ARM64_X17, 1); // mov xN+1, x17
#elif defined(_WIN32)
    // Windows x64: 16-byte struct by value is passed by hidden pointer;
    // our wrapper's UINT128 args use the SysV-style 2-reg convention only
    // on SysV; on Win64 the wrapper must receive them by pointer instead.
    static const X86Reg aw[] = {X86_RCX, X86_RDX, X86_R8, X86_R9};
    x86_mov_rr(cg_sec, 8, aw[n], cg_x86_reg[addr]); // movq %addr, %arg (pointer)
#else
    static const X86Reg al[] = {X86_RDI, X86_RSI, X86_RDX, X86_RCX, X86_R8};
    x86_mov_rm(cg_sec, 8, al[n], x86_mem(cg_x86_reg[addr], 0)); // movq (addr), %arg
    x86_mov_rm(cg_sec, 8, al[n + 1], x86_mem(cg_x86_reg[addr], 8)); // movq 8(addr), %arg+1
#endif
}

// Store the two-GP-reg UINT128 return (rax:rdx / x0:x1) into the 16-byte
// slot whose address is in VReg `dst`.
static void emit_dec128_ret(VReg dst) {
#ifdef ARCH_ARM64
    arm64_str_uoff(cg_sec, 3, ARM64_X0, REG(dst), 0); // str x0, [dst]
    arm64_str_uoff(cg_sec, 3, ARM64_X1, REG(dst), 1); // str x1, [dst, #8]
#elif defined(_WIN32)
    asm_mov_rax_mem(cg_sec, dst); // movq %rax, (dst)
    asm_mov_rdx_mem8(cg_sec, dst); // movq %rdx, 8(dst)
#else
    asm_mov_rax_mem(cg_sec, dst); // movq %rax, (dst)
    asm_mov_rdx_mem8(cg_sec, dst); // movq %rdx, 8(dst)
#endif
}

static void emit_direct_call(char *name, bool is_asm_label);

// Call a __bid_* helper preserving live VRegs (same scheme as
// emit_bitint_call: push/pop r10/r11 on x86, per-reg spill on arm64).
static void emit_dec_call(char *fn) {
#ifdef ARCH_ARM64
    int saved = used_regs & ((1 << NUM_REGS) - 1);
    int arm_saved[NUM_REGS];
    bool arm_pushed[NUM_REGS];
    for (int i = 0; i < NUM_REGS; i++) {
        arm_saved[i] = -1;
        arm_pushed[i] = false;
    }
    for (int i = 0; i < NUM_REGS; i++) {
        if (saved & (1 << i)) {
            int d0 = spill_depth[i];
            int off = spill_offset(i);
            arm_pushed[i] = (spill_depth[i] > d0);
            arm_saved[i] = off;
            if (off <= 255)
                asm_stur_fp_phy(cg_sec, cg_arm_reg[i], off);
            else {
                VReg base = alloc_reg();
                asm_mov_imm(cg_sec, base, 8, off);
                asm_sub_reg_fp_reg(cg_sec, base, base, 8);
                asm_str_reg_off_phy(cg_sec, cg_arm_reg[i], base, 8, 0);
                free_reg(base);
            }
        }
    }
#else
    asm_push(cg_sec, X86_R10);
    asm_push(cg_sec, X86_R11);
#endif
    emit_direct_call(fn, false);
#ifdef ARCH_ARM64
    for (int i = 0; i < NUM_REGS; i++) {
        if (saved & (1 << i)) {
            int off = arm_saved[i];
            if (off <= 255)
                arm64_ldur(cg_sec, 1, cg_arm_reg[i], ARM64_X29, -off);
            else {
                VReg base = alloc_reg();
                asm_mov_imm(cg_sec, base, 8, off);
                asm_sub_reg_fp_reg(cg_sec, base, base, 8);
                VReg tmp = alloc_reg();
                asm_ldr_reg_off(cg_sec, tmp, base, 8, 0);
                asm_mov_phy_reg(cg_sec, cg_arm_reg[i], tmp, 1);
                free_reg(tmp);
                free_reg(base);
            }
            if (arm_pushed[i])
                pop_spill_slot(i);
        }
    }
#else
    asm_pop(cg_sec, X86_R11);
    asm_pop(cg_sec, X86_R10);
#endif
}

// Generate a _Decimal32/64/128 node. Returns a GP VReg holding the BID
// bits (sd/dd) or the address of a 16-byte slot (td).
static VReg gen_decimal(Node *node) {
    if (!node) return R_NONE;
    if (opt_g) emit_loc(node);

    Type *ty = node->ty;
    // For comparisons the node's own type is int; for casts to non-decimal
    // (int/float) the decimal type comes from the source operand. The
    // decimal type always comes from the operands.
    if (!is_decimal(ty)) {
        if (node->lhs && node->lhs->ty && is_decimal(node->lhs->ty))
            ty = node->lhs->ty;
        else if (node->rhs && node->rhs->ty && is_decimal(node->rhs->ty))
            ty = node->rhs->ty;
    }
    bool is128 = ty->kind == TY_DECIMAL128;
    const char *sfx = dec_sfx(ty);

    switch (node->kind) {

    case ND_NUM: {
        // Compile-time-folded BID bits from the parser.
        if (is128) {
            VReg addr = alloc_wide_addr(16);
#ifdef ARCH_ARM64
            asm_mov_imm_phy(cg_sec, ARM64_X16, 8, (uint64_t)node->val);
            asm_str_reg_off_phy(cg_sec, ARM64_X16, addr, 8, 0);
            asm_mov_imm_phy(cg_sec, ARM64_X16, 8, (uint64_t)node->val2);
            asm_str_reg_off_phy(cg_sec, ARM64_X16, addr, 8, 8);
#else
            asm_movabs_phy(cg_sec, X86_RAX, (uint64_t)node->val);
            asm_mov_rax_mem(cg_sec, addr); // movq %rax, (addr)
            asm_movabs_phy(cg_sec, X86_RAX, (uint64_t)node->val2);
            asm_movq_rax_mem8(cg_sec, addr); // movq %rax, 8(addr)
#endif
            return addr;
        }
        VReg r = alloc_reg();
#ifdef ARCH_ARM64
        asm_mov_imm(cg_sec, r, 8, node->val);
#else
        asm_movabs_phy(cg_sec, cg_x86_reg[r], (uint64_t)node->val);
#endif
        return r;
    }

    case ND_LVAR:
    case ND_MEMBER: {
        // decimal32/64: load the BID value from the lvalue address into a
        // GP reg. decimal128: keep the address (slot-based, like int128).
        if (!is128) {
            VReg addr = gen_addr(node);
            VReg r = alloc_reg();
#ifdef ARCH_ARM64
            asm_ldr_reg_off(cg_sec, r, addr, ty->size, 0); // ldr w/x{r}, [addr]
#else
            asm_mov_mem_reg(cg_sec, r, addr, ty->size); // movl/movq (addr), %r
#endif
            free_reg(addr);
            return r;
        }
        return gen_addr(node);
    }

    case ND_DEREF:
        return gen(node->lhs);

    case ND_COMMA:
    case ND_CHAIN: {
        int r = gen(node->lhs);
        if (r >= 0) free_reg(r);
        return gen_decimal(node->rhs);
    }

    case ND_ASSIGN: {
        VReg rhs = gen_decimal(node->rhs);
        VReg lhs = gen_addr(node->lhs);
        if (is128) {
            emit_bitint_copy_bytes(lhs, rhs, 16);
            free_reg(rhs);
            return lhs;
        }
#ifdef ARCH_ARM64
        asm_str_reg_off(cg_sec, rhs, lhs, ty->size, 0); // str w/x{rhs}, [x{lhs}]
#else
        asm_mov_reg_mem(cg_sec, rhs, lhs, ty->size); // movl/movq %rhs, (lhs)
#endif
        free_reg(rhs);
        return lhs;
    }
    case ND_CAST: {
        Type *from = node->lhs->ty;
        Type *to = node->ty;
        if (is_decimal(from) && is_decimal(to)) {
            // decimal -> decimal: extend or truncate.
            VReg src = gen_decimal(node->lhs);
            if (from->size == to->size) {
                return src; // same width: no-op
            }
            char fn[64];
            const char *fs = dec_sfx(from);
            const char *ts = dec_sfx(to);
            // width order: sd(4) < dd(8) < td(16)
            snprintf(fn, sizeof(fn), "__bid_%s%s%s2", from->size < to->size ? "extend" : "trunc",
                     fs, ts);
            if (from->size < to->size) {
                // extend sd->dd / sd->td / dd->td: source in one GP reg.
                if (to->kind == TY_DECIMAL128) {
                    // sd/dd -> td: result is 16 bytes (rax:rdx) -> slot.
                    VReg slot = alloc_wide_addr(16);
                    emit_dec_arg(src, 0);
                    emit_dec_call(fn);
                    emit_dec128_ret(slot);
                    free_reg(src);
                    return slot;
                }
                VReg dst = alloc_reg();
                emit_dec_arg(src, 0);
                emit_dec_call(fn);
#ifdef ARCH_ARM64
                asm_mov_phy_phy(cg_sec, cg_arm_reg[dst], ARM64_X0, 1);
#else
                x86_mov_rr(cg_sec, 8, cg_x86_reg[dst], X86_RAX);
#endif
                free_reg(src);
                return dst;
            } else {
                // trunc dd->sd / td->sd / td->dd: source 64-bit (dd) or
                // 128-bit slot (td). Route 128 through a slot.
                if (to->kind == TY_DECIMAL128) {
                    VReg slot = alloc_wide_addr(16);
                    emit_dec128_arg(src, 0);
                    emit_dec_call(fn);
                    emit_dec128_ret(slot);
                    free_reg(src);
                    return slot;
                }
                VReg dst = alloc_reg();
                if (from->kind == TY_DECIMAL128) {
                    // td -> sd/dd: load the slot's low word (the value's
                    // low 64 bits carry the BID payload for sd/dd targets
                    // after truncation). Use the trunc helper with slot.
                    VReg slot = alloc_wide_addr(16);
                    emit_bitint_copy_bytes(slot, src, 16);
                    free_reg(slot);
                    // placeholder: reuse low word
                    VReg lo = alloc_reg();
#ifdef ARCH_ARM64
                    asm_ldr_reg_off(cg_sec, lo, src, 8, 0);
                    asm_mov_phy_reg(cg_sec, cg_arm_reg[dst], cg_arm_reg[lo], 1);
#else
                    asm_mov_mem_reg(cg_sec, lo, src, 8);
                    x86_mov_rr(cg_sec, 8, cg_x86_reg[dst], cg_x86_reg[lo]);
#endif
                    free_reg(lo);
                } else {
                    // dd -> sd: trunc helper
                    emit_dec_arg(src, 0);
                    emit_dec_call(fn);
#ifdef ARCH_ARM64
                    asm_mov_phy_phy(cg_sec, cg_arm_reg[dst], ARM64_X0, 1);
#else
                    x86_mov_rr(cg_sec, 8, cg_x86_reg[dst], X86_RAX);
#endif
                }
                free_reg(src);
                return dst;
            }
        }
        // decimal <-> integer
        if (is_decimal(from) && is_integer(to)) {
            // decimal -> int: __bid_fix{uns}{sd,dd,td}{si,di,ti}
            char fn[64];
            const char *fs = dec_sfx(from);
            const char *ti = to->size == 8 ? "di" : "si";
            snprintf(fn, sizeof(fn), "__bid_fix%s%s%s", to->is_unsigned ? "uns" : "", fs, ti);
            VReg src = gen_decimal(node->lhs);
            if (from->kind == TY_DECIMAL128) {
                emit_dec128_arg(src, 0);
            } else {
                emit_dec_arg(src, 0);
            }
            free_reg(src);
            emit_dec_call(fn);
            VReg dst = alloc_reg();
#ifdef ARCH_ARM64
            asm_mov_phy_phy(cg_sec, cg_arm_reg[dst], ARM64_X0, 1);
#else
            x86_mov_rr(cg_sec, 8, cg_x86_reg[dst], X86_RAX);
#endif
            return dst;
        }
        if (is_integer(from) && is_decimal(to)) {
            // int -> decimal: __bid_float{,uns}{si,di,ti}{sd,dd,td}
            char fn[64];
            const char *ts = dec_sfx(to);
            const char *si = from->size == 8 ? "di" : "si";
            snprintf(fn, sizeof(fn), "__bid_float%s%s%s", from->is_unsigned ? "uns" : "", si, ts);
            VReg src = gen(node->lhs);
            emit_dec_arg(src, 0);
            free_reg(src);
            emit_dec_call(fn);
            if (to->kind == TY_DECIMAL128) {
                VReg slot = alloc_wide_addr(16);
                emit_dec128_ret(slot);
                return slot;
            }
            VReg dst = alloc_reg();
#ifdef ARCH_ARM64
            asm_mov_phy_phy(cg_sec, cg_arm_reg[dst], ARM64_X0, 1);
#else
            x86_mov_rr(cg_sec, 8, cg_x86_reg[dst], X86_RAX);
#endif
            return dst;
        }
        // decimal <-> binary float: the __bid_trunc* / __bid_extend*
        // helpers take/return the binary float in the FP register (XMM0
        // on x86 SysV), matching the platform float ABI — NOT a GP reg.
        if (is_decimal(from) && is_flonum(to)) {
            char fn[64];
            const char *fs = dec_sfx(from);
            const char *tf = to->size == 4 ? "sf" : "df";
            snprintf(fn, sizeof(fn), "__bid_trunc%s%s", fs, tf);
            VReg src = gen_decimal(node->lhs);
            if (from->kind == TY_DECIMAL128)
                emit_dec128_arg(src, 0);
            else
                emit_dec_arg(src, 0);
            free_reg(src);
            emit_dec_call(fn);
            VReg dst = alloc_reg();
#ifdef ARCH_ARM64
            asm_fmov_f2i(cg_sec, dst, 0, 1); // fmov x{dst}, d0
#else
            x86_movq_xmm_r(cg_sec, REG(dst), X86_XMM0); // movq %xmm0, %dst
#endif
            return dst;
        }
        if (is_flonum(from) && is_decimal(to)) {
            char fn[64];
            const char *fs = from->size == 4 ? "sf" : "df";
            const char *ts = dec_sfx(to);
            snprintf(fn, sizeof(fn), "__bid_extend%s%s", fs, ts);
            VReg src = gen(node->lhs);
#ifdef ARCH_ARM64
            asm_fmov_i2f(cg_sec, 0, src, 1); // fmov d0, x{src}
#else
            asm_movq_r_xmm(cg_sec, X86_XMM0, src); // movq %src, %xmm0
#endif
            free_reg(src);
            emit_dec_call(fn);
            if (to->kind == TY_DECIMAL128) {
                VReg slot = alloc_wide_addr(16);
                emit_dec128_ret(slot);
                return slot;
            }
            VReg dst = alloc_reg();
#ifdef ARCH_ARM64
            asm_mov_phy_phy(cg_sec, cg_arm_reg[dst], ARM64_X0, 1);
#else
            x86_mov_rr(cg_sec, 8, cg_x86_reg[dst], X86_RAX);
#endif
            return dst;
        }
        // decimal -> bool: nonzero BID bits are truthy (zero has all bits
        // clear; NaN/Inf have nonzero bits, hence truthy like C float).
        if (is_decimal(from) && to->kind == TY_BOOL) {
            VReg src = gen_decimal(node->lhs);
            VReg dst = alloc_reg();
#ifdef ARCH_ARM64
            asm_mov_phy_reg(cg_sec, cg_arm_reg[dst], cg_arm_reg[src], 1);
#else
            x86_mov_rr(cg_sec, 8, cg_x86_reg[dst], cg_x86_reg[src]);
#endif
            free_reg(src);
            return dst;
        }
        // Fallback: copy the raw bits (used for decimal32/64 identity
        // casts and casts to void-ish targets).
        {
            VReg src = gen_decimal(node->lhs);
            VReg dst = alloc_reg();
#ifdef ARCH_ARM64
            asm_mov_phy_reg(cg_sec, cg_arm_reg[dst], cg_arm_reg[src], 1);
#else
            x86_mov_rr(cg_sec, 8, cg_x86_reg[dst], cg_x86_reg[src]);
#endif
            free_reg(src);
            return dst;
        }
    }

    case ND_ADD:
    case ND_SUB:
    case ND_MUL:
    case ND_DIV: {
        char fn[64];
        snprintf(fn, sizeof(fn), "%s%s3", dec_op_name(node->kind), sfx);
        if (is128) {
            VReg a = gen_decimal(node->lhs); // 16-byte slot address
            VReg b = gen_decimal(node->rhs);
            VReg slot = alloc_wide_addr(16);
            emit_dec128_arg(a, 0);
            free_reg(a);
            emit_dec128_arg(b, 2);
            free_reg(b);
            emit_dec_call(fn);
            emit_dec128_ret(slot);
            return slot;
        }
        VReg a = gen_decimal(node->lhs);
        VReg b = gen_decimal(node->rhs);
        emit_dec_arg(a, 0);
        free_reg(a);
        emit_dec_arg(b, 1);
        free_reg(b);
        emit_dec_call(fn);
        VReg dst = alloc_reg();
#ifdef ARCH_ARM64
        asm_mov_phy_phy(cg_sec, cg_arm_reg[dst], ARM64_X0, 1);
#else
        x86_mov_rr(cg_sec, 8, cg_x86_reg[dst], X86_RAX);
#endif
        return dst;
    }

    case ND_EQ:
    case ND_NE:
    case ND_LT:
    case ND_LE: {
        char fn[64];
        snprintf(fn, sizeof(fn), "%s%s2", dec_cmp_name(node->kind), sfx);
        if (is128) {
            VReg a = gen_decimal(node->lhs);
            VReg b = gen_decimal(node->rhs);
            emit_dec128_arg(a, 0);
            free_reg(a);
            emit_dec128_arg(b, 2);
            free_reg(b);
            emit_dec_call(fn);
        } else {
            VReg a = gen_decimal(node->lhs);
            VReg b = gen_decimal(node->rhs);
            emit_dec_arg(a, 0);
            free_reg(a);
            emit_dec_arg(b, 1);
            free_reg(b);
            emit_dec_call(fn);
        }
        VReg dst = alloc_reg();
#ifdef ARCH_ARM64
        asm_mov_phy_phy(cg_sec, cg_arm_reg[dst], ARM64_X0, 1);
#else
        x86_mov_rr(cg_sec, 8, cg_x86_reg[dst], X86_RAX);
#endif
        return dst;
    }

    case ND_NEG: {
        // BID sign bit flip: bit 31 (sd) / 63 (dd) / 127 (td).
        if (is128) {
            VReg src = gen_decimal(node->lhs);
            VReg slot = alloc_wide_addr(16);
            emit_bitint_copy_bytes(slot, src, 16);
            free_reg(src);
            VReg hi = alloc_reg();
            VReg lo = alloc_reg();
#ifdef ARCH_ARM64
            asm_ldr_reg_off(cg_sec, hi, slot, 8, 8);
            asm_ldr_reg_off(cg_sec, lo, slot, 8, 0);
            asm_xor_imm(cg_sec, hi, 8, INT32_MIN); // flip bit 63 of high word
            asm_str_reg_off(cg_sec, hi, slot, 8, 8);
#else
            asm_mov_mem_rax(cg_sec, slot);
            asm_mov_mem8_rdx(cg_sec, slot);
            x86_xor_ri(cg_sec, 8, X86_RDX, INT32_MIN); // flip bit 63 of hi word
            asm_mov_rax_mem(cg_sec, slot);
            asm_mov_rdx_mem8(cg_sec, slot);
#endif
            free_reg(hi);
            free_reg(lo);
            return slot;
        }
        VReg src = gen_decimal(node->lhs);
        uint64_t sign_bit = ty->size == 4 ? ((uint64_t)1 << 31) : ((uint64_t)1 << 63);
#ifdef ARCH_ARM64
        arm64_eor_imm(cg_sec, 1, REG(src), REG(src), sign_bit); // eor x{src}, x{src}, #sign
#else
        asm_movabs_phy(cg_sec, X86_RAX, sign_bit);
        x86_xor_rr(cg_sec, 8, cg_x86_reg[src], X86_RAX); // xor %rax, %src
#endif
        return src;
    }

    default:
        // Unhandled decimal op: fall back to the generic path (likely a
        // diagnostic will surface; most scalar ops route here first).
        return gen_int128(node);
    }
}


// True if `fn`'s body contains a call to `fn` itself by name. A GNU
// extern-inline wrapper that does this is a fortify-style fallback (the
// body's self-call is meant to bind to the real libc symbol); emitting a
// local copy would shadow it and recurse forever (see the gnu_ext_inline
// handling in codegen()).
static bool fn_self_references(Function *fn) {
    if (!fn || !fn->body || !fn->name)
        return false;
    // walk all nodes for a call whose funcname/lhs resolves to fn->name
    // (a small recursive walker; ND_FUNCALL names are interned pointers)
    // [uses a simple iterative stack to avoid recursion depth issues]
    Node *stack[4096];
    int sp = 0;
    stack[sp++] = fn->body;
    while (sp > 0) {
        Node *n = stack[--sp];
        if (!n) continue;
        if (n->kind == ND_FUNCALL) {
            if (n->funcname == fn->name)
                return true;
            if (n->lhs && n->lhs->kind == ND_LVAR && n->lhs->var &&
                n->lhs->var->name == fn->name)
                return true;
        }
        if (sp + 8 >= 4096) break; // defensive; bodies this deep don't exist
        if (n->lhs) stack[sp++] = n->lhs;
        if (n->rhs) stack[sp++] = n->rhs;
        if (n->cond) stack[sp++] = n->cond;
        if (n->then) stack[sp++] = n->then;
        if (n->els) stack[sp++] = n->els;
        if (n->init) stack[sp++] = n->init;
        if (n->inc) stack[sp++] = n->inc;
        if (n->stmt_expr_result) stack[sp++] = n->stmt_expr_result;
        for (Node *c = n->body; c; c = c->next)
            stack[sp++] = c;
        for (Node *c = n->args; c; c = c->next)
            stack[sp++] = c;
    }
    return false;
}

static bool var_needs_got(LVar *var) {
    if (var->is_local) return false;
    if (var->is_static) return false;
    return opt_pic;
}
#ifdef _WIN32
static bool var_needs_refptr(LVar *var) {
    if (var->is_local || var->is_static || var->is_function || var->is_tls)
        return false;
    return var->is_extern;
}

static const char *cg_emit_refptr_slot(const char *sym_name) {
    const char *refptr = format(".refptr.%s", sym_name);
    if (objfile_find_sym(cg_obj, refptr) >= 0)
        return refptr;
    int sym_sidx = objfile_find_sym(cg_obj, sym_name);
    if (sym_sidx < 0)
        sym_sidx = objfile_add_sym(cg_obj, sym_name, SEC_UNDEF, 0, 0, SB_GLOBAL, ST_NOTYPE);
    SecBuf *prev_sec = cg_sec;
    cg_set_section(SEC_RODATA);
    size_t off = cg_sec->len;
    secbuf_emit64le(cg_sec, 0);
    objfile_add_sym(cg_obj, refptr, SEC_RODATA, off, 8, SB_LOCAL, ST_NOTYPE);
    objfile_add_reloc(cg_obj, SEC_RODATA, off, sym_sidx, R_X86_64_64, 0);
    cg_sec = prev_sec;
    return refptr;
}

#ifndef ARCH_ARM64
static size_t asm_mov_refptr_rip_reg(SecBuf *s, int r, const char *sym_name) {
    const char *refptr = cg_emit_refptr_slot(sym_name);
    size_t off = s->len;
    X86Mem m = {X86_RIP, X86_NOREG, 1, 0};
    x86_mov_rm(s, 8, REG(r), m);
    if (!cg_dry_run) {
        int sidx = objfile_find_sym(cg_obj, refptr);
        if (sidx < 0) sidx = objfile_add_sym(cg_obj, refptr, SEC_UNDEF, 0, 0, SB_LOCAL, ST_NOTYPE);
        objfile_add_reloc(cg_obj, SEC_TEXT, off + 3, sidx, R_X86_64_PC32, -4);
    }
    asm_record(ASM_MOV_RRBP, off, s->len - off, r, -1, -1, 8, 0, 0, refptr, 0, -1, false);
    return s->len - off;
}
#endif
#endif


#if 0
static char *func_asm_name(char *name) {
    for (TLItem *item = all_items; item; item = item->next) {
        if (item->kind == TL_FUNC && item->fn->name == name)
            return item->fn->asm_name ? item->fn->asm_name : item->fn->name;
    }
    return name;
}
#endif

// Mach-O symbols get a leading underscore
static const char *sym_name(const char *name) {
#if defined(__APPLE__)
    if (name[0] == '.' || name[0] == '/')
        return name;
    return format("_%s", name);
#else
    return name;
#endif
}

// glibc's fortify headers (bits/string_fortified.h, bits/unistd.h, ...)
// use `__REDIRECT`/`extern __inline` to map a plain name (readlinkat,
// stpcpy, memcpy, ...) to its `__*_chk` counterpart for the CALL, where
// the extra __builtin_object_size argument is synthesized. That redirect
// must NOT apply when the function's ADDRESS is taken: `&readlinkat` is
// the plain libc symbol (the _chk variant has a different 5-arg
// signature, so a function-pointer call through it reads a garbage
// buflen and glibc's fortify check aborts — diffutils' careadlinkat
// passes `readlinkat` as a function pointer). A C-level name that does
// not start with "__" and whose asm label ends in "_chk" is such a
// redirect: use the C name for address-of purposes.
static bool is_fortify_redirect(const char *cname, const char *asm_label) {
    if (!asm_label || !cname || (cname[0] == '_' && cname[1] == '_'))
        return false;
    size_t n = strlen(asm_label);
    return n > 4 && strcmp(asm_label + n - 4, "_chk") == 0;
}

// Assembly label for a variable: respects __asm__ names (used as-is) and
// applies sym_name() to regular C identifiers.
static const char *var_sym_label(LVar *var) {
    if (var->asm_name && !is_fortify_redirect(var->name, var->asm_name))
        return var->asm_name;
    if (is_asm_reserved(var->name)) return format(".L_rcc_%s", var->name);
    return sym_name(var->name);
}

static const char *asm_sym_name(const char *name);
// Assembly label for a function: respects __asm__ names (used as-is) and
// applies sym_name() to regular C identifiers.
static const char *func_label(char *name) {
    uint32_t h = func_hash_name(name) % FUNC_HASH_SIZE;
    for (TLItem *item = func_htab[h]; item; item = item->hash_next)
        if (item->fn->name == name) {
            if (item->fn->asm_name && !is_fortify_redirect(item->fn->name, item->fn->asm_name))
                return item->fn->asm_name;
            return asm_sym_name(sym_name(item->fn->name));
        }
    return asm_sym_name(sym_name(name));
}

// Assembly-safe symbol name: quotes non-ASCII identifiers for LLVM assembler
static const char *asm_sym_name(const char *name) {
#if defined(__APPLE__)
    const char *s = name;
    while (*s) {
        if ((unsigned char)*s < 0x20 || (unsigned char)*s > 0x7E)
            return format("\"%s\"", name);
        s++;
    }
#endif
    return name;
}

static void emit_direct_call(char *name, bool is_asm_label) {
    if (cg_dry_run) return;
    if (is_asm_reserved(name))
        name = format(".L_rcc_%s", name);
    const char *label = is_asm_label ? asm_sym_name(name) : func_label(name);
    size_t off = asm_call_label(cg_sec); // bl %s
    int sidx = objfile_find_sym(cg_obj, label);
#ifdef ARCH_ARM64
    if (sidx >= 0 && cg_obj->syms[sidx].section == SEC_TEXT) {
        // Same-section function: patch displacement directly, no relocation.
        int32_t disp = (int32_t)((int64_t)cg_obj->syms[sidx].offset - (int64_t)off);
        uint32_t insn = 0x94000000 | (((uint32_t)(disp / 4)) & 0x03FFFFFF);
        secbuf_patch32le(cg_sec, off, insn);
        return;
    }
    if (sidx < 0)
        sidx = objfile_add_sym(cg_obj, label, SEC_UNDEF, 0, 0, SB_GLOBAL, ST_FUNC);
    objfile_add_reloc(cg_obj, SEC_TEXT, off, sidx, R_AARCH64_CALL26, 0);
#else
    if (sidx >= 0 && cg_obj->syms[sidx].section == SEC_TEXT) {
        // Same-section function: patch displacement directly, no relocation.
        // This avoids blowing the COFF 16-bit per-section relocation limit
        // in files with many local calls (e.g. 101_cleanup).
        int32_t disp = (int32_t)((int64_t)cg_obj->syms[sidx].offset - (int64_t)(off + 5));
        secbuf_patch32le(cg_sec, off + 1, (uint32_t)disp);
        return;
    }
    if (sidx < 0)
        sidx = objfile_add_sym(cg_obj, label, SEC_UNDEF, 0, 0, SB_GLOBAL, ST_FUNC);
    // call label
    objfile_add_reloc(cg_obj, SEC_TEXT, off + 1, sidx, R_X86_64_PLT32, -4);
#endif
}

// Call a rcc_bitint_* helper, preserving every live VReg across the call.
// The helpers are ordinary functions: they clobber the caller-saved
// registers (r10-r15 on x86-64 SysV/Win64, x10-x15 on AAPCS64) that rcc's
// register allocator uses for its virtual registers. gen_funcall saves
// these around every call it emits; gen_bitint's helper calls bypass that
// marshalling, so they must do their own save/restore here or a live
// enclosing-expression value (e.g. a printf format-string address held in
// r10) is silently destroyed by the helper's own codegen. Save to each
// VReg's per-register spill slot; restore from the same slot after.
static void emit_bitint_call(char *fn) {
    // Preserve r10/r11 (VRegs 0/1) across the helper call UNCONDITIONALLY,
    // via plain stack push/pop — the one scratch area the register
    // allocator never touches for register values (frame slots collide:
    // the allocator owns that memory). The helpers are real functions that
    // clobber the caller-saved registers rcc's allocator uses for VRegs
    // 0/1; a raw emit_direct_call bypasses gen_funcall's saved_scratch
    // marshalling, so a hoisted enclosing value (e.g. a printf
    // format-string address loaded into r10 before this helper call) would
    // be silently destroyed. Two 8-byte pushes keep rsp 16-byte aligned
    // at the call (required by SysV), and the pops restore exactly. This
    // is the same push/pop pattern the inline memcmp/strlen/memcpy
    // expansions use around their raw instructions. rbx/r12-r15 (VRegs 2+)
    // are callee-saved and survive the call.
#ifdef ARCH_ARM64
    // AAPCS64: x10-x15 are all caller-saved (rcc uses them as VRegs 0-5).
    // Save every currently-live one (used_regs); use x16/x17 as scratch
    // for large offsets. Matches the x86 push/pop intent.
    int saved = used_regs & ((1 << NUM_REGS) - 1);
    int arm_saved[NUM_REGS];
    bool arm_pushed[NUM_REGS];
    for (int i = 0; i < NUM_REGS; i++) {
        arm_saved[i] = -1;
        arm_pushed[i] = false;
    }
    for (int i = 0; i < NUM_REGS; i++) {
        if (saved & (1 << i)) {
            int d0 = spill_depth[i];
            int off = spill_offset(i); // push when depth was 0
            arm_pushed[i] = (spill_depth[i] > d0);
            arm_saved[i] = off;
            if (off <= 255)
                asm_stur_fp_phy(cg_sec, cg_arm_reg[i], off); // str x{i}, [x29, #-off]
            else {
                // Load the slot address into a VReg base (arm64_str_uoff
                // expects a VReg, not a physical reg), then store through it.
                VReg base = alloc_reg();
                asm_mov_imm(cg_sec, base, 8, off); // mov x{base}, #off
                asm_sub_reg_fp_reg(cg_sec, base, base, 8); // sub x{base}, x29, x{base}
                asm_str_reg_off_phy(cg_sec, cg_arm_reg[i], base, 8, 0); // str x{i}, [x{base}]
                free_reg(base);
            }
        }
    }
#else
    asm_push(cg_sec, X86_R10); // pushq %r10
    asm_push(cg_sec, X86_R11); // pushq %r11
#endif
    emit_direct_call(fn, false);
#ifdef ARCH_ARM64
    for (int i = 0; i < NUM_REGS; i++) {
        if (saved & (1 << i)) {
            int off = arm_saved[i];
            if (off <= 255)
                arm64_ldur(cg_sec, 1, cg_arm_reg[i], ARM64_X29, -off); // ldr x{i}, [x29, #-off]
            else {
                VReg base = alloc_reg();
                asm_mov_imm(cg_sec, base, 8, off); // mov x{base}, #off
                asm_sub_reg_fp_reg(cg_sec, base, base, 8); // sub x{base}, x29, x{base}
                VReg tmp = alloc_reg();
                asm_ldr_reg_off(cg_sec, tmp, base, 8, 0); // ldr x{tmp}, [x{base}]
                asm_mov_phy_reg(cg_sec, cg_arm_reg[i], tmp, 1); // mov x{i}, x{tmp}
                free_reg(tmp);
                free_reg(base);
            }
            if (arm_pushed[i])
                pop_spill_slot(i); // keep the allocator's depth balanced
        }
    }
#else
    asm_pop(cg_sec, X86_R11); // popq %r11
    asm_pop(cg_sec, X86_R10); // popq %r10
#endif
}

static bool var_has_cleanup(LVar *var) {
    if (!var->is_local) return false;
    if (var->defer_stmt) return true;
    if (var->cleanup_func) return true;
    return var->ty->kind == TY_ARRAY && var->ty->base && var->ty->base->cleanup_func;
}

static void emit_cleanup_var(LVar *var) {
    if (var->cleanup_func) {
#ifdef ARCH_ARM64
        if (var->offset <= 4095)
            asm_add_fp_imm(cg_sec, ARM64_X0, -var->offset); // add x0, x29, #{-var->offset}
        else {
            int v = var->offset;
            emit_mov_imm64(ARM64_X16, v & 0xffff); // mov x16, #v
            v >>= 16;
            int s = 16;
            while (v) {
                asm_movk(cg_sec, ARM64_X16, 1, (uint16_t)(v & 0xffff), s); // movk x16, #v, lsl #s
                v >>= 16;
                s += 16;
            }
            asm_sub_x0_fp_x16(cg_sec); // sub x0, x29, x16
        }
#elif defined(_WIN32)
        asm_lea_rbp(cg_sec, X86_RCX, 8, var->offset); // lea -offset(%rbp), %rcx
#else
        asm_lea_rbp(cg_sec, X86_RDI, 8, var->offset); // lea -offset(%rbp), %rdi
#endif
        emit_direct_call(var->cleanup_func, false);
        return;
    }
    // Array whose element type carries __cleanup__: call per element, LIFO
    char *func = var->ty->base->cleanup_func;
    int elem_size = var->ty->base->size;
    int nelem = elem_size ? var->ty->size / elem_size : 0;
    for (int i = nelem - 1; i >= 0; i--) {
#ifdef ARCH_ARM64
        int off = var->offset - i * elem_size;
        if (off <= 4095)
            asm_add_fp_imm(cg_sec, ARM64_X0, -off);
        else {
            int v = off;
            emit_mov_imm64(ARM64_X16, v & 0xffff); // mov x16, #v
            v >>= 16;
            int s = 16;
            while (v) {
                asm_movk(cg_sec, 16, 1, (uint16_t)(v & 0xffff), s); // movk x16, #v, lsl #s
                v >>= 16;
                s += 16;
            }
            asm_sub_x0_fp_x16(cg_sec); // sub x0, x29, x16
        }
#elif defined(_WIN32)
        asm_lea_rbp(cg_sec, X86_RCX, 8, var->offset - i * elem_size); // lea [rbp-off], rcx
#else
        asm_lea_rbp(cg_sec, X86_RDI, 8, var->offset - i * elem_size); // lea [rbp-off], rdi
#endif
        emit_direct_call(func, false);
    }
}

static void emit_cleanup_range(LVar *begin, LVar *end) {
    for (LVar *var = begin; var && var != end; var = var->next) {
        if (!var->is_local) continue;
        if (var->defer_stmt) {
            // A `defer` body is arbitrary code, not a fixed call shape --
            // compile it in place (dispatches through the same gen()
            // every other statement uses). Re-emitted once per exit path
            // that reaches it (this range, plus a separate copy injected
            // at parse time for the enclosing scope's own fall-through --
            // see append_cleanup_range()), matching how VLA/cleanup_func
            // teardown already duplicates its own code per exit path
            // rather than outlining into a shared subroutine.
            VReg r = gen(var->defer_stmt);
            if (r != -1) free_reg(r);
            continue;
        }
        if (var_has_cleanup(var))
            emit_cleanup_var(var);
    }
}

#ifdef ARCH_ARM64
static int arm64_hfa_info(Type *ty, int *elem_size) {
    if (!ty)
        return -1;
    if (is_flonum(ty) && (ty->size == 4 || ty->size == 8)) {
        if (*elem_size == 0)
            *elem_size = ty->size;
        else if (*elem_size != ty->size)
            return -1;
        return 1;
    }
    if (ty->kind == TY_COMPLEX && ty->base && is_flonum(ty->base)) {
        int base_sz = ty->base->size;
        if (base_sz != 4 && base_sz != 8)
            return -1;
        if (*elem_size == 0)
            *elem_size = base_sz;
        else if (*elem_size != base_sz)
            return -1;
        return 2;
    }
    if (ty->kind != TY_STRUCT)
        return -1;
    int count = 0;
    for (Member *mem = ty->members; mem; mem = mem->next) {
        int sub = arm64_hfa_info(mem->ty, elem_size);
        if (sub <= 0)
            return -1;
        count += sub;
        if (count > 4)
            return -1;
    }
    return count > 0 ? count : -1;
}

static int arm64_hfa_count(Type *ty, int *elem_size) {
    int es = 0;
    int n = arm64_hfa_info(ty, &es);
    if (n <= 0) {
        if (elem_size)
            *elem_size = 0;
        return 0;
    }
    if (elem_size)
        *elem_size = es;
    return n;
}
#endif

#ifdef ARCH_ARM64
// ARM64: emit load from [fp, #-offset] into x16 (uses x17 for large offsets)
static void arm64_load_from_fp_minus(int offset, Arm64Reg dst) {
    if (offset <= 255)
        asm_ldur_fp_phy(cg_sec, dst, offset); // ldr dst, [x29, #-offset]
    else if (offset <= 4095) {
        asm_sub_fp_imm(cg_sec, ARM64_X17, offset); // sub x17, x29, #offset
        asm_ldr_reg(cg_sec, dst, ARM64_X17); // ldr dst, [x17]
    } else {
        unsigned v = offset;
        emit_mov_imm64(ARM64_X17, (uint64_t)v); // mov r17, #v
        asm_sub_fp_reg(cg_sec, ARM64_X17, ARM64_X17); // sub x17, x29, x17
        asm_ldr_reg(cg_sec, dst, ARM64_X17); // ldr dst, [x17]
    }
}

// ARM64: emit load from [base, #-offset] into dst (uses x17 for large
// offsets) - like arm64_load_from_fp_minus but with a caller-supplied
// base register instead of the fixed x29, for VLA-local references
// chain-walked into an ancestor frame (see gen()/gen_addr()'s
// TY_VLA ND_LVAR branches).
static void arm64_load_from_reg_minus(Arm64Reg base, int offset, Arm64Reg dst) {
    if (offset >= 0 && offset <= 255) {
        arm64_ldur(cg_sec, 1, dst, base, -offset); // ldur dst, [base, #-offset]
    } else if (offset >= 0 && offset <= 4095) {
        arm64_sub_imm(cg_sec, 1, ARM64_X17, base, offset, 0); // sub x17, base, #offset
        asm_ldr_reg(cg_sec, dst, ARM64_X17); // ldr dst, [x17]
    } else {
        unsigned v = (unsigned)offset;
        emit_mov_imm64(ARM64_X17, (uint64_t)v); // mov x17, #v
        arm64_sub_reg(cg_sec, 1, ARM64_X17, base, ARM64_X17, ARM64_LSL, 0); // sub x17, base, x17
        asm_ldr_reg(cg_sec, dst, ARM64_X17); // ldr dst, [x17]
    }
}

// ARM64: emit store src to [fp, #-offset] (uses x17 for large offsets)
static void arm64_store_to_fp_minus(int offset) {
    // src fixed to x16
    if (offset <= 255)
        asm_stur_fp_phy(cg_sec, ARM64_X16, offset); // str x16, [x29, #-offset]
    else if (offset <= 4095) {
        asm_sub_fp_imm(cg_sec, ARM64_X17, offset); // sub x17,x29,#offset
        asm_str_reg(cg_sec, ARM64_X16, ARM64_X17); // str x16, [x17]
    } else {
        unsigned v = offset;
        emit_mov_imm64(ARM64_X17, v & 0xffff); // mov x17, #v
        v >>= 16;
        int s = 16;
        while (v) {
            asm_movk(cg_sec, 17, 1, (uint16_t)(v & 0xffff), s); // movk x17, #v, lsl #s
            v >>= 16;
            s += 16;
        }
        asm_sub_fp_reg(cg_sec, ARM64_X17, ARM64_X17); // sub x17, x29, x17
        asm_str_reg(cg_sec, ARM64_X16, ARM64_X17); // str x16, [x17]
    }
}
#endif

// Restore SP past any VLAs leaving scope (enables VLA reuse on re-entry).
// Locals list is newest-first; the last VLA in range is the outermost one.
static void emit_vla_dealloc(LVar *begin, LVar *end) {
    LVar *outermost_vla = NULL;
    for (LVar *v = begin; v && v != end; v = v->next)
        if (v->ty->kind == TY_VLA || ((v->ty->kind == TY_STRUCT || v->ty->kind == TY_UNION) && v->ty->vla_len_expr))
            outermost_vla = v;
    if (outermost_vla) {
#ifdef ARCH_ARM64
        arm64_load_from_fp_minus(outermost_vla->offset, ARM64_X16);
        asm_mov_sp_reg(cg_sec, ARM64_X16); // mov sp, x16
#else
        asm_mov_rbp(cg_sec, X86_RSP, 8, outermost_vla->offset); // mov [rbp-outermost_vla->offse], X86_RSP
#endif
    }
}

/* Other platforms still have it. windows deprecated it.
   Use a unique name to avoid conflicts with CRT import stubs. */
static void emit_alloca(void) {
    size_t alloca_sym_start = cg_sec->len;
    // Local: the helper is emitted into every TU that uses alloca(), and
    // multiple TUs get linked together (e.g. bash: lib/glob/glob.o and
    // redir.o both define it); a strong *global* definition in each fails
    // the link with "multiple definition of `__rcc_alloca'". It is only ever
    // called from within the same TU (emit_direct_call), so a private
    // static copy per object is correct and avoids any cross-TU symbol.
    cg_def_fn_label("__rcc_alloca");
#ifdef ARCH_ARM64
    // alloca(size): x0=size → round up, sub sp, return new sp
    asm_add_self_imm(cg_sec, ARM64_X0, 15); // add x0, x0, #15
    asm_and_self_imm(cg_sec, ARM64_X0, (uint64_t)(int64_t)(-16)); // and x0, x0, #-16
    asm_sub_sp_sp_reg(cg_sec, ARM64_X0); // sub sp, sp, x0
    asm_mov_reg_sp(cg_sec, ARM64_X0); // mov x0, sp
    asm_ret(cg_sec); // ret
    cg_patch_fn_size("__rcc_alloca", alloca_sym_start);
#else
    // alloca via call: return addr was pushed; pop it, adjust rsp, push back
    asm_pop(cg_sec, X86_RDX); // popq %rdx  (save return addr)
#ifdef _WIN32
    x86_mov_rr(cg_sec, 8, X86_RAX, X86_RCX); // movq %rcx, %rax  (Windows: arg in rcx)
#else
    x86_mov_rr(cg_sec, 8, X86_RAX, X86_RDI); // movq %rdi, %rax  (Linux: arg in rdi)
#endif
    x86_add_ri(cg_sec, 8, X86_RAX, 15); // addq $15, %rax
    x86_and_ri(cg_sec, 8, X86_RAX, -16); // andq $-16, %rax
    {
        int c1 = ++rcc_label_count;
        int c2 = ++rcc_label_count;
        cg_def_label(format(".Lalloca1.%d", c1));
        x86_cmp_ri(cg_sec, 8, X86_RAX, 4096); // cmpq $4096, %rax
        size_t jb_off = asm_jcc_label(cg_sec, X86_B); // jb .Lalloca2
        // testq %rax, -4096(%rsp) — page-probe stack guard
        secbuf_emit8(cg_sec, 0x48); // REX.W
        secbuf_emit8(cg_sec, 0x85); // TEST r/m64, r64
        secbuf_emit8(cg_sec, 0x84); // ModRM: mod=10, reg=0(RAX), r/m=4(SIB)
        secbuf_emit8(cg_sec, 0x24); // SIB: base=RSP
        secbuf_emit32le(cg_sec, (uint32_t)-4096); // disp32 = -4096
        x86_sub_ri(cg_sec, 8, X86_RSP, 4096); // subq $4096, %rsp
        x86_sub_ri(cg_sec, 8, X86_RAX, 4096); // subq $4096, %rax
        size_t jmp1 = asm_jmp_label(cg_sec); // jmp .Lalloca1
        asm_fixup_add(cg_sec, jmp1, format(".Lalloca1.%d", c1), 0);
        cg_def_label(format(".Lalloca2.%d", c2));
        asm_fixup_add(cg_sec, jb_off, format(".Lalloca2.%d", c2), 1);
    }
    x86_sub_rr(cg_sec, 8, X86_RSP, X86_RAX); // subq %rax, %rsp
    x86_mov_rr(cg_sec, 8, X86_RAX, X86_RSP); // movq %rsp, %rax
    asm_push(cg_sec, X86_RDX); // pushq %rdx
    asm_ret(cg_sec); // ret
    cg_patch_fn_size("__rcc_alloca", alloca_sym_start);
#endif
}

bool va_arg_need_copy(Type *ty) {
    if (ty->size > 8 && ty->size <= 16) {
        if (ty->kind == TY_STRUCT || ty->kind == TY_UNION) {
            for (Member *m = ty->members; m; m = m->next) {
                if (is_flonum(m->ty))
                    return true;
            }
        }
    }
    return false;
}
// True if `ty` (a struct/union/array/scalar) contains no floating-point
// leaf anywhere in its (recursive) layout -- every scalar is
// integer/pointer-class. Used to decide SysV/AAPCS64 small-aggregate
// register return (see struct_returns_in_gp_regs below).
static bool type_is_all_integer(Type *ty) {
    if (!ty) return true;
    switch (ty->kind) {
    case TY_FLOAT:
    case TY_DOUBLE:
    case TY_LDOUBLE:
    case TY_COMPLEX:
        return false;
    case TY_ARRAY:
        return type_is_all_integer(ty->base);
    case TY_STRUCT:
    case TY_UNION:
        for (Member *m = ty->members; m; m = m->next)
            if (!type_is_all_integer(m->ty)) return false;
        return true;
    default:
        return true;
    }
}

// True if `ty` is a struct/union <=16 bytes, entirely INTEGER-class (no
// float/double/long-double/_Complex leaf anywhere) -- the common
// SysV/AAPCS64 shape (div_t, ldiv_t, lldiv_t, imaxdiv_t, and most small
// integer-only PODs) that the real ABI returns in two GP registers
// (RAX:RDX on x86-64, X0:X1 on ARM64) with NO hidden return pointer.
// Real bug: rcc unconditionally used a hidden return pointer for every
// struct/union return regardless of size, so calling any *external*
// (non-rcc-compiled) function with this return shape silently
// corrupted every argument after the phantom hidden-pointer arg and
// read garbage/zero out of the (unwritten) buffer instead of RAX:RDX —
// found via bash's `imaxdiv()`/`div()`/`ldiv()`/`lldiv()` calls in its
// own arithmetic evaluator (expr.c), which made every `%`/`/` inside
// `$(( ))` return 0 or a fixed garbage value. Anything else (>16 bytes,
// or containing a float/double/_Complex member anywhere) keeps using
// the pre-existing hidden-pointer path unchanged -- full SSE/HFA-class
// register return isn't implemented here.
static bool struct_returns_in_gp_regs(Type *ty) {
    return ty && (ty->kind == TY_STRUCT || ty->kind == TY_UNION) &&
        ty->size > 0 && ty->size <= 16 && type_is_all_integer(ty);
}

static VReg gen_funcall(Node *node, VReg hidden_ret_reg) {
    int nargs = 0;
    for (Node *arg = node->args; arg; arg = arg->next)
        nargs++;

    Node *argv_stk[64];
    VReg arg_regs_stk[64];
    int arg_sizes_stk[64];
    bool arg_is_float_stk[64];
    int arg_stage_stk[64]; // per-arg staging slot (0 = not staged)
#ifdef _WIN32
    int arg_gp_idx_stk[64];
    int arg_fp_idx_stk[64];
    int arg_stack_idx_stk[64];
#elif defined(ARCH_ARM64)
    int arg_gp_idx_stk[64];
    int arg_fp_idx_stk[64];
    int arg_stack_idx_stk[64];
    int arg_hfa_count_stk[64];
    int arg_hfa_elem_size_stk[64];
#else
    int arg_gp_idx_stk[64];
    int arg_fp_idx_stk[64];
    int arg_stack_idx_stk[64];
#endif

    Node **argv;
    VReg *arg_regs;
    int *arg_sizes;
    bool *arg_is_float;
    __attribute__((unused)) int *arg_stage;
#ifdef _WIN32
    __attribute__((unused)) int *arg_gp_idx;
    int *arg_fp_idx;
    __attribute__((unused)) int *arg_stack_idx;
#elif defined(ARCH_ARM64)
    int *arg_gp_idx;
    int *arg_fp_idx;
    int *arg_stack_idx;
    int *arg_hfa_count;
    int *arg_hfa_elem_size;
#else
    int *arg_gp_idx;
    int *arg_fp_idx;
    int *arg_stack_idx;
#endif

    if (nargs <= 64) {
        argv = argv_stk;
        arg_regs = arg_regs_stk;
        arg_sizes = arg_sizes_stk;
        arg_is_float = arg_is_float_stk;
        arg_stage = arg_stage_stk;
#ifdef _WIN32
        arg_gp_idx = arg_gp_idx_stk;
        arg_fp_idx = arg_fp_idx_stk;
        arg_stack_idx = arg_stack_idx_stk;
#elif defined(ARCH_ARM64)
        arg_gp_idx = arg_gp_idx_stk;
        arg_fp_idx = arg_fp_idx_stk;
        arg_stack_idx = arg_stack_idx_stk;
        arg_hfa_count = arg_hfa_count_stk;
        arg_hfa_elem_size = arg_hfa_elem_size_stk;
#else
        arg_gp_idx = arg_gp_idx_stk;
        arg_fp_idx = arg_fp_idx_stk;
        arg_stack_idx = arg_stack_idx_stk;
#endif
    } else {
        argv = arena_alloc(sizeof(Node *) * nargs);
        arg_regs = arena_alloc(sizeof(VReg) * nargs);
        arg_sizes = arena_alloc(sizeof(int) * nargs);
        arg_is_float = arena_alloc(sizeof(bool) * nargs);
        arg_stage = arena_alloc(sizeof(int) * nargs);
#ifdef _WIN32
        arg_gp_idx = arena_alloc(sizeof(int) * nargs);
        arg_fp_idx = arena_alloc(sizeof(int) * nargs);
        arg_stack_idx = arena_alloc(sizeof(int) * nargs);
#elif defined(ARCH_ARM64)
        arg_gp_idx = arena_alloc(sizeof(int) * nargs);
        arg_fp_idx = arena_alloc(sizeof(int) * nargs);
        arg_stack_idx = arena_alloc(sizeof(int) * nargs);
        arg_hfa_count = arena_alloc(sizeof(int) * nargs);
        arg_hfa_elem_size = arena_alloc(sizeof(int) * nargs);
#else
        arg_gp_idx = arena_alloc(sizeof(int) * nargs);
        arg_fp_idx = arena_alloc(sizeof(int) * nargs);
        arg_stack_idx = arena_alloc(sizeof(int) * nargs);
#endif
    }

    int idx = 0;
    for (Node *arg = node->args; arg; arg = arg->next)
        argv[idx++] = arg;

    char *call_target = node->funcname;
    bool is_asm_call = false;
    if (!call_target && node->lhs && node->lhs->var && node->lhs->var->is_function)
        call_target = node->lhs->var->name;
    // __asm__("sym") on a declaration (glibc __REDIRECT): call the renamed symbol.
    // asm_name already has the Mach-O prefix; skip sym_name() at call site.
    if (node->lhs && node->lhs->var && node->lhs->var->is_function &&
        node->lhs->var->asm_name) {
        call_target = node->lhs->var->asm_name;
        is_asm_call = true;
    } else if (call_target) {
        // A funcname-only call (e.g. a clone spliced in by the inliner, or
        // an implicit call): the target's declaration may carry an
        // __asm__("sym") rename (mingw's `void __mingw_chk_fail_warn(void)
        // __asm__("__chk_fail")`). Without this the call references the
        // literal name, which has no symbol of its own.
        LVar *g = find_global_name(call_target);
        if (g && g->is_function && g->asm_name) {
            call_target = g->asm_name;
            is_asm_call = true;
        }
    }
    // glibc __REDIRECT: extern readlink(...) __asm__("__readlink_chk")
    // adds a trailing buflen arg.  The call site has the right arity
    // for the C-level name, so use that instead of the asm label -- but
    // only when the C-level name is itself a real, directly-linkable
    // public symbol (glibc's convention: never starts with "__"). Names
    // like __poll_chk_warn/__vsnprintf_chk_warn are themselves internal
    // _chk-suffixed redirect-only declarations whose asm_name (e.g.
    // __poll_chk) is the ONLY real symbol -- also ends in "_chk", so this
    // heuristic used to fire for them too and discarded the correct
    // asm_name in favor of the literal (unlinkable) C name, producing an
    // undefined-symbol link error. Two glibc __REDIRECT chains sharing a
    // "_chk" suffix on both ends must be told apart by more than just the
    // suffix.
    // (Bare _chk calls from inlined fortify bodies are already rewritten
    //  to the base function by the preprocessor.)
    if (is_asm_call && call_target && node->lhs->var->name &&
        !(node->lhs->var->name[0] == '_' && node->lhs->var->name[1] == '_')) {
        size_t tlen = strlen(call_target);
        if (tlen > 4 && !strcmp(call_target + tlen - 4, "_chk"))
            call_target = node->lhs->var->name;
    }

    // Rename __builtin_* math functions to their library counterparts.
    // The type signature was set by declare_builtin_on_demand() in the parser.
    bool is_builtin_call = (call_target && !strncmp(call_target, "__builtin_", 10));
    if (is_builtin_call) {
        if (!strcmp(call_target, "__builtin_powf")) call_target = "powf";
        else if (!strcmp(call_target, "__builtin_pow"))
            call_target = "pow";
        else if (!strcmp(call_target, "__builtin_fmaxf"))
            call_target = "fmaxf";
        else if (!strcmp(call_target, "__builtin_fmax"))
            call_target = "fmax";
        else if (!strcmp(call_target, "__builtin_fminf"))
            call_target = "fminf";
        else if (!strcmp(call_target, "__builtin_fmin"))
            call_target = "fmin";
        else if (!strcmp(call_target, "__builtin_fmaf"))
            call_target = "fmaf";
        else if (!strcmp(call_target, "__builtin_fma"))
            call_target = "fma";
        else if (!strcmp(call_target, "__builtin_fabsf"))
            call_target = "fabsf";
        else if (!strcmp(call_target, "__builtin_fabs"))
            call_target = "fabs";
        else if (!strcmp(call_target, "__builtin_copysignf"))
            call_target = "copysignf";
        else if (!strcmp(call_target, "__builtin_copysign"))
            call_target = "copysign";
        else if (!strcmp(call_target, "__builtin_creal"))
            call_target = "creal";
        else if (!strcmp(call_target, "__builtin_crealf"))
            call_target = "crealf";
        else if (!strcmp(call_target, "__builtin_cimag"))
            call_target = "cimag";
        else if (!strcmp(call_target, "__builtin_cimagf"))
            call_target = "cimagf";
    }

    // __builtin_ia32_* SIMD intrinsics (SSE/SSE2/SSSE3/SSE4.x): the
    // real GCC headers wrap every intrinsic in one of these calls, so
    // dispatch them all through gen_ia32_builtin (the old sqrt*/pshufb
    // special cases are a subset of it).
    if (call_target && !strncmp(call_target, "__builtin_ia32_", 15)) {
#ifdef ARCH_ARM64
        if (!strcmp(call_target, "__builtin_ia32_sqrtps") ||
            !strcmp(call_target, "__builtin_ia32_sqrtss") ||
            !strcmp(call_target, "__builtin_ia32_rsqrtps") ||
            !strcmp(call_target, "__builtin_ia32_sqrtpd") ||
            !strcmp(call_target, "__builtin_ia32_sqrtsd"))
            return gen_vector_unary_builtin(node);
        if (!strcmp(call_target, "__builtin_ia32_pshufb128"))
            return gen_vector_binary_builtin(node);
        error_tok(node->tok, "__builtin_ia32_%s: not implemented on this target",
                  call_target + 15);
        return R_NONE;
#else
        return gen_ia32_builtin(node);
#endif
    }
    // Check for __attribute__((warning/error/diagnose_if)) on function
    if (!cg_dry_run && node->lhs && node->lhs->var) {
        LVar *fn_var = node->lhs->var;
        if (fn_var->diag_error)
            error_tok(node->tok, "call to '%s': %s", fn_var->name, fn_var->diag_error);
        if (fn_var->diag_warning)
            warn_tok(node->tok, "call to '%s': %s", fn_var->name, fn_var->diag_warning);
        // diagnose_if entries are parsed but not emitted (per-call-site
        // evaluation not yet implemented; condition references params)
    }
    if (call_target && is_asm_reserved(call_target))
        call_target = format(".L_rcc_%s", call_target);
    init_local_builtins();
    // alloca() takes exactly one argument. Every specialized alloca
    // codegen path below unconditionally reads node->args (the size
    // expression) assuming it exists; calling alloca() with zero
    // arguments (a plain, unprototyped implicit-declaration call --
    // alloca needs no #include to get this special codegen, unlike an
    // ordinary function, so there's no prototype-based arg-count check
    // to catch it earlier) left `gen(node->args)` reading a NULL Node,
    // producing an internal "Invalid register -1" crash deep in codegen
    // instead of a real diagnostic. Reject it here instead, matching
    // real GCC's "too few arguments to function 'alloca'; expected 1,
    // have 0" (GCC treats alloca as a builtin with a known prototype
    // even without any declaration in scope).
    if (call_target && call_target == bi_s_alloca && nargs != 1)
        error_tok(node->tok, "too %s arguments to function 'alloca'; expected 1, have %d",
                  nargs < 1 ? "few" : "many", nargs);
    // __builtin_unreachable(): reaching it is undefined behaviour, so emit no
    // code (GCC does the same). Statements following it in a block are dead and
    // are elided at -O1 (see the ND_BLOCK loop).
    if (call_target && call_target == bi_unreachable)
        return -1;
    /*
    if (0 && opt_O1 && call_target && nargs >= 2) {
        if ((strcmp(call_target, "__printf_chk") == 0 && nargs >= 2) ||
            (strcmp(call_target, "__vprintf_chk") == 0 && nargs == 3)) {
            node->args = argv[1];
            call_target = call_target[2] == 'p' ? bi_s_printf : bi_s_vprintf;
            for (int j = 0; j < nargs - 1; j++)
                argv[j] = argv[j + 1];
            nargs--;
        } else if ((strcmp(call_target, "__fprintf_chk") == 0 && nargs >= 3) ||
                   (strcmp(call_target, "__vfprintf_chk") == 0 && nargs == 4)) {
            Node *flag = argv[1];
            argv[0]->next = flag->next;
            flag->next = NULL;
            call_target = call_target[2] == 'f' ? bi_s_fprintf : bi_s_vfprintf;
            for (int j = 1; j < nargs - 1; j++)
                argv[j] = argv[j + 1];
            nargs--;
            if (node->args == flag)
                node->args = argv[0];
        }
    }
    */

    // Optimize // Optimize hint; // %s
    //          (void)fprintf(fp, "%s", arg) → fputs(arg, fp)
    // puts/fputs return values differ from printf/fprintf, so only when unused.
    bool discard_result = cg_discard_result;
    cg_discard_result = false;
    if (discard_result && opt_O1 && call_target &&
        !(node->ty && (node->ty->kind == TY_STRUCT || node->ty->kind == TY_UNION))) {
        if (nargs == 2 && call_target == bi_s_printf) {
            Node *fmt = node->args;
            if (fmt && fmt->kind == ND_STR) {
                for (StrLit *s = all_strs; s; s = s->next) {
                    if (s->id == fmt->str_id && s->prefix == 0 && strcmp(s->str, "%s\n") == 0) {
                        call_target = bi_s_puts;
                        node->args = fmt->next;
                        nargs = 1;
                        argv[0] = node->args;
                        break;
                    }
                }
            }
        } else if (nargs == 3 && call_target == bi_s_fprintf) {
            Node *fp_arg = node->args;
            Node *fmt = fp_arg ? fp_arg->next : NULL;
            if (fmt && fmt->kind == ND_STR) {
                for (StrLit *s = all_strs; s; s = s->next) {
                    if (s->id == fmt->str_id && s->prefix == 0 && strcmp(s->str, "%s") == 0) {
                        call_target = bi_s_fputs;
                        Node *str_arg = fmt->next;
                        node->args = str_arg;
                        str_arg->next = fp_arg;
                        fp_arg->next = NULL;
                        nargs = 2;
                        argv[0] = node->args;
                        argv[1] = fp_arg;
                        break;
                    }
                }
            }
        }
    }

#ifdef _WIN32
    // codeql[cpp/commented-out-code]: AT&T mnemonic legend for cg_x86_argreg below, kept for reference
    //char *argreg32[] = {"%ecx", "%edx", "%r8d", "%r9d"};
    //char *argreg64[] = {"%rcx", "%rdx", "%r8", "%r9"};
    X86Reg cg_x86_argreg[] = {X86_RCX, X86_RDX, X86_R8, X86_R9};
    // codeql[cpp/commented-out-code]: legend for the Win64 XMM arg registers, kept for reference
    //char *argxmm[] = {"%xmm0", "%xmm1", "%xmm2", "%xmm3"};
    int shadow_space = 32;
    int max_gp_args = 4;
#elif defined(ARCH_ARM64)
    // AAPCS64: 8 GP arg regs (x0-x7), 8 SIMD/FP arg regs (v0-v7)
    // x8 is the indirect result register for struct returns
    // codeql[cpp/commented-out-code]: AAPCS64 register-name legend for the arg-passing scheme described above
    //char *argreg32[] = {"w0", "w1", "w2", "w3", "w4", "w5", "w6", "w7"};
    //char *argreg64[] = {"x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7"};
    //char *argxmm[] = {"d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7"};
    int max_gp_args = 8;
    int max_fp_args = 8;
    // codeql[cpp/commented-out-code]: documents that AAPCS64 needs no shadow space, unlike Win64 above
    //int shadow_space = 0;
#else
    // codeql[cpp/commented-out-code]: AT&T mnemonic legend for cg_x86_argreg below, kept for reference
    //char *argreg32[] = {"%edi", "%esi", "%edx", "%ecx", "%r8d", "%r9d"};
    //char *argreg64[] = {"%rdi", "%rsi", "%rdx", "%rcx", "%r8", "%r9"};
    X86Reg cg_x86_argreg[] = {X86_RDI, X86_RSI, X86_RDX, X86_RCX, X86_R8, X86_R9};
    // codeql[cpp/commented-out-code]: legend for the SysV XMM arg registers, kept for reference
    //char *argxmm[] = {"%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7"};
    int max_gp_args = 6;
    int max_fp_args = 8;
    int shadow_space = 0;
#endif

    bool has_hidden_retbuf = node->ty &&
        (
#ifdef _WIN32
                                 ((node->ty->kind == TY_STRUCT || node->ty->kind == TY_UNION) && node->ty->size > 8) || (node->ty->kind == TY_COMPLEX && node->ty->size > 8) || (node->ty->kind == TY_BITINT && node->ty->size > 16)
#elif defined(ARCH_ARM64)
                                 ((node->ty->kind == TY_STRUCT || node->ty->kind == TY_UNION) && !struct_returns_in_gp_regs(node->ty)) || node->ty->kind == TY_COMPLEX || (node->ty->kind == TY_BITINT && node->ty->size > 16)
#else
                                 ((node->ty->kind == TY_STRUCT || node->ty->kind == TY_UNION) && !struct_returns_in_gp_regs(node->ty)) || (node->ty->kind == TY_BITINT && node->ty->size > 16)
#endif
        );

    // Cross-architecture builtins (x86_64 and arm64)
    if (call_target && !has_hidden_retbuf) {
        VReg bi_result = gen_builtin_call(node, call_target, gen);
        if (bi_result != R_NONE)
            return bi_result;
    }

    // Inline expansion for common libc builtins (x86_64 only for now)
#ifndef ARCH_ARM64
    if (opt_O1 && call_target && !has_hidden_retbuf) {
        // Inline expansion for common libc builtins
        bool is_memset = call_target == bi_s_memset || call_target == bi_memset;
        bool is_memcpy = call_target == bi_s_memcpy || call_target == bi_memcpy;
        bool is_memcmp = call_target == bi_s_memcmp || call_target == bi_memcmp;
        bool is_strlen = call_target == bi_s_strlen || call_target == bi_strlen;
        bool is_strcmp = call_target == bi_s_strcmp || call_target == bi_strcmp;
        bool is_strchr = call_target == bi_s_strchr || call_target == bi_strchr;

        if (is_memset || is_memcpy) {
            Node *dst = node->args;
            Node *v2 = dst ? dst->next : NULL;
            Node *len = v2 ? v2->next : NULL;
            if (dst && v2 && len && !len->next && free_reg_count() >= 4) {
                VReg r = alloc_reg();
                VReg dst_r = gen(dst);
                VReg v2_r = gen(v2);
                VReg len_r = gen(len);
                asm_mov_reg_reg(cg_sec, r, dst_r, 8); // mov rdst_r -> rr (return dst)

                asm_cld(cg_sec); // cld
                asm_push(cg_sec, X86_RDI); // pushq %rdi
                if (is_memcpy) asm_push(cg_sec, X86_RSI); // pushq %rsi
                asm_push(cg_sec, X86_RCX); // pushq %rcx
                x86_mov_rr(cg_sec, 8, X86_RDI, REG(dst_r));
                x86_mov_rr(cg_sec, 8, X86_RCX, REG(len_r));
                if (is_memset) {
                    x86_movzx(cg_sec, 4, 1, X86_RAX, REG(v2_r));
                    x86_rep_prefix(cg_sec);
                    x86_stosb(cg_sec); // rep stosb
                } else {
                    x86_mov_rr(cg_sec, 8, X86_RSI, REG(v2_r));
                    x86_rep_prefix(cg_sec);
                    x86_movsb(cg_sec); // rep movsb
                }
                asm_pop(cg_sec, X86_RCX); // popq %rdi
                if (is_memcpy) asm_pop(cg_sec, X86_RSI); // pop rX86_RSI
                asm_pop(cg_sec, X86_RDI); // cld

                free_reg(dst_r);
                free_reg(v2_r);
                free_reg(len_r);

                return r;
            }
        }

        if (is_memcmp) {
            Node *src1 = node->args;
            Node *src2 = src1 ? src1->next : NULL;
            Node *len = src2 ? src2->next : NULL;
            if (src1 && src2 && len && !len->next && free_reg_count() >= 3) {
                VReg s1_r = gen(src1);
                VReg s2_r = gen(src2);
                VReg len_r = gen(len);

                asm_cld(cg_sec); // cld
                asm_push(cg_sec, X86_RDI); // pushq %rdi
                asm_push(cg_sec, X86_RSI); // pushq %rsi
                asm_push(cg_sec, X86_RCX); // pushq %rcx
                int cl = ++rcc_label_count;
                x86_mov_rr(cg_sec, 8, X86_RDI, REG(s1_r));
                x86_mov_rr(cg_sec, 8, X86_RSI, REG(s2_r));
                x86_mov_rr(cg_sec, 8, X86_RCX, REG(len_r));
                // Set ZF=1 before repz cmpsb: with rcx=0 the repz is a
                // no-op that leaves flags untouched, making the jne below
                // branch on garbage.  Observed as Lua empty-string
                // memcmp("", "", 0) returning non-zero with rcc -O2.
                x86_xor_rr(cg_sec, 4, X86_RAX, X86_RAX); // ZF=1
                x86_rep_prefix(cg_sec);
                x86_cmpsb(cg_sec); // repe cmpsb
                {
                    size_t o = asm_jcc_label(cg_sec, X86_NE);
                    asm_fixup_add(cg_sec, o, format(".L.memcmp_diff.%d", cl), 1);
                }
                x86_xor_rr(cg_sec, 4, X86_RAX, X86_RAX); // xorl %eax, %eax (equal)
                {
                    size_t o = asm_jmp_label(cg_sec);
                    asm_fixup_add(cg_sec, o, format(".L.memcmp_end.%d", cl), 0);
                }
                cg_def_label(format(".L.memcmp_diff.%d", cl));
                x86_movzx_rm(cg_sec, 4, 1, X86_RAX, x86_mem(X86_RDI, -1)); // movsbl -1(%rdi),%eax
                x86_movzx_rm(cg_sec, 4, 1, X86_RCX, x86_mem(X86_RSI, -1)); // movsbl -1(%rsi),%ecx
                x86_sub_rr(cg_sec, 4, X86_RAX, X86_RCX); // subl %ecx, %eax
                cg_def_label(format(".L.memcmp_end.%d", cl));
                asm_pop(cg_sec, X86_RCX);
                asm_pop(cg_sec, X86_RSI);
                asm_pop(cg_sec, X86_RDI);
                free_reg(s1_r);
                free_reg(s2_r);
                free_reg(len_r);
                int r = alloc_reg();
                asm_mov_retval(cg_sec, r, 4); // movl %eax, rr
                return r;
            }
        }

        if (is_strlen) {
            Node *str = node->args;
            if (str && !str->next && free_reg_count() >= 1) {
                VReg str_r = gen(str);

                asm_cld(cg_sec); // cld
                asm_push(cg_sec, X86_RDI); // pushq %rdi
                asm_push(cg_sec, X86_RCX); // pushq %rcx
                x86_mov_rr(cg_sec, 8, X86_RDI, REG(str_r));
                x86_xor_rr(cg_sec, 1, X86_RAX, X86_RAX); // xorb %%al, %%al
                x86_mov_ri(cg_sec, 8, X86_RCX, -1); // movq $-1, %rcx
                x86_repne_prefix(cg_sec);
                x86_scasb(cg_sec); // repne scasb
                x86_not_r(cg_sec, 8, X86_RCX); // notq %rcx
                x86_sub_ri(cg_sec, 8, X86_RCX, 1); // decq %rcx
                x86_mov_rr(cg_sec, 8, X86_RAX, X86_RCX);
                asm_pop(cg_sec, X86_RCX); // popq %rcx
                asm_pop(cg_sec, X86_RDI); // popq %rdi

                free_reg(str_r);

                int r = alloc_reg();
                asm_mov_retval(cg_sec, r, 8); // movq %rax, rr
                return r;
            }
        }

        if (is_strcmp) {
            Node *s1 = node->args;
            Node *s2 = s1 ? s1->next : NULL;
            if (s1 && s2 && !s2->next && free_reg_count() >= 2) {
                VReg r = gen(s1);
                VReg r2 = gen(s2);
                int cl = ++rcc_label_count;
                asm_push(cg_sec, X86_RDI); // pushq %rdi
                asm_push(cg_sec, X86_RSI); // pushq %rsi
                x86_mov_rr(cg_sec, 8, X86_RDI, REG(r));
                x86_mov_rr(cg_sec, 8, X86_RSI, REG(r2));
                cg_def_label(format(".L.strcmp_loop.%d", cl));
                x86_movzx_rm(cg_sec, 4, 1, X86_RAX, x86_mem(X86_RDI, 0)); // movb (%rdi),%%al
                x86_cmp_rm(cg_sec, 1, X86_RAX, x86_mem(X86_RSI, 0)); // cmpb (%rsi),%%al
                {
                    size_t o = asm_jcc_label(cg_sec, X86_NE);
                    asm_fixup_add(cg_sec, o, format(".L.strcmp_diff.%d", cl), 1);
                }
                x86_test_rr(cg_sec, 1, X86_RAX, X86_RAX); // testb %%al,%%al
                {
                    size_t o = asm_jcc_label(cg_sec, X86_Z);
                    asm_fixup_add(cg_sec, o, format(".L.strcmp_eq.%d", cl), 1);
                }
                x86_inc_r(cg_sec, 8, X86_RDI); // incq %rdi
                x86_inc_r(cg_sec, 8, X86_RSI); // incq %rsi
                {
                    size_t o = asm_jmp_label(cg_sec);
                    asm_fixup_add(cg_sec, o, format(".L.strcmp_loop.%d", cl), 0);
                }
                cg_def_label(format(".L.strcmp_diff.%d", cl));
                x86_movzx(cg_sec, 4, 1, X86_RAX, X86_RAX); // movzbl %%al,%eax
                x86_movzx_rm(cg_sec, 4, 1, X86_RCX, x86_mem(X86_RSI, 0)); // movzbl (%rsi),%ecx
                x86_sub_rr(cg_sec, 4, X86_RAX, X86_RCX); // subl %ecx,%eax
                {
                    size_t o = asm_jmp_label(cg_sec);
                    asm_fixup_add(cg_sec, o, format(".L.strcmp_end.%d", cl), 0);
                }
                cg_def_label(format(".L.strcmp_eq.%d", cl));
                x86_xor_rr(cg_sec, 4, X86_RAX, X86_RAX); // xorl %eax,%eax
                cg_def_label(format(".L.strcmp_end.%d", cl));
                asm_pop(cg_sec, X86_RSI); // popq %rsi
                asm_pop(cg_sec, X86_RDI); // popq %rdi
                free_reg(r);
                free_reg(r2);
                int ret = alloc_reg();
                asm_mov_retval(cg_sec, ret, 4); // movl %eax, rret
                return ret;
            }
        }

        if (is_strchr) {
            Node *s = node->args;
            Node *c = s ? s->next : NULL;
            if (s && c && !c->next && free_reg_count() >= 2) {
                VReg sr = gen(s);
                VReg cr = gen(c);
                int cl = ++rcc_label_count;
                asm_push(cg_sec, X86_RDI); // pushq %rdi
                asm_push(cg_sec, X86_RCX); // pushq %rcx
                x86_mov_rr(cg_sec, 8, X86_RDI, REG(sr));
                x86_movzx(cg_sec, 4, 1, X86_RAX, REG(cr)); // movzbl %s, %eax
                cg_def_label(format(".L.strchr.%d", cl)); // .L.strchr_loop.%d:
                x86_cmp_rm(cg_sec, 1, X86_RAX, x86_mem(X86_RDI, 0)); // cmpb %%al, (%rdi)
                {
                    size_t o = asm_jcc_label(cg_sec, X86_E);
                    asm_fixup_add(cg_sec, o, format(".L.strchr_end.%d", cl), 1);
                }
                x86_cmp_mi(cg_sec, 1, x86_mem(X86_RDI, 0), 0); // cmpb $0, (%rdi)
                {
                    size_t o = asm_jcc_label(cg_sec, X86_E);
                    asm_fixup_add(cg_sec, o, format(".L.strchr_ret.%d", cl), 1);
                }
                x86_inc_r(cg_sec, 8, X86_RDI); // incq %rdi
                {
                    size_t o = asm_jmp_label(cg_sec);
                    asm_fixup_add(cg_sec, o, format(".L.strchr.%d", cl), 0);
                }
                cg_def_label(format(".L.strchr_end.%d", cl)); // .L.strchr_found.%d:
                x86_mov_rr(cg_sec, 8, X86_RAX, X86_RDI); // movq %rdi, %rax
                {
                    size_t o = asm_jmp_label(cg_sec);
                    asm_fixup_add(cg_sec, o, format(".L.strchr_done.%d", cl), 0);
                }
                cg_def_label(format(".L.strchr_ret.%d", cl)); // .L.strchr_null.%d:
                x86_xor_rr(cg_sec, 4, X86_RAX, X86_RAX); // xorl %eax, %eax
                cg_def_label(format(".L.strchr_done.%d", cl)); // .L.strchr_end.%d:
                asm_pop(cg_sec, X86_RCX); // popq %rcx
                asm_pop(cg_sec, X86_RDI); // popq %rdi
                free_reg(sr);
                free_reg(cr);
                int ret = alloc_reg();
                asm_mov_retval(cg_sec, ret, 8); // movq %rax, rret
                return ret;
            }
        }
    }
#endif /* !ARCH_ARM64 — inline builtins */

#ifdef ARCH_ARM64
    // Inline alloca: directly adjust sp without any register save/restore
    // (save/restore around a bl __rcc_alloca would use the stack, which alloca moves)
    if (call_target && call_target == bi_s_alloca) {
        fn_uses_alloca = true; // no alloca_needed: inlined, helper not emitted
        VReg ra = gen(node->args);
        // Round up to 16 bytes and adjust sp
        asm_add_imm(cg_sec, ra, 8, 15); // add ra, ra, #15
        asm_and_imm(cg_sec, ra, 8, -16); // and ra, ra, #-16
        asm_sub_sp_sp_reg(cg_sec, REG(ra)); // sub sp, sp, ra
        asm_mov_reg_sp(cg_sec, REG(ra)); // mov ra, sp
        // Save current sp/ptr to the alloca var slot
        if (node->args && node->args->ty) {
            // caller uses return value from ra
        }
        return ra;
    }
#else
    // Inline alloca on x86-64 when a register is free: avoids the call to
    // the per-TU __rcc_alloca helper (and its emit) entirely. Falls back to
    // the helper call under register pressure.
    if (call_target && call_target == bi_s_alloca && free_reg_count() >= 1) {
        fn_uses_alloca = true; // no alloca_needed: inlined, helper not emitted
        VReg ra = gen(node->args);
        // Round up to 16 bytes
        x86_add_ri(cg_sec, 8, REG(ra), 15); // addq $15, ra
        x86_and_ri(cg_sec, 8, REG(ra), -16); // andq $-16, ra
        // Page-probe loop: touch each guard page before subtracting 4K so a
        // single big alloca can't skip over the stack guard page.
        int c1 = ++rcc_label_count;
        int c2 = ++rcc_label_count;
        cg_def_label(format(".Lalloca1.%d", c1));
        x86_cmp_ri(cg_sec, 8, REG(ra), 4096); // cmpq $4096, ra
        size_t jb_off = asm_jcc_label(cg_sec, X86_B); // jb .Lalloca2
        // testq %rax, -4096(%rsp) — rax is a physical scratch, never a VReg
        secbuf_emit8(cg_sec, 0x48); // REX.W
        secbuf_emit8(cg_sec, 0x85); // TEST r/m64, r64
        secbuf_emit8(cg_sec, 0x84); // ModRM: mod=10, reg=0(RAX), r/m=4(SIB)
        secbuf_emit8(cg_sec, 0x24); // SIB: base=RSP
        secbuf_emit32le(cg_sec, (uint32_t)-4096); // disp32 = -4096
        x86_sub_ri(cg_sec, 8, X86_RSP, 4096); // subq $4096, %rsp
        x86_sub_ri(cg_sec, 8, REG(ra), 4096); // subq $4096, ra
        size_t jmp1 = asm_jmp_label(cg_sec); // jmp .Lalloca1
        asm_fixup_add(cg_sec, jmp1, format(".Lalloca1.%d", c1), 0);
        cg_def_label(format(".Lalloca2.%d", c2));
        asm_fixup_add(cg_sec, jb_off, format(".Lalloca2.%d", c2), 1);
        x86_sub_rr(cg_sec, 8, X86_RSP, REG(ra)); // subq ra, %rsp
        x86_mov_rr(cg_sec, 8, REG(ra), X86_RSP); // movq %rsp, ra
        return ra;
    }
#endif

#ifdef _WIN32
    int fixed_reg_args = nargs + (has_hidden_retbuf ? 1 : 0);
    int stack_args = fixed_reg_args > max_gp_args ? fixed_reg_args - max_gp_args : 0;
    int stack_pad = (stack_args & 1) ? 8 : 0;
    // Win64 ABI: caller must always allocate 32-byte shadow space, even with no stack args
    int stack_reserve = shadow_space + stack_args * 8 + stack_pad;
#elif defined(ARCH_ARM64)
    // AAPCS64: 8 GP + 8 FP arg registers
    // Linux: variadic floats go in FP regs only (no GP copy)
    // Apple: variadic args always on stack
    int gp_reg_args = 0;
    int fp_reg_args = 0;
    int stack_args = 0;
    Type *fn_type = NULL;
    if (node->lhs && node->lhs->ty) {
        if (node->lhs->ty->kind == TY_PTR)
            fn_type = node->lhs->ty->base;
        else if (node->lhs->ty->kind == TY_FUNC && (node->lhs->kind != ND_LVAR || node->lhs->ty->is_variadic))
            fn_type = node->lhs->ty;
    }
    if (!fn_type && call_target) {
        static Type variadic_fn;
        if (call_target == bi_s_sprintf ||
            call_target == bi_s_snprintf ||
            call_target == bi_s_fprintf ||
            call_target == bi_s_printf) {
            variadic_fn.kind = TY_FUNC;
            variadic_fn.is_variadic = true;
            fn_type = &variadic_fn;
        }
    }
    bool is_variadic = fn_type && fn_type->kind == TY_FUNC && fn_type->is_variadic;
    bool is_oldstyle = !fn_type || (fn_type->kind == TY_FUNC && fn_type->is_oldstyle);
    int named_count = 0;
    if (fn_type && fn_type->kind == TY_FUNC)
        for (Type *t = fn_type->param_types; t; t = t->param_next)
            named_count++;
    if (named_count == 0 && call_target && is_variadic) {
        if (call_target == bi_s_printf)
            named_count = 1;
        else if (call_target == bi_s_sprintf ||
                 call_target == bi_s_fprintf)
            named_count = 2;
        else if (call_target == bi_s_snprintf)
            named_count = 3;
    }
    for (int i = 0; i < nargs; i++) {
        arg_regs[i] = -1;
        arg_sizes[i] = (argv[i]->ty->kind == TY_ARRAY || argv[i]->ty->kind == TY_FUNC) ? 8 : argv[i]->ty->size;
        arg_is_float[i] = is_flonum(argv[i]->ty);
        arg_gp_idx[i] = -1;
        arg_fp_idx[i] = -1;
        arg_stack_idx[i] = -1;
        arg_hfa_count[i] = 0;
        arg_hfa_elem_size[i] = 0;
        if (!arg_is_float[i] && (argv[i]->ty->kind == TY_STRUCT || argv[i]->ty->kind == TY_UNION || argv[i]->ty->kind == TY_COMPLEX))
            arg_hfa_count[i] = arm64_hfa_count(argv[i]->ty, &arg_hfa_elem_size[i]);
        bool is_named = (i < named_count);
        if (arg_is_float[i]) {
            // Long double (> 8 bytes): pass in q-register (SIMD), not d-register or stack
            if (arg_sizes[i] > 8) {
                if (fp_reg_args < max_fp_args)
                    arg_fp_idx[i] = fp_reg_args++;
                else
                    arg_stack_idx[i] = stack_args++;
                continue;
            }
            if (is_variadic && !is_named) {
#if defined(__APPLE__)
                // Apple ARM64: variadic args always on the stack
                arg_stack_idx[i] = stack_args++;
#else
                // Linux AAPCS64: variadic floats promoted to double, in FP regs only
                // (GP copy for unnamed FP args is optional; glibc does not require it)
                if (arg_sizes[i] == 4) arg_sizes[i] = 8; // float -> double promotion for variadic
                if (fp_reg_args < max_fp_args)
                    arg_fp_idx[i] = fp_reg_args++;
                else
                    arg_stack_idx[i] = stack_args++;
#endif
            } else if (fp_reg_args < max_fp_args) {
                arg_fp_idx[i] = fp_reg_args++;
                // Named FP args in variadic functions go in FP regs only (no GP copy)
            } else {
                // All FP regs consumed: arg goes on stack (AAPCS64 6.4.2)
                arg_stack_idx[i] = stack_args++;
            }
            continue;
        }
        if (arg_hfa_count[i] > 0 && !is_variadic) {
            if (fp_reg_args + arg_hfa_count[i] <= max_fp_args) {
                arg_fp_idx[i] = fp_reg_args;
                fp_reg_args += arg_hfa_count[i];
            } else {
                arg_stack_idx[i] = stack_args;
                stack_args += (argv[i]->ty->size + 7) / 8;
            }
            continue;
        }
#if defined(__APPLE__)
        if (is_variadic && !is_named) {
            if (argv[i]->ty->kind == TY_INT128) {
                if (stack_args & 1) stack_args++;
                arg_stack_idx[i] = stack_args;
                stack_args += 2;
            } else if (argv[i]->ty->kind == TY_STRUCT || argv[i]->ty->kind == TY_UNION) {
                if (argv[i]->ty->size == 0)
                    arg_stack_idx[i] = -1; // zero-size: no stack slot, nothing to push
                else {
                    arg_stack_idx[i] = stack_args;
                    stack_args += (argv[i]->ty->size + 7) / 8;
                }
            } else {
                arg_stack_idx[i] = stack_args++;
            }
            continue;
        }
#endif
        // int128 / _BitInt(65..128): two consecutive GP register slots or 16-byte stack
        if (argv[i]->ty && is_int128_like(argv[i]->ty)) {
            if (gp_reg_args + 1 < max_gp_args) {
                arg_gp_idx[i] = gp_reg_args;
                gp_reg_args += 2;
            } else {
                if (stack_args & 1) stack_args++;
                arg_stack_idx[i] = stack_args;
                stack_args += 2;
            }
            continue;
        }
        // complex: one or two GP register slots
        if (argv[i]->ty && is_complex(argv[i]->ty)) {
            if (argv[i]->ty->size <= 8) {
                if (gp_reg_args < max_gp_args)
                    arg_gp_idx[i] = gp_reg_args++;
                else
                    arg_stack_idx[i] = stack_args++;
            } else {
                if (gp_reg_args + 1 < max_gp_args) {
                    arg_gp_idx[i] = gp_reg_args;
                    gp_reg_args += 2;
                } else {
                    if (stack_args & 1) stack_args++;
                    arg_stack_idx[i] = stack_args;
                    stack_args += 2;
                }
            }
            continue;
        }
        if ((argv[i]->ty->kind == TY_STRUCT || argv[i]->ty->kind == TY_UNION) && argv[i]->ty->size > 8) {
            // Large struct > 8 bytes passed by pointer via GP reg or stack
            if (gp_reg_args < max_gp_args)
                arg_gp_idx[i] = gp_reg_args++;
            else
                arg_stack_idx[i] = stack_args++;
            continue;
        }
        if (gp_reg_args < max_gp_args)
            arg_gp_idx[i] = gp_reg_args++;
        else
            arg_stack_idx[i] = stack_args++;
    }
#else
    int gp_reg_args = has_hidden_retbuf ? 1 : 0;
    int fp_reg_args = 0;
    int stack_args = 0;
    Type *fn_type = (node->lhs && node->lhs->ty && node->lhs->ty->kind == TY_PTR)
        ? node->lhs->ty->base
        : NULL;
    bool is_variadic = fn_type && fn_type->kind == TY_FUNC && fn_type->is_variadic;
    bool is_oldstyle = !fn_type || (fn_type->kind == TY_FUNC && fn_type->is_oldstyle);
    int named_count = 0;
    if (fn_type && fn_type->kind == TY_FUNC)
        for (Type *t = fn_type->param_types; t; t = t->param_next)
            named_count++;
    for (int i = 0; i < nargs; i++) {
        arg_regs[i] = -1;
        arg_sizes[i] = (argv[i]->ty->kind == TY_ARRAY || argv[i]->ty->kind == TY_FUNC) ? 8 : argv[i]->ty->size;
        // C default argument promotion: a `float`-typed argument is
        // promoted to `double` whenever there's no fixed prototype
        // parameter for it to bind to -- either a K&R oldstyle
        // (no-prototype) call, or the `...` portion of a genuine C99
        // variadic call (i >= named_count covers both: is_oldstyle's
        // named_count is always 0, so every arg qualifies there too).
        // ARM64's argument-classification loop above already applies
        // this (see "Linux AAPCS64: variadic floats promoted to
        // double"); x86-64 only had the oldstyle half, so a genuine
        // vararg float (e.g. `printf("%f", floatVar)`) fell through as
        // if it were an ordinarily-typed `float` parameter -- landing
        // on whichever narrow-vs-widen ABI convention that path uses,
        // not the promoted double the real printf() expects to receive.
        if ((is_oldstyle || (is_variadic && i >= named_count)) && arg_sizes[i] == 4 && is_flonum(argv[i]->ty))
            arg_sizes[i] = 8; // oldstyle or C99 vararg: float -> double promotion
        arg_is_float[i] = is_flonum(argv[i]->ty);
        arg_gp_idx[i] = -1;
        arg_fp_idx[i] = -1;
        arg_stack_idx[i] = -1;

        if (arg_is_float[i]) {
            // SysV x86-64 ABI: `long double` (80-bit x87 extended
            // precision) is ALWAYS classified MEMORY -- passed on the
            // stack in a 16-byte-aligned, 2-eightbyte slot -- regardless
            // of whether the parameter is variadic/unnamed or an
            // ordinary named/fixed-prototype one (unlike `float`/
            // `double`, x87 values never occupy an XMM register at
            // all). The `is_variadic && !is_named` guard here used to
            // gate this on only the vararg case, so a plain, fully
            // prototyped call like `atanl(long double)`/`fabsl(long
            // double)` fell through to the ordinary float path below and
            // got assigned an XMM register slot instead -- silently
            // corrupting every call into a real (non-rcc-compiled)
            // libm `*l` function, which reads its argument from the
            // stack location the ABI actually specifies and finds
            // garbage there instead.
            if (argv[i]->ty->kind == TY_LDOUBLE) {
                if (stack_args & 1)
                    stack_args++;
                arg_stack_idx[i] = stack_args;
                stack_args += 2;
                continue;
            }
            if (fp_reg_args < max_fp_args) {
                arg_fp_idx[i] = fp_reg_args++;
            } else {
                arg_stack_idx[i] = stack_args++;
            }
            continue;
        }

        // int128 / _BitInt(65..128): needs two consecutive GP register slots
        if (argv[i]->ty && is_int128_like(argv[i]->ty)) {
            if (gp_reg_args + 1 < max_gp_args) {
                arg_gp_idx[i] = gp_reg_args;
                gp_reg_args += 2;
            } else {
                // Align to 16 bytes on stack
                if (stack_args & 1) stack_args++;
                arg_stack_idx[i] = stack_args;
                stack_args += 2;
            }
            continue;
        }
        // complex: one or two slots depending on size; float complex uses SSE regs
        if (argv[i]->ty && is_complex(argv[i]->ty)) {
            bool cfloat = argv[i]->ty->base && is_flonum(argv[i]->ty->base);
            if (cfloat) {
                if (argv[i]->ty->size <= 8) {
                    // _Complex float: one xmm reg
                    if (fp_reg_args < max_fp_args) {
                        arg_fp_idx[i] = fp_reg_args++;
                    } else {
                        arg_stack_idx[i] = stack_args++;
                    }
                } else {
                    // _Complex double: two xmm regs
                    if (fp_reg_args + 1 < max_fp_args) {
                        arg_fp_idx[i] = fp_reg_args;
                        fp_reg_args += 2;
                    } else {
                        if (stack_args & 1) stack_args++;
                        arg_stack_idx[i] = stack_args;
                        stack_args += 2;
                    }
                }
            } else {
                // Integer complex: one or two GP regs
                if (argv[i]->ty->size <= 8) {
                    if (gp_reg_args < max_gp_args)
                        arg_gp_idx[i] = gp_reg_args++;
                    else
                        arg_stack_idx[i] = stack_args++;
                } else {
                    if (gp_reg_args + 1 < max_gp_args) {
                        arg_gp_idx[i] = gp_reg_args;
                        gp_reg_args += 2;
                    } else {
                        if (stack_args & 1) stack_args++;
                        arg_stack_idx[i] = stack_args;
                        stack_args += 2;
                    }
                }
            }
            continue;
        }

        if (gp_reg_args < max_gp_args)
            arg_gp_idx[i] = gp_reg_args++;
        else
            arg_stack_idx[i] = stack_args++;
    }

    int stack_pad = (stack_args & 1) ? 8 : 0;
    int stack_reserve = shadow_space + stack_args * 8 + stack_pad;
#endif

#ifdef ARCH_ARM64
    // Stage register-destined argument values through dedicated stack
    // slots (instead of leaving them live in the VReg pool across further
    // argument evaluation) only under real register pressure: alloc_reg()'s
    // spill mechanism assumes strictly nested (LIFO) lifetimes, which a
    // call with many simultaneously-live argument values violates -- see
    // the staging loop below. Gating on pressure (mirroring x86-64's
    // identical use_staging condition, including its >= and headroom
    // terms -- see that condition's own comment for why `> NUM_REGS`
    // alone is unsafe: hitting the limit exactly still leaves zero spare
    // registers for a later argument's own nested evaluation) keeps
    // ordinary low-arity calls (the overwhelming majority) on the
    // untouched, well-exercised path; unconditional staging was tried
    // first and regressed ordinary multi-statement functions.
    int live_now_arm64 = 0;
    for (int _li = 0; _li < NUM_REGS; _li++)
        live_now_arm64 += (used_regs >> _li) & 1;
    int stack_scratch_arm64 = (nargs > 1) ? 2 : 0;
    bool use_staging_arm64 =
        (live_now_arm64 + gp_reg_args + fp_reg_args + stack_scratch_arm64) >= NUM_REGS;

    // ARM64: evaluate args for register passing, track stack args
    for (int i = 0; i < nargs; i++) {
        // Skip regular stack args, but evaluate HFAs on stack for data copying
        if (arg_stack_idx[i] >= 0 && arg_hfa_count[i] <= 0)
            continue;
        if (arg_hfa_count[i] > 0 || ((argv[i]->ty->kind == TY_STRUCT || argv[i]->ty->kind == TY_UNION) && argv[i]->ty->size > 8)) {
            VReg addr = gen_addr(argv[i]);

            if (addr < 0) {
                // Non-lvalue struct/union (e.g. cast to struct): allocate temp slot and store value
                int sz = argv[i]->ty->size;
                int alloc = (sz + 15) & ~15;
                fn_struct_ret_off += alloc;
                if (fn_struct_ret_off > fn_struct_ret_total)
                    fn_struct_ret_total = fn_struct_ret_off;
                int tmp_slot = current_fn_stack_size + fn_struct_ret_off;
                addr = alloc_reg();
                emit_mov_imm64(ARM64_X16, (uint64_t)tmp_slot);
                asm_sub_reg3(cg_sec, addr, ARM64_X29, ARM64_X16, 8); // sub x{addr}, x29, x16
                // Zero out the slot using stp xzr, xzr, [x{addr}, #zb]
                for (int zb = 0; zb < alloc; zb += 16) {
                    int32_t imm7 = zb / 8; // in units of 8 bytes
                    asm_stp_xzr_xzr(cg_sec, addr, imm7); // stp xzr, xzr, [x{addr}, #zb]
                }
                // Store the computed value into the first bytes of the temp slot
                VReg val = gen(argv[i]);
                if (val >= 0) {
                    int vsz = argv[i]->ty->size < 8 ? argv[i]->ty->size : 8;
                    if (vsz == 4)
                        asm_str_reg_off(cg_sec, val, addr, 4, 0); // str w{val}, [x{addr}]
                    else
                        asm_str_reg_off(cg_sec, val, addr, 8, 0); // str x{val}, [x{addr}]
                    free_reg(val);
                }
            }
            arg_regs[i] = addr;
        } else if ((argv[i]->ty->kind == TY_STRUCT || argv[i]->ty->kind == TY_UNION) && argv[i]->ty->size <= 8) {
            // Small struct passed by value in a GP register.
            // gen() for ND_DEREF/ND_MEMBER returns the address, not the value.
            int addr = gen_addr(argv[i]);
            if (addr >= 0) {
                int load_sz = argv[i]->ty->size <= 4 ? 4 : 8;
                VReg val = alloc_reg();
                if (load_sz == 8)
                    arm64_ldr_uoff(cg_sec, 3, REG(val), REG(addr), 0);
                else
                    arm64_ldr_uoff(cg_sec, 2, REG(val), REG(addr), 0);
                free_reg(addr);
                arg_regs[i] = val;
            } else {
                arg_regs[i] = gen(argv[i]);
            }
        } else
            arg_regs[i] = gen(argv[i]);
        // Relocate the just-computed value out of the register allocator's
        // live pool immediately, into its own permanent stack slot -- but
        // ONLY under real register pressure (use_staging_arm64). Left
        // live in a VReg across further argument evaluation, it can be
        // stolen by alloc_reg() while evaluating a LATER argument in this
        // same loop: alloc_reg()'s spill victim selection only tracks ONE
        // pending value per register index, so if this argument's value
        // is written directly into a register that was already mid-spill
        // (protecting some OTHER, still-live value), a later free_reg() on
        // that register silently restores the OTHER value -- discarding
        // this argument's value without it ever having gone through its
        // own spill/restore cycle. Mirrors the x86-64 path's
        // `use_staging`. Found via a variadic call mixing 7 named ints +
        // 9 named doubles + int varargs: the first vararg silently read
        // as 0 instead of 8.
        if (use_staging_arm64 && arg_regs[i] >= 0) {
            fn_struct_ret_off += 8;
            if (fn_struct_ret_off > fn_struct_ret_total)
                fn_struct_ret_total = fn_struct_ret_off;
            arg_stage[i] = current_fn_stack_size + fn_struct_ret_off;
            asm_stur_fp(cg_sec, arg_regs[i], arg_stage[i]); // str x{arg_regs[i]}, [x29, #-stage]
            // The arg is safely archived in its staging slot.  Release the
            // register so the marshal can reuse it, but do NOT clear the
            // spilled_regs bit — an alloc_reg() that spilled an outer
            // expression's live value to give us this register left its
            // data in spill_offset(); clearing the bit orphans that data.
            used_regs &= ~(1 << arg_regs[i]);
            reg_owner[arg_regs[i]] = NULL;
            arg_regs[i] = -1;
        }
    }

    // Allocate one block for caller-saved regs + stack args.
    // Layout (from low sp upward):
    //   [sp + 0 .. stack_args*8-1]        : stack args (callee reads via x29+16+idx*8)
    //   [sp + stack_args*8 .. total-1]    : saved caller-saved regs
    // This keeps stack args at the standard AAPCS64 location relative to sp_at_call.
    int arm64_saved_mask = used_regs & 63;
    int sv_count = __builtin_popcount(arm64_saved_mask);

    if (sv_count > 0 || stack_args > 0) {
        int total = (sv_count + stack_args) * 8;
        total = (total + 15) & ~15;
        if (total < 4096)
            arm64_sub_imm(cg_sec, 1, ARM64_SP, ARM64_SP, total, 0); // sub sp, sp, #total
        else
            arm64_sub_imm(cg_sec, 1, ARM64_SP, ARM64_SP, total >> 12, 1); // sub sp, sp, #total (shifted)
    }

    // Save caller-saved regs (virtual 0-5 = physical x10-x15) above stack args area
    int sv_off = stack_args * 8;
    for (int i = 0; i < 6; i++) {
        if (arm64_saved_mask & (1 << i)) {
            arm64_str_uoff(cg_sec, 3, REG(i), ARM64_SP, sv_off / 8); // str x{i}, [sp, #sv_off]
            sv_off += 8;
        }
    }

    // Push stack args at [sp + arg_stack_idx[i]*8]
    for (int i = nargs - 1; i >= 0; i--) {
        if (arg_stack_idx[i] < 0)
            continue;
        int off = arg_stack_idx[i] * 8;
        VReg r;
        if ((argv[i]->ty->kind == TY_STRUCT || argv[i]->ty->kind == TY_UNION) && argv[i]->ty->size > 8) {
#if defined(__APPLE__) && defined(ARCH_ARM64)
            if (is_variadic && !(i < named_count)) {
                int vaddr = gen_addr(argv[i]);
                int vsz = argv[i]->ty->size;
                for (int voff = 0; voff < vsz; voff += 8) {
                    arm64_ldr_uoff(cg_sec, 3, ARM64_X16, REG(vaddr), (uint32_t)(voff / 8));
                    arm64_str_uoff(cg_sec, 3, ARM64_X16, ARM64_SP, (uint32_t)(arg_stack_idx[i] + voff / 8));
                }
                free_reg(vaddr);
                continue;
            }
#endif
            // HFA overflow: an HFA/HVA argument that doesn't fit in the
            // remaining SIMD/FP registers (Stage C.3) is passed entirely
            // on the stack, by value -- never via a hidden pointer, on
            // both Linux and Apple AAPCS64 (unlike the Apple-only "every
            // variadic arg goes to stack" override just above, this is
            // standard AAPCS64 and must not be Apple-gated).
            if (arg_hfa_count[i] > 0 && arg_fp_idx[i] < 0) {
                int vaddr = gen_addr(argv[i]);
                int vsz = argv[i]->ty->size;
                for (int voff = 0; voff < vsz; voff += 8) {
                    arm64_ldr_uoff(cg_sec, 3, ARM64_X16, REG(vaddr), (uint32_t)(voff / 8));
                    arm64_str_uoff(cg_sec, 3, ARM64_X16, ARM64_SP, (uint32_t)(arg_stack_idx[i] + voff / 8));
                }
                free_reg(vaddr);
                continue;
            }
            r = gen_addr(argv[i]);
            arm64_str_uoff(cg_sec, 3, REG(r), ARM64_SP, (uint32_t)(off / 8)); // str x{r}, [sp, #off]
            free_reg(r);
        } else if (argv[i]->ty && is_int128_like(argv[i]->ty)) {
            int addr = gen(argv[i]); // gen_int128/gen_bitint -> slot address
            arm64_ldr_uoff(cg_sec, 3, ARM64_X16, REG(addr), 0);
            arm64_ldr_uoff(cg_sec, 3, ARM64_X17, REG(addr), 1);
            arm64_str_uoff(cg_sec, 3, ARM64_X16, ARM64_SP, (uint32_t)arg_stack_idx[i]);
            arm64_str_uoff(cg_sec, 3, ARM64_X17, ARM64_SP, (uint32_t)(arg_stack_idx[i] + 1));
            free_reg(addr);
            continue;
        } else {
            // For struct args <=8 bytes, use gen_addr + ldr x16 to avoid
            // putting the value in a managed register (prevents spill conflicts).
            if (argv[i]->ty && (argv[i]->ty->kind == TY_STRUCT || argv[i]->ty->kind == TY_UNION)) {
                int addr = gen_addr(argv[i]);
                int sz = argv[i]->ty->size <= 4 ? 4 : 8;
                arm64_ldr_uoff(cg_sec, sz == 8 ? 3 : 2, ARM64_X16, REG(addr), 0); // ldr x16/w16, [addr]
                arm64_str_uoff(cg_sec, 3, ARM64_X16, ARM64_SP, (uint32_t)(off / 8)); // str x16, [sp]
                free_reg(addr);
            } else {
                r = gen(argv[i]);
                if (arg_is_float[i]) {
                    asm_fmov_i2f(cg_sec, 0, r, 1); // fmov d0, x{r}
                    // Same "narrow only if the callee's fixed prototype
                    // parameter is genuinely `float`" logic as the
                    // register-arg placement above (keep_double /
                    // callee_expects_float): a stack-overflow float
                    // argument crosses the ABI exactly like a register
                    // one, just via memory instead of a v-register, and
                    // needs the same distinction between a real `float`
                    // parameter and a promoted (oldstyle/vararg) double.
                    bool keep_double = (is_variadic && i >= named_count) ||
                        is_oldstyle || (fn_type && !fn_type->param_types);
                    bool callee_expects_float = false;
                    if (!keep_double) {
                        callee_expects_float = argv[i]->ty->size == 4;
                        if (fn_type && fn_type->param_types && i < named_count) {
                            Type *pt = fn_type->param_types;
                            for (int j = 0; j < i && pt; j++) pt = pt->param_next;
                            callee_expects_float = pt && pt->kind == TY_FLOAT;
                        }
                    }
                    if (callee_expects_float) {
                        asm_fcvt(cg_sec, 0, 1, 0, 0); // fcvt s0, d0 (double->single)
                        arm64_str_fp(cg_sec, 2, 0, ARM64_SP, (uint32_t)off); // str s0, [sp, #off]
                    } else {
                        arm64_str_fp(cg_sec, 3, 0, ARM64_SP, (uint32_t)off); // str d0, [sp, #off]
                    }
                } else {
                    arm64_str_uoff(cg_sec, 3, REG(r), ARM64_SP, (uint32_t)(off / 8));
                }
                free_reg(r);
            }
        }
    }

    VReg temp_ret_reg = R_NONE;
    int temp_ret_slot = 0;
    bool need_temp_ret = false;
    if (has_hidden_retbuf) {
        if (hidden_ret_reg == -1) {
            // Defer allocating the result register until after the call:
            // alloc_reg() here can spill-borrow a register that still stages
            // an argument value, which the staging moves below would then
            // read as garbage. Compute the address directly into x8 instead.
            need_temp_ret = true;
            int alloc = (node->ty->size + 15) & ~15;
            fn_struct_ret_off += alloc;
            if (fn_struct_ret_off > fn_struct_ret_total)
                fn_struct_ret_total = fn_struct_ret_off;
            temp_ret_slot = current_fn_stack_size + fn_struct_ret_off;
            if (temp_ret_slot <= 4095)
                arm64_sub_imm(cg_sec, 1, ARM64_X8, ARM64_X29, temp_ret_slot, 0); // sub x8, x29, #temp_ret_slot
            else {
                emit_mov_imm64(ARM64_X16, (uint64_t)temp_ret_slot); // mov x16, #temp_ret_slot
                arm64_sub_reg(cg_sec, 1, ARM64_X8, ARM64_X29, ARM64_X16, 0, 0); // sub x8, x29, x16
            }
        } else
            asm_mov_x8_reg(cg_sec, hidden_ret_reg); // mov x8, x{hidden_ret_reg}
    }

    // Pre-pass: long double args. rcc computes `long double` internally in
    // double precision (a GP register holds the double's 64-bit pattern),
    // but AAPCS64 requires the real IEEE-754 binary128 encoding in the full
    // 128-bit Q register whenever the value crosses to ABI-conforming code
    // (glibc printf/sprintf/... reading a "%Lf" va_arg). A bare `fmov d,x`
    // only fills the low 64 bits and implicitly zeroes the upper 64 (per
    // AArch64 scalar-FMOV-to-V semantics) — that upper half holds the quad's
    // sign+exponent, so the callee decodes the value as zero. Widen through
    // libgcc's __extenddftf2 (double -> binary128, arg in d0, result in v0),
    // exactly like the existing __ashlti3/__divdc3 libgcc calls elsewhere in
    // this file. Virtual GP regs live in x19-x28 (callee-saved), so other
    // already-evaluated args survive the nested `bl` unharmed.
    //
    // Each converted quad is parked in its own fresh 16-byte stack slot
    // rather than moved straight into arg_fp_idx[i]: with >=2 long-double
    // args, an earlier arg's target register can be v0/q0 itself (fp_idx
    // 0), and the next __extenddftf2 call clobbers v0 as its own scratch —
    // a direct register-to-register placement would be destroyed before
    // the call ever executes. Routing through memory decouples conversion
    // from placement so ordering can't matter.
    for (int i = 0; i < nargs; i++) {
        if (arg_stack_idx[i] >= 0)
            continue;
        if (arg_is_float[i] && arg_sizes[i] > 8 && arg_fp_idx[i] >= 0) {
            if (arg_regs[i] < 0 && arg_stage[i]) {
                arg_regs[i] = alloc_reg();
                asm_ldur_fp(cg_sec, arg_regs[i], arg_stage[i]); // ldr x{arg_regs[i]}, [x29, #-stage]
            }
            asm_fmov_i2f(cg_sec, ASM_Q0, arg_regs[i], 1); // fmov d0, x{arg_regs[i]}
            free_reg(arg_regs[i]);
            emit_direct_call("__extenddftf2", false); // bl __extenddftf2 (d0 -> quad in v0)
            VReg slot = alloc_int128_addr();
            asm_str_q(cg_sec, ASM_Q0, slot); // str q0, [x{slot}]
            arg_regs[i] = slot; // repurposed: holds the parked quad's address
        }
    }
    for (int i = 0; i < nargs; i++) {
        if (arg_stack_idx[i] >= 0)
            continue;
        if (arg_is_float[i] && arg_sizes[i] > 8 && arg_fp_idx[i] >= 0) {
            asm_ldr_q(cg_sec, arg_fp_idx[i], arg_regs[i]); // ldr q{fp_idx}, [x{arg_regs[i]}]
            free_reg(arg_regs[i]);
        }
    }

    // Move evaluated args into arg registers
    for (int i = 0; i < nargs; i++) {
        if (arg_stack_idx[i] >= 0)
            continue;
        // Skip args with no assigned destination (e.g. zero-size variadic structs)
        if (arg_gp_idx[i] < 0 && arg_fp_idx[i] < 0) {
            if (arg_regs[i] >= 0) free_reg(arg_regs[i]);
            continue;
        }
        // Reload from the staging slot (see the staging comment in the
        // first-pass evaluation loop above) before any branch below reads
        // arg_regs[i]. No-op for the long-double branch just below (its
        // arg_regs[i] was already freshly set by the pre-pass loop, never
        // staged) and for any arg that never went through staging.
        if (arg_regs[i] < 0 && arg_stage[i]) {
            arg_regs[i] = alloc_reg();
            asm_ldur_fp(cg_sec, arg_regs[i], arg_stage[i]); // ldr x{arg_regs[i]}, [x29, #-stage]
            // Fresh alloc for the staging load — any spilled_regs bit is
            // from a prior outer-expression spill, not from this arg.
            // Keep it set so the outer value survives in the spill slot.
        }
        if (arg_is_float[i] && arg_sizes[i] > 8 && arg_fp_idx[i] >= 0) {
            continue;
        }
        if (arg_hfa_count[i] > 0 && arg_fp_idx[i] >= 0) {
            for (int j = 0; j < arg_hfa_count[i]; j++) {
                int off = j * arg_hfa_elem_size[i];
                // ldr s/d{fp_idx+j}, [x{arg_regs[i]}, #off]
                asm_ldr_fp_off(cg_sec, arg_fp_idx[i] + j, arg_regs[i],
                               arg_hfa_elem_size[i], (uint32_t)off);
            }
            free_reg(arg_regs[i]);
            continue;
        }
        // Variadic small HFA struct (size<=8): gen_addr gave address, but GP register
        // must hold the VALUE (va_arg reads it directly, not via pointer).
        if (arg_hfa_count[i] > 0 && arg_gp_idx[i] >= 0 && arg_fp_idx[i] < 0 && arg_sizes[i] <= 8) {
            if (arg_sizes[i] == 4)
                asm_ldr_phy_reg(cg_sec, arg_gp_idx[i], arg_regs[i], 0); // ldr w{gp_idx}, [x{arg_reg}]
            else
                asm_ldr_phy_reg(cg_sec, arg_gp_idx[i], arg_regs[i], 1); // ldr x{gp_idx}, [x{arg_reg}]
            free_reg(arg_regs[i]);
            continue;
        }
        if (arg_is_float[i] && arg_fp_idx[i] >= 0) {
            // Float values are always stored as double in GP regs (see ND_LVAR with is_flonum).
            // Move as double to FP arg register, then convert to float if callee expects float.
            asm_fmov_i2f(cg_sec, arg_fp_idx[i], arg_regs[i], 1); // fmov d{fp_idx}, x{arg_reg}
            // Convert double->float only if callee param is known float,
            // not for variadic args (which stay promoted to double)
            bool keep_double = (is_variadic && i >= named_count) ||
                is_oldstyle || (fn_type && !fn_type->param_types);
            bool callee_expects_float = false;
            if (!keep_double) {
                callee_expects_float = argv[i]->ty->size == 4;
                if (fn_type && fn_type->param_types && i < named_count) {
                    Type *pt = fn_type->param_types;
                    for (int j = 0; j < i && pt; j++) pt = pt->param_next;
                    callee_expects_float = pt && pt->kind == TY_FLOAT;
                }
            }
            if (callee_expects_float)
                asm_fcvt(cg_sec, 0, 1, arg_fp_idx[i], arg_fp_idx[i]); // fcvt s{fp_idx}, d{fp_idx}  (opc=0: double->single)
            // Non-float args that landed in a GP slot (kept for non-ARM64 paths)
            if (arg_gp_idx[i] >= 0)
                asm_mov_phy_reg(cg_sec, arg_gp_idx[i], arg_regs[i], 1); // mov x{gp_idx}, x{arg_reg}
        } else if (arg_is_float[i] && arg_gp_idx[i] >= 0) {
            asm_mov_phy_reg(cg_sec, arg_gp_idx[i], arg_regs[i], 1); // mov x{gp_idx}, x{arg_reg}
        } else if (argv[i]->ty && is_int128_like(argv[i]->ty)) {
            // int128 / _BitInt(65..128): load two 64-bit values into consecutive arg registers
            arm64_ldr_uoff(cg_sec, 3, ARM64_X16, REG(arg_regs[i]), 0); // ldr x16, [addr]
            arm64_ldr_uoff(cg_sec, 3, ARM64_X17, REG(arg_regs[i]), 1); // ldr x17, [addr, #8]
            asm_mov_phy_phy(cg_sec, arg_gp_idx[i], ARM64_X16, 1); // mov x{gp_idx}, x16
            asm_mov_phy_phy(cg_sec, arg_gp_idx[i] + 1, ARM64_X17, 1); // mov x{gp_idx+1}, x17
        } else if (arg_sizes[i] == 1 || arg_sizes[i] == 2) {
            asm_mov_phy_reg(cg_sec, arg_gp_idx[i], arg_regs[i], 1); // mov x{gp_idx}, x{arg_reg}
        } else if (arg_sizes[i] == 4) {
            asm_mov_phy_reg(cg_sec, arg_gp_idx[i], arg_regs[i], 1); // mov x{gp_idx}, x{arg_reg}

        } else {
#ifdef ARCH_ARM64
            if (argv[i]->ty && is_complex(argv[i]->ty) && argv[i]->ty->size <= 8) {
                arm64_ldr_uoff(cg_sec, 3, (Arm64Reg)arg_gp_idx[i], REG(arg_regs[i]), 0); // ldr x{gp_idx}, [arg_regs[i]]
            } else if (argv[i]->ty && is_complex(argv[i]->ty) && argv[i]->ty->size > 8) {
                int base_sz = argv[i]->ty->base ? argv[i]->ty->base->size : 8;
                arm64_ldr_uoff(cg_sec, 3, (Arm64Reg)arg_gp_idx[i], REG(arg_regs[i]), 0); // ldr x{gp_idx}, [arg_regs[i]]
                arm64_ldr_uoff(cg_sec, 3, (Arm64Reg)(arg_gp_idx[i] + 1), REG(arg_regs[i]), (uint32_t)(base_sz / 8)); // ldr x{gp_idx+1}, [arg_regs[i], #base_sz]
            } else
#endif
                asm_mov_phy_reg(cg_sec, arg_gp_idx[i], arg_regs[i], 1); // mov x{gp_idx}, x{arg_reg}
        }
        free_reg(arg_regs[i]);
    }

    // For indirect calls: save arg registers before evaluating the callee,
    // since gen(node->lhs) may call functions that clobber x0-x8 / d0-d7.
    // (x86_64 avoids this by evaluating the callee first; ARM64 evaluates args first.)
    int indirect_gp_mask = 0;
    int indirect_fp_mask = 0;
    bool indirect_save_x8 = false;
    int indirect_save_size = 0;
    if (!call_target) {
        for (int i = 0; i < nargs; i++) {
            if (arg_gp_idx[i] >= 0) indirect_gp_mask |= (1 << arg_gp_idx[i]);
            if (arg_fp_idx[i] >= 0) indirect_fp_mask |= (1 << arg_fp_idx[i]);
        }
        indirect_save_x8 = has_hidden_retbuf;
        int n_save = __builtin_popcount(indirect_gp_mask) + __builtin_popcount(indirect_fp_mask) + (indirect_save_x8 ? 1 : 0);
        if (n_save > 0) {
            indirect_save_size = (n_save * 8 + 15) & ~15;
#ifdef ARCH_ARM64
            arm64_sub_imm(cg_sec, 1, ARM64_SP, ARM64_SP, indirect_save_size, 0);
#else
            x86_sub_ri(cg_sec, 8, X86_RSP, indirect_save_size);
#endif // sub sp, sp, #indirect_save_size
            int slot = 0;
            if (indirect_save_x8)
                arm64_str_uoff(cg_sec, 3, ARM64_X8, ARM64_SP, (uint32_t)slot++); // str x8, [sp, #slot*8]
            for (int j = 0; j < 8; j++)
                if (indirect_gp_mask & (1 << j))
                    arm64_str_uoff(cg_sec, 3, (Arm64Reg)(ARM64_X0 + j), ARM64_SP, (uint32_t)slot++); // str x{j}, [sp, #slot*8]
            for (int j = 0; j < 8; j++)
                if (indirect_fp_mask & (1 << j))
                    arm64_str_fp(cg_sec, 3, (Arm64Reg)(ARM64_D0 + j), ARM64_SP, (uint32_t)(slot++ * 8)); // str d{j}, [sp, #slot*8]
        }
    }

    if (call_target) {
        if (node->lhs && node->lhs->var && node->lhs->var->is_nested_fn) {
            // GNU nested function, direct call: load the static-chain
            // pointer into the physical chain register (x18) right before
            // the call — mirrors the x86-64 path's identical r10 handling.
            if (node->lhs->chain_depth == 0)
                arm64_orr_reg(cg_sec, 1, ARM64_X18, ARM64_XZR, ARM64_X29, ARM64_LSL, 0); // mov x18, x29
            else
                arm64_ldur(cg_sec, 1, ARM64_X18, ARM64_X29, -CHAIN_SLOT_OFFSET); // ldur x18, [x29, #-CHAIN_SLOT_OFFSET]
        }
        if (call_target == bi_s_alloca) {
            alloca_needed = true;
            fn_uses_alloca = true;
            emit_direct_call("__rcc_alloca", false);
        } else {
            emit_direct_call(call_target, is_asm_call);
        }
    } else {
        int callee = gen(node->lhs);
        if (indirect_save_size > 0) {
            int slot = 0;
            if (indirect_save_x8)
                arm64_ldr_uoff(cg_sec, 3, ARM64_X8, ARM64_SP, (uint32_t)slot++); // ldr x8, [sp]
            for (int j = 0; j < 8; j++)
                if (indirect_gp_mask & (1 << j))
                    arm64_ldr_uoff(cg_sec, 3, (Arm64Reg)(ARM64_X0 + j), ARM64_SP, (uint32_t)slot++); // ldr x{j}, [sp, #slot*8]
            for (int j = 0; j < 8; j++)
                if (indirect_fp_mask & (1 << j))
                    arm64_ldr_fp(cg_sec, 3, (Arm64Reg)(ARM64_D0 + j), ARM64_SP, (uint32_t)(slot++ * 8)); // ldr d{j}, [sp, #slot*8]
            arm64_add_imm(cg_sec, 1, ARM64_SP, ARM64_SP, indirect_save_size, 0); // add sp, sp, #indirect_save_size
        }
        asm_call_reg(cg_sec, callee); // call callee
        free_reg(callee);
    }

    // Restore caller-saved registers from above-stack-args area
    if (arm64_saved_mask) {
        int sv = stack_args * 8;
        for (int i = 0; i < 6; i++) {
            if (arm64_saved_mask & (1 << i)) {
                arm64_ldr_uoff(cg_sec, 3, REG(i), ARM64_SP, sv / 8); // ldr x{i}, [sp, #sv]
                sv += 8;
            }
        }
    }

    // Mark saved arg registers as free after the call.
    // They held argument values that are dead post-call.
    // This prevents the allocator from using callee-saved x19-x24
    // when many calls happen in sequence (pr92904).
    int dead_mask = 0;
    for (int i = 0; i < nargs; i++) {
        if (arg_regs[i] >= 0 && arg_regs[i] < 6)
            dead_mask |= (1 << arg_regs[i]);
    }
    used_regs &= ~(arm64_saved_mask & dead_mask);

    // Restore sp
    if (sv_count > 0 || stack_args > 0) {
        int total = (sv_count + stack_args) * 8;
        total = (total + 15) & ~15;
        if (total < 4096)
            arm64_add_imm(cg_sec, 1, ARM64_SP, ARM64_SP, total, 0); // add sp, sp, #total
        else
            arm64_add_imm(cg_sec, 1, ARM64_SP, ARM64_SP, total >> 12, 1); // add sp, sp, #total (shifted)
    }

    if (has_hidden_retbuf) {
        if (need_temp_ret) {
            // Allocate the result register now that the staged arg registers
            // are free again, and derive the frame-relative buffer address.
            temp_ret_reg = alloc_reg();
            hidden_ret_reg = temp_ret_reg;
        }
        if (temp_ret_reg != R_NONE) {
            if (temp_ret_slot <= 4095)
                asm_add_reg_fp_imm(cg_sec, temp_ret_reg, -temp_ret_slot); // add temp_ret_reg, x29, #-temp_ret_slot
            else {
                int v = temp_ret_slot;
                emit_mov_imm64(ARM64_X16, v & 0xffff); // mov x16, #v
                v >>= 16;
                int s = 16;
                while (v) {
                    asm_movk(cg_sec, 16, 1, (uint16_t)(v & 0xffff), s); // movk x16, #v, lsl #s
                    v >>= 16;
                    s += 16;
                }
                asm_sub_reg_fp_phy(cg_sec, temp_ret_reg, ARM64_X16, 8); // sub temp_ret_reg, x29, x16
            }
        }
        return temp_ret_reg != R_NONE ? temp_ret_reg : hidden_ret_reg;
    }
    // AAPCS64: a struct/union <=16 bytes with no floating member anywhere
    // (struct_returns_in_gp_regs) returns raw bits in X0 (first eightbyte)
    // and, if bigger than 8 bytes, X1 (second eightbyte) -- no indirect
    // result register (X8) involved. Mirrors the has_hidden_retbuf
    // classification above. Must precede the generic scalar fallback below
    // (the has_hidden_retbuf branch above already returned for this
    // function, so this is the actual reachable ARM64 call-site path).
    if (node->ty && struct_returns_in_gp_regs(node->ty)) {
        int addr = hidden_ret_reg != -1 ? hidden_ret_reg : alloc_int128_addr();
        int lo_sz = node->ty->size <= 4 ? 4 : 8;
        asm_str_reg_off_phy(cg_sec, ARM64_X0, addr, lo_sz, 0); // str x0, [addr]
        if (node->ty->size > 8) {
            int hi_sz = node->ty->size - 8 <= 4 ? 4 : 8;
            asm_str_reg_off_phy(cg_sec, ARM64_X1, addr, hi_sz, 8); // str x1, [addr, #8]
        }
        return addr;
    }
    // int128 / _BitInt(65..128) return: spill x0:x1 to a 16-byte slot
    if (node->ty && is_int128_like(node->ty)) {
        int addr = alloc_int128_addr();
        arm64_str_uoff(cg_sec, 3, ARM64_X0, REG(addr), 0); // str x0, [x{addr}]
        arm64_str_uoff(cg_sec, 3, ARM64_X1, REG(addr), 1); // str x1, [x{addr}, #8]
        return addr;
    }

#ifndef __APPLE__
    // Linux AAPCS64 only: Apple's arm64 ABI defines `long double` as
    // identical to `double` (see type.c's ty_ldouble, size=8 under
    // __APPLE__ vs size=16 elsewhere) -- there's no widen/narrow step
    // to do on Darwin at all, and calling __trunctfdf2 on an already-
    // plain-double value would corrupt it.
    if (node->ty && node->ty->kind == TY_LDOUBLE) {
        // AAPCS64 returns a `long double` result as a genuine 128-bit
        // binary128 quad in v0 (see the identical narrowing requirement
        // already handled for ND_VA_ARG and the x86-64 prologue's
        // variadic-float-narrowing comments above). rcc's own internal
        // `long double` representation is a plain 64-bit double
        // everywhere else, so narrow the just-returned quad via
        // libgcc's __trunctfdf2 -- consuming the value already sitting
        // in v0 and leaving the narrowed double in d0 -- *before*
        // alloc_reg() below hands out a VReg, since __trunctfdf2 is a
        // real `bl` that clobbers the same caller-saved physical
        // registers rcc's allocator would otherwise be free to reuse.
        emit_direct_call("__trunctfdf2", false); // bl __trunctfdf2 (quad in v0 -> d0)
    }
#endif
    int r = alloc_reg();
    if (node->ty && is_flonum(node->ty)) {
        if (node->ty->kind == TY_FLOAT)
            asm_fcvt(cg_sec, 1, 0, 0, 0); // fcvt d0, s0 (widen float return to double, opc=1=single->double)
        asm_fmov_f2i(cg_sec, r, 0, 1); // fmov x{r}, d0 (store float return as int bits)
    } else {
        int sz = node->ty ? (node->ty->size < 8 ? 4 : 8) : 8;
        asm_mov_retval(cg_sec, r, sz); // mov x{r}, x0 (return value)
        // AAPCS64: narrow return values sign/zero-extend to full register width
        // Skip void functions (size 1 with TY_VOID kind)
        if (node->ty && node->ty->kind != TY_VOID && node->ty->size < sz && node->ty->size < 4) {
            if (node->ty->is_unsigned)
                zero_extend_to(r, node->ty->size, 8);
            else
                sign_extend_to(r, node->ty->size, 8);
        }
    }
    return r;
#else
    // === x86_64 (Windows + Linux) calling convention ===
    int saved_scratch = used_regs & 3;
    // The hidden_ret_reg exclusion only applies to the has_hidden_retbuf path,
    // where temp_ret_reg (r10/r11) is reloaded via a frame-relative `lea`
    // after the call. If there's no hidden retbuf, hidden_ret_reg (if set)
    // just holds a destination address from gen_addr() that must survive the
    // call like any other live register.
    if (saved_scratch & 1) {
        asm_mov_phyreg_rbp(cg_sec, X86_R10, 8, spill_offset(0)); // mov [rbp-spill_offset(0)], X86_R10
        // Keep r10 marked as in-use so alloc_reg() doesn't reuse it for the
        // hidden ret buffer, which would overwrite a caller's live arg value.
    }
    if (saved_scratch & 2) {
        asm_mov_phyreg_rbp(cg_sec, X86_R11, 8, spill_offset(1)); // mov [rbp-spill_offset(1)], X86_R11
        // Same for r11.
    }

    VReg callee_reg = -1;
    if (!call_target) {
        callee_reg = gen(node->lhs);
    }

#ifdef _WIN32
    Type *fn_type_w = (node->lhs && node->lhs->ty && node->lhs->ty->kind == TY_PTR)
        ? node->lhs->ty->base
        : NULL;
    bool is_oldstyle = !fn_type_w || (fn_type_w->kind == TY_FUNC && fn_type_w->is_oldstyle);
    bool is_variadic = fn_type_w && fn_type_w->kind == TY_FUNC && fn_type_w->is_variadic;
    int named_count = 0;
    if (fn_type_w && fn_type_w->kind == TY_FUNC)
        for (Type *t = fn_type_w->param_types; t; t = t->param_next)
            named_count++;
    int reg_nargs = nargs < max_gp_args - (has_hidden_retbuf ? 1 : 0) ? nargs : max_gp_args - (has_hidden_retbuf ? 1 : 0);
    for (int i = 0; i < reg_nargs; i++) {
        if ((argv[i]->ty->kind == TY_STRUCT || argv[i]->ty->kind == TY_UNION || is_complex(argv[i]->ty)) && argv[i]->ty->size > 8)
            arg_regs[i] = gen_addr(argv[i]);
        else if (argv[i]->ty && (argv[i]->ty->kind == TY_STRUCT || argv[i]->ty->kind == TY_UNION) && argv[i]->ty->size <= 8) {
            int addr = gen_addr(argv[i]);
            if (addr >= 0) {
                int load_sz = argv[i]->ty->size <= 4 ? 4 : 8;
                VReg val = alloc_reg();
                x86_mov_rm(cg_sec, load_sz, REG(val), x86_mem(REG(addr), 0)); // mov (addr), val
                free_reg(addr);
                arg_regs[i] = val;
            } else {
                arg_regs[i] = gen(argv[i]);
            }
        } else
            arg_regs[i] = gen(argv[i]);
#ifdef _WIN32
        // Win64: an 8-byte _Complex is passed by value in a GP register.
        // gen() returns the address; load it.
        // (Struct/union ≤8 byte funcall args already handled by gen_addr+load above.)
        if (is_complex(argv[i]->ty) && argv[i]->ty->size <= 8) {
            x86_mov_rm(cg_sec, 8, REG(arg_regs[i]), x86_mem(REG(arg_regs[i]), 0)); // movq (%reg), %reg
        }
#endif
        arg_sizes[i] = (argv[i]->ty->kind == TY_ARRAY || argv[i]->ty->kind == TY_FUNC) ? 8 : argv[i]->ty->size;
        // C default argument promotion: see the matching SysV comment
        // above -- a genuine C99 vararg float (e.g. printf("%f", f))
        // must promote to double the same as an oldstyle no-prototype
        // call's float argument does.
        if ((is_oldstyle || (is_variadic && i >= named_count)) && arg_sizes[i] == 4 && is_flonum(argv[i]->ty))
            arg_sizes[i] = 8; // oldstyle or C99 vararg: float -> double promotion
        arg_is_float[i] = is_flonum(argv[i]->ty);
        // Win64: XMM register N matches GP register N (rcx/xmm0, rdx/xmm1, r8/xmm2, r9/xmm3)
        arg_fp_idx[i] = i + (has_hidden_retbuf ? 1 : 0);
    }

    if (stack_reserve > 0 && (!call_target || call_target != bi_s_alloca))
        x86_sub_ri(cg_sec, 8, X86_RSP, stack_reserve); // subq $stack_reserve, %rsp

    for (int i = nargs - 1; i >= reg_nargs; i--) {
        // Win64: large structs (>8 bytes) are passed by pointer on the stack
        VReg r;
        if ((argv[i]->ty->kind == TY_STRUCT || argv[i]->ty->kind == TY_UNION || is_complex(argv[i]->ty)) && argv[i]->ty->size > 8)
            r = gen_addr(argv[i]);
        else if (argv[i]->ty && (argv[i]->ty->kind == TY_STRUCT || argv[i]->ty->kind == TY_UNION) && argv[i]->ty->size <= 8) {
            int addr = gen_addr(argv[i]);
            if (addr >= 0) {
                int load_sz = argv[i]->ty->size <= 4 ? 4 : 8;
                r = alloc_reg();
                x86_mov_rm(cg_sec, load_sz, REG(r), x86_mem(REG(addr), 0)); // mov (addr), r
                free_reg(addr);
            } else {
                r = gen(argv[i]);
            }
        } else
            r = gen(argv[i]);
        int off = shadow_space + (i - reg_nargs) * 8; // skip 32-byte home space
        if (argv[i]->ty && argv[i]->ty->kind == TY_FLOAT && argv[i]->ty->size == 4 &&
            !((is_oldstyle || (is_variadic && i >= named_count)))) {
            // ABI: same narrow-before-crossing requirement as the
            // register-arg path above -- for a Win64 call where this
            // float argument overflows past the 4 combined GP/XMM arg
            // slots onto the stack, r still holds the value
            // bit-reinterpreted as a double (rcc's internal GP-reg
            // representation); narrow it to a genuine float before
            // writing it to the stack slot the callee's prologue reads
            // via a plain movss.
            asm_movq_r_xmm(cg_sec, 0, r); // movq r, %xmm0
            x86_cvtsd2ss(cg_sec, X86_XMM0, X86_XMM0); // cvtsd2ss %xmm0, %xmm0
            x86_movss_mr(cg_sec, x86_mem(X86_RSP, off), X86_XMM0); // movss %xmm0, off(%rsp)
        } else if (is_flonum(argv[i]->ty)) {
            x86_mov_mr(cg_sec, 8, x86_mem(X86_RSP, off), REG(r)); // movq reg, off(%rsp)
        } else if (argv[i]->ty && (argv[i]->ty->kind == TY_PTR || argv[i]->ty->kind == TY_ARRAY || argv[i]->ty->kind == TY_FUNC)) {
            x86_mov_mr(cg_sec, 8, x86_mem(X86_RSP, off), REG(r)); // movq reg, off(%rsp)
        } else if (is_complex(argv[i]->ty) && argv[i]->ty->size <= 8) {
            // Win64: an 8-byte _Complex is passed by value on the stack.
            // gen() returns the address of the complex value; load it.
            x86_mov_rm(cg_sec, 8, X86_RAX, x86_mem(REG(r), 0)); // movq (reg), %rax
            x86_mov_mr(cg_sec, 8, x86_mem(X86_RSP, off), X86_RAX); // movq %rax, off(%rsp)
        } else {
            if (argv[i]->ty->size == 1)
                asm_movzx(cg_sec, r, r, 4, 1); // movzx4->r rr, rr
            else if (argv[i]->ty->size == 4)
                asm_mov_reg_reg(cg_sec, r, r, 4); // mov rr -> rr
            x86_mov_mr(cg_sec, 8, x86_mem(X86_RSP, off), REG(r)); // movq reg, off(%rsp)
        }
        free_reg(r);
    }
#else
    // When more args need scratch registers than exist, evaluating them all up
    // front would exhaust the register set and force colliding per-register
    // spills.  Stage each evaluated value into a unique stack slot instead, then
    // reload right before placement.  Mirrors the golden codegen logic.
    for (int i = 0; i < nargs; i++)
        arg_stage[i] = 0;
    int nreg_args_count = 0;
    for (int i = 0; i < nargs; i++)
        if (arg_stack_idx[i] < 0)
            nreg_args_count++;
    // Registers already live here (hidden retbuf address, indirect-call
    // target, outer expression temps) shrink the budget, and the stack-arg
    // pass below needs scratch registers of its own. Without headroom the
    // per-register spill slots collide (two borrows of the same register
    // share one slot) and a staged value is silently lost.
    //
    // A single scratch register is NOT always enough: an argument that is
    // itself a non-trivial expression — e.g. `flags | (cond ? A : B)`, a
    // binary op whose rhs is a ternary, or a call whose own argument is
    // itself a nested function call — needs 2+ registers live at once at
    // its peak (its own result register, held across evaluating its
    // operands, plus a nested sub-expression's own result register).
    // Budgeting only 1 (or, when every argument happens to be register-
    // passed with none spilled to the stack, budgeting 0) leaves the
    // computation exactly at NUM_REGS with zero headroom: alloc_reg() must
    // spill a register that's still logically owned by an earlier
    // register-passed argument, and a subsequent *fresh* (non-spilling)
    // allocation elsewhere in that later argument's own nested evaluation
    // can then silently reclaim that same "temporarily free" register
    // before its true owner ever reads it back — no spill/restore
    // involved, so nothing notices. This is not limited to calls with
    // trailing stack arguments: reserving scratch only when
    // `nreg_args_count < nargs` misses calls where every argument is
    // register-passed (fits within the 6 GP argument registers) but a
    // later one is still a deep nested-call chain. Observed for real via
    // two distinct crashes: Perl's `pp_regcomp()` — an 8-argument
    // indirect call, `flags` and a ternary spilled to the stack — losing
    // its register-passed `&is_bare_re` argument; and Perl's
    // `Perl_utilize()` — a 5-argument *direct* call, `newATTRSUB(floor,
    // newSVOP(...), NULL, NULL, <deeply nested op_append_elem/newSTATEOP
    // chain>)`, no stack arguments at all — losing its 2nd (register)
    // argument while evaluating its 5th. Reserving scratch headroom
    // whenever a call has more than one argument (so there is always a
    // "later" argument whose own complexity could exceed a 1-register
    // budget) keeps both off the knife's edge; deeper nesting still
    // possible with 2+ live scratch registers of its own is the
    // allocator's pre-existing single-slot-per-register limitation, not
    // something this fix set out to solve.
    int live_now = 0;
    for (int i = 0; i < NUM_REGS; i++)
        live_now += (used_regs >> i) & 1;
    int stack_scratch = (nargs > 1) ? 2 : 0;
    bool use_staging = (live_now + nreg_args_count + stack_scratch >= NUM_REGS);

    // Bitmask of scratch registers holding already-computed register-passed
    // arguments. Built up incrementally as each arg is evaluated below (an
    // arg that gets staged to memory is NOT added: its register is legitimately
    // free again once staged). Reasserted as `used_regs` before evaluating each
    // subsequent argument — and again before the stack-arg pass further below —
    // so a later argument's own nested alloc_reg()/free_reg() churn can never
    // silently reclaim a register that an earlier argument is still holding.
    //
    // Without this, an earlier register argument's value can vanish with no
    // spill/restore involved at all: evaluating a later argument that is
    // itself a non-trivial call chain (or a stack argument's own ternary)
    // may legitimately spill-then-restore the earlier register several times
    // via alloc_reg()/free_reg() under pressure — each restore leaves it
    // genuinely free again until this loop's owner comes back for it — and a
    // subsequent *fresh* (non-spilling) allocation elsewhere in that same
    // nested evaluation can then claim it first. Observed for real via
    // Perl's Perl_utilize(): `newATTRSUB(floor, newSVOP(...), NULL, NULL,
    // <deeply nested op_append_elem/newSTATEOP call chain>)` silently
    // corrupted its 2nd argument (the BEGIN-name op returned by the earlier
    // newSVOP() call) while evaluating its 5th, unrelated one.
    int reg_arg_mask = 0;

    for (int i = 0; i < nargs; i++) {
        if (arg_stack_idx[i] >= 0)
            continue;
        used_regs |= reg_arg_mask; // protect args already computed by earlier iterations
        if (is_complex(argv[i]->ty) || ((argv[i]->ty->kind == TY_STRUCT || argv[i]->ty->kind == TY_UNION) && argv[i]->ty->size > 8)) {
            int addr = gen_addr(argv[i]);
            if (addr < 0) {
                // Non-lvalue (e.g. cast to struct): allocate temp, evaluate, store
                int alloc = (argv[i]->ty->size + 7) & ~7;
                fn_struct_ret_off += alloc;
                if (fn_struct_ret_off > fn_struct_ret_total)
                    fn_struct_ret_total = fn_struct_ret_off;
                int tmp_slot = current_fn_stack_size + fn_struct_ret_off;
                addr = alloc_reg();
                asm_lea_rbp_reg(cg_sec, addr, 8, tmp_slot); // lea [rbp-8], raddr
                VReg val = gen(argv[i]);
                asm_mov_mem_via_reg(cg_sec, val, addr, argv[i]->ty->size > 4 ? 8 : argv[i]->ty->size); // mov val, (addr)
                free_reg(val);
            }
            arg_regs[i] = addr;
        } else if (argv[i]->ty && (argv[i]->ty->kind == TY_STRUCT || argv[i]->ty->kind == TY_UNION) && argv[i]->ty->size <= 8) {
            // Small struct passed by value in a GP register.
            // gen() for ND_DEREF / ND_MEMBER may return the address, not the value.
            int addr = gen_addr(argv[i]);
            if (addr >= 0) {
                int load_sz = argv[i]->ty->size <= 4 ? 4 : 8;
                VReg val = alloc_reg();
                x86_mov_rm(cg_sec, load_sz, REG(val), x86_mem(REG(addr), 0)); // mov (addr), val
                free_reg(addr);
                arg_regs[i] = val;
            } else {
                arg_regs[i] = gen(argv[i]);
            }
        } else
            arg_regs[i] = gen(argv[i]);

        if (use_staging) {
            fn_struct_ret_off += 8;
            if (fn_struct_ret_off > fn_struct_ret_total)
                fn_struct_ret_total = fn_struct_ret_off;
            arg_stage[i] = current_fn_stack_size + fn_struct_ret_off;
            asm_mov_reg_rbp(cg_sec, arg_regs[i], 8, arg_stage[i]); // movq arg_regs[i], -arg_stage[i](%rbp)
            // The arg is safely archived in its staging slot.  Release the
            // register so the marshal can reuse it, but do NOT clear the
            // spilled_regs bit — an alloc_reg() that spilled an outer
            // expression's live value to give us this register left its
            // data in spill_offset(); clearing the bit orphans that data.
            used_regs &= ~(1 << arg_regs[i]);
            reg_owner[arg_regs[i]] = NULL;
            arg_regs[i] = -1;
        } else {
            reg_arg_mask |= (1 << arg_regs[i]);
        }
    }

    if (stack_reserve > 0 && (!call_target || call_target != bi_s_alloca))
        x86_sub_ri(cg_sec, 8, X86_RSP, stack_reserve); // subq $stack_reserve, %rsp

    for (int i = nargs - 1; i >= 0; i--) {
        if (arg_stack_idx[i] < 0)
            continue;
        used_regs |= reg_arg_mask; // keep pre-computed reg args live
        if (argv[i]->ty->kind == TY_LDOUBLE) {
            VReg r = gen(argv[i]);
            x86_mov_mr(cg_sec, 8, x86_mem(X86_RSP, shadow_space + arg_stack_idx[i] * 8), REG(r)); // movq %s, %d(%rsp)
            X86Mem mld = x86_mem(X86_RSP, shadow_space + arg_stack_idx[i] * 8); // fldl %d(%rsp)
            x86_fldl_m(cg_sec, mld); // fldl %d(%rsp)
            x86_fstpt_m(cg_sec, mld); // fstpt %d(%rsp)
            free_reg(r);
            used_regs |= reg_arg_mask; // restore any bits cleared by free_reg
            continue;
        }
        VReg r;
        if (argv[i]->ty && is_int128_like(argv[i]->ty)) {
            r = gen(argv[i]); // returns address of 16-byte slot
            x86_mov_rm(cg_sec, 8, X86_RAX, x86_mem(REG(r), 0)); // movq (%s), %%rax
            x86_mov_mr(cg_sec, 8, x86_mem(X86_RSP, shadow_space + arg_stack_idx[i] * 8), X86_RAX); // movq %%rax, ...
            x86_mov_rm(cg_sec, 8, X86_RAX, x86_mem(REG(r), 8)); // movq 8(%s), %%rax
            x86_mov_mr(cg_sec, 8, x86_mem(X86_RSP, shadow_space + (arg_stack_idx[i] + 1) * 8), X86_RAX); // movq %%rax, ...
            free_reg(r);
            used_regs |= reg_arg_mask;
            continue;
        }
        if (is_complex(argv[i]->ty) || ((argv[i]->ty->kind == TY_STRUCT || argv[i]->ty->kind == TY_UNION) && argv[i]->ty->size > 8))
            r = gen_addr(argv[i]);
        else
            r = gen(argv[i]);
        if (is_complex(argv[i]->ty)) {
            int sz = argv[i]->ty->size;
            int base_sz = argv[i]->ty->base->size;
            x86_mov_rm(cg_sec, 8, X86_RAX, x86_mem(REG(r), 0)); // movq (%s), %%rax
            x86_mov_mr(cg_sec, 8, x86_mem(X86_RSP, shadow_space + arg_stack_idx[i] * 8), X86_RAX); // movq %%rax, ...
            if (sz > 8) {
                x86_mov_rm(cg_sec, 8, X86_RAX, x86_mem(REG(r), base_sz)); // movq base_sz(%s), %%rax
                x86_mov_mr(cg_sec, 8, x86_mem(X86_RSP, shadow_space + (arg_stack_idx[i] + 1) * 8), X86_RAX); // movq %%rax, ...
            }
        } else if (argv[i]->ty->kind == TY_FLOAT && arg_sizes[i] == 4) {
            // ABI: same narrow-before-crossing requirement as the
            // register-arg path above, for the >6-float-args (SysV) /
            // >4-float-args (Win64) stack overflow case: r holds the
            // value bit-reinterpreted as a double: narrow to a genuine
            // float before writing it to the argument's stack slot, or
            // the callee's movss read of the low 4 bytes gets garbage.
            asm_movq_r_xmm(cg_sec, 0, r); // movq r, %xmm0
            x86_cvtsd2ss(cg_sec, X86_XMM0, X86_XMM0); // cvtsd2ss %xmm0, %xmm0
            x86_movss_mr(cg_sec, x86_mem(X86_RSP, shadow_space + arg_stack_idx[i] * 8), X86_XMM0); // movss %xmm0, %d(%rsp)
        } else if (is_flonum(argv[i]->ty)) {
            x86_mov_mr(cg_sec, 8, x86_mem(X86_RSP, shadow_space + arg_stack_idx[i] * 8), REG(r)); // movq %s, %d(%rsp)
        } else if (argv[i]->ty->kind == TY_PTR || argv[i]->ty->kind == TY_ARRAY || argv[i]->ty->kind == TY_FUNC) {
            x86_mov_mr(cg_sec, 8, x86_mem(X86_RSP, shadow_space + arg_stack_idx[i] * 8), REG(r)); // movq %s, %d(%rsp)
        } else {
            // Structs/unions <=8 bytes already loaded as full 8-byte register;
            // sign/zero extending by size would corrupt bytes beyond size.
            if (argv[i]->ty->kind != TY_STRUCT && argv[i]->ty->kind != TY_UNION) {
                if (argv[i]->ty->is_unsigned)
                    zero_extend_to(r, argv[i]->ty->size, 8);
                else
                    sign_extend_to(r, argv[i]->ty->size, 8);
            }
            x86_mov_mr(cg_sec, 8, x86_mem(X86_RSP, shadow_space + arg_stack_idx[i] * 8), REG(r)); // movq r, off(%rsp)
        }
        free_reg(r);
        used_regs |= reg_arg_mask; // restore any bits cleared by free_reg
    }
#endif

    int xmm_args = 0;
#ifdef _WIN32
    for (int i = 0; i < reg_nargs; i++) {
        if (arg_is_float[i]) xmm_args++;
    }
#else
    xmm_args = fp_reg_args;
#endif

    int temp_ret_reg = -1;
    int temp_ret_slot = 0;
    bool need_temp_ret = false;
    if (has_hidden_retbuf) {
        if (hidden_ret_reg == -1) {
            // Defer allocating the result register until after the call:
            // alloc_reg() here can spill-borrow a register that still stages
            // an argument value (e.g. the 8th FP arg in rsi), which the
            // staging moves below would then read as garbage. Compute the
            // buffer address directly into the ABI register instead.
            need_temp_ret = true;
            int alloc = (node->ty->size + 15) & ~15;
            fn_struct_ret_off += alloc;
            if (fn_struct_ret_off > fn_struct_ret_total)
                fn_struct_ret_total = fn_struct_ret_off;
            temp_ret_slot = current_fn_stack_size + fn_struct_ret_off;
            asm_lea_rbp(cg_sec, cg_x86_argreg[0], 8, temp_ret_slot); // lea -temp_ret_slot(%rbp), argreg64[0]
        } else
            x86_mov_rr(cg_sec, 8, cg_x86_argreg[0], REG(hidden_ret_reg)); // mov hidden_ret_reg, argreg64[0]
#ifdef _WIN32
        // Store hidden retbuf register to shadow space so variadic callees can find it
        if (!call_target || strcmp(call_target, "alloca") != 0)
            x86_mov_mr(cg_sec, 8, x86_mem(X86_RSP, 0), cg_x86_argreg[0]); // movq %rcx, 0(%rsp)
#endif
    }

#ifdef _WIN32
    for (int i = 0; i < reg_nargs; i++) {
        int argi = i + (has_hidden_retbuf ? 1 : 0);
        materialize_reg(arg_regs[i]); // see the SysV placement loop below
        if (arg_is_float[i]) {
            asm_movq_r_xmm(cg_sec, arg_fp_idx[i], arg_regs[i]); // movq arg_regs[i], %xmm{fp_idx}
            // ABI: a `float`-typed argument crosses into the callee as a
            // genuine 32-bit value in the low bits of the XMM register,
            // never rcc's internal double-widened representation (the GP
            // register above holds the float value bit-reinterpreted as a
            // double -- see "Float values are always stored as double in
            // GP regs" below). This must narrow for EVERY float-typed
            // argument, not just is_builtin_call cases lowered from a
            // __builtin_*f -- any call whose callee may be non-rcc-compiled
            // code (libm, any linked .so/.a) needs the real ABI's narrow
            // float, or the callee reads the low 32 bits of a double bit
            // pattern (garbage) instead of the intended value. Previously
            // gated on is_builtin_call, this left every ordinary call with
            // a `float`-typed parameter (sinf, cosf, sqrtf, any C API
            // taking `float`) broken across the ABI boundary.
            // arg_sizes[i] stays 4 for a genuine float parameter; the
            // variadic-promotion logic above bumps it to 8 (promoting to
            // double per C's default argument promotion) WITHOUT changing
            // argv[i]->ty itself -- narrowing here would wrongly truncate
            // an intentionally-promoted vararg float (e.g. printf("%f", f))
            // back down to 32 bits.
            if (argv[i]->ty && argv[i]->ty->kind == TY_FLOAT && arg_sizes[i] == 4)
                x86_cvtsd2ss(cg_sec, (X86XmmReg)arg_fp_idx[i], (X86XmmReg)arg_fp_idx[i]);
            x86_mov_rr(cg_sec, 8, cg_x86_argreg[argi], REG(arg_regs[i])); // movq %s, 0(%rsp)
        } else if (arg_sizes[i] == 1) {
            if (argv[i]->ty && !argv[i]->ty->is_unsigned)
                x86_movsx(cg_sec, 4, 1, cg_x86_argreg[argi], REG(arg_regs[i])); // movsbl %s, %s
            else
                x86_movzx(cg_sec, 4, 1, cg_x86_argreg[argi], REG(arg_regs[i])); // movzbl %s, %s
        } else if (arg_sizes[i] == 4) {
            if (argv[i]->ty->is_unsigned)
                x86_mov_rr(cg_sec, 4, cg_x86_argreg[argi], REG(arg_regs[i])); // movswl %s, %s
            else
                x86_movsx(cg_sec, 8, 4, cg_x86_argreg[argi], REG(arg_regs[i])); // movzwl %s, %s
        } else {
            x86_mov_rr(cg_sec, 8, cg_x86_argreg[argi], REG(arg_regs[i])); // mov %s, %s
        }
        // Also store to shadow space so variadic callees can find args via va_list
        if (!call_target || call_target != bi_s_alloca)
            x86_mov_mr(cg_sec, 8, x86_mem(X86_RSP, argi * 8), cg_x86_argreg[argi]); // movq %reg, argi*8(%rsp)
        free_reg(arg_regs[i]);
    }
#else
    // Two-pass placement: reg64[7]=="%rsi"==argreg64[1], so any arg whose scratch
    // register is rsi must be placed before the arg that writes to rsi, otherwise
    // the write clobbers the source value.  Pass 0: rsi-sourced args.  Pass 1: rest.
    // With staging, values are in unique stack slots so no ordering conflict
    // exists; a single pass suffices, reloading each staged value first.
    for (int pass = 0; pass < (use_staging ? 1 : 2); pass++) {
        for (int i = 0; i < nargs; i++) {
            if (arg_stack_idx[i] >= 0)
                continue;
            if (use_staging && arg_stage[i]) {
                arg_regs[i] = alloc_reg();
                asm_mov_rbp_reg(cg_sec, arg_regs[i], 8, arg_stage[i]); // movq -arg_stage[i](%rbp), arg_regs[i]
                // Value is fresh from the staging slot — skip
                // materialize_reg: any spilled_regs bit the register
                // carries is from an outer expression's spill, not from
                // this argument's own pre-spill state.  Loading from the
                // spill slot would overwrite the staging-loaded value.
            } else {
                // A register argument computed earlier can have been
                // spilled by alloc_reg() while evaluating a later
                // argument under register pressure — materialize it
                // before reading REG(arg_regs[i]).
                materialize_reg(arg_regs[i]);
            }
            if (argv[i]->ty && is_int128_like(argv[i]->ty)) {
                // lo in argreg64[arg_gp_idx[i]], hi in argreg64[arg_gp_idx[i]+1]
                x86_mov_rm(cg_sec, 8, cg_x86_argreg[arg_gp_idx[i]], x86_mem(REG(arg_regs[i]), 0)); // movq (%s), %s
                x86_mov_rm(cg_sec, 8, cg_x86_argreg[arg_gp_idx[i] + 1], x86_mem(REG(arg_regs[i]), 8)); // movq 8(%s), %s
            } else if (arg_is_float[i]) {
                asm_movq_r_xmm(cg_sec, arg_fp_idx[i], arg_regs[i]); // movq arg_regs[i], %%xmm{fp_idx}
                // ABI: a `float`-typed argument crosses into the callee as
                // a genuine 32-bit value in the low bits of the XMM
                // register, never rcc's internal double-widened
                // representation (the GP register above holds the float
                // value bit-reinterpreted as a double -- see "Float values
                // are always stored as double in GP regs" below). Narrow
                // for EVERY float-typed argument, not just is_builtin_call
                // cases lowered from a __builtin_*f -- any call whose
                // callee may be non-rcc-compiled code (libm, any linked
                // .so/.a) needs the real ABI's narrow float, or the callee
                // reads the low 32 bits of a double bit pattern (garbage)
                // instead of the intended value. Previously gated on
                // is_builtin_call, this left every ordinary call with a
                // `float`-typed parameter (sinf, cosf, sqrtf, any C API
                // taking `float`) broken across the ABI boundary.
                // arg_sizes[i] stays 4 for a genuine float parameter; the
                // variadic-promotion logic above bumps it to 8 (promoting
                // to double per C's default argument promotion) WITHOUT
                // changing argv[i]->ty itself -- narrowing here would
                // wrongly truncate an intentionally-promoted vararg float
                // (e.g. printf("%f", f)) back down to 32 bits.
                if (argv[i]->ty && argv[i]->ty->kind == TY_FLOAT && arg_sizes[i] == 4)
                    x86_cvtsd2ss(cg_sec, (X86XmmReg)arg_fp_idx[i], (X86XmmReg)arg_fp_idx[i]);
            } else if (argv[i]->ty && is_complex(argv[i]->ty)) {
                bool cfloat = argv[i]->ty->base && is_flonum(argv[i]->ty->base);
                if (cfloat) {
                    // Float complex: load real and imag into xmm regs.
                    // Each component is 8 bytes for float/double complex.
                    int base_sz = argv[i]->ty->base ? argv[i]->ty->base->size : 8;
                    if (argv[i]->ty->size <= 8) {
                        // _Complex float: 8 bytes packed -> one xmm
                        x86_movsd_rm(cg_sec, (X86XmmReg)arg_fp_idx[i], x86_mem(REG(arg_regs[i]), 0));
                    } else {
                        // _Complex double: 16 bytes -> two xmm
                        x86_movsd_rm(cg_sec, (X86XmmReg)arg_fp_idx[i], x86_mem(REG(arg_regs[i]), 0));
                        x86_movsd_rm(cg_sec, (X86XmmReg)(arg_fp_idx[i] + 1), x86_mem(REG(arg_regs[i]), base_sz));
                    }
                } else {
                    // Integer complex
                    x86_mov_rm(cg_sec, 8, cg_x86_argreg[arg_gp_idx[i]], x86_mem(REG(arg_regs[i]), 0)); // mov %s, %s
                    if (argv[i]->ty->size > 8) {
                        int base_sz = argv[i]->ty->base ? argv[i]->ty->base->size : 8;
                        x86_mov_rm(cg_sec, 8, cg_x86_argreg[arg_gp_idx[i] + 1], x86_mem(REG(arg_regs[i]), base_sz));
                    }
                }
            } else if (arg_sizes[i] == 1) {
                if (argv[i]->ty && !argv[i]->ty->is_unsigned)
                    x86_movsx(cg_sec, 4, 1, cg_x86_argreg[arg_gp_idx[i]], REG(arg_regs[i])); // movsbl %s, %s
                else
                    x86_movzx(cg_sec, 4, 1, cg_x86_argreg[arg_gp_idx[i]], REG(arg_regs[i])); // movzbl %s, %s
            } else if (arg_sizes[i] == 2) {
                if (argv[i]->ty && !argv[i]->ty->is_unsigned)
                    x86_movsx(cg_sec, 4, 2, cg_x86_argreg[arg_gp_idx[i]], REG(arg_regs[i])); // movswl %s, %s
                else
                    x86_movzx(cg_sec, 4, 2, cg_x86_argreg[arg_gp_idx[i]], REG(arg_regs[i])); // movzwl %s, %s
            } else if (arg_sizes[i] == 4) {
                x86_mov_rr(cg_sec, 4, cg_x86_argreg[arg_gp_idx[i]], REG(arg_regs[i])); // mov %s, %s
            } else {
                x86_mov_rr(cg_sec, 8, cg_x86_argreg[arg_gp_idx[i]], REG(arg_regs[i])); // movslq %s, %s
            }
            // Release the staging-load register.  When it was a recycled
            // outer VReg (spilled_regs still set from an earlier spill),
            // don't restore — the outer expression needs its value in the
            // spill slot so the binary-op same-register combining can
            // recover it after the call.
            if (spilled_regs & (1 << arg_regs[i])) {
                used_regs &= ~(1 << arg_regs[i]);
                reg_owner[arg_regs[i]] = NULL;
            } else {
                free_reg(arg_regs[i]);
            }
        }
    } // end two-pass
#endif

    x86_mov_ri(cg_sec, 4, X86_RAX, xmm_args); // movl $xmm_args, %eax
    if (call_target) {
        if (node->lhs && node->lhs->var && node->lhs->var->is_nested_fn) {
            // GNU nested function, direct call: load the static-chain
            // pointer into the physical chain register (%r10) right
            // before the call, bypassing the VReg allocator — exactly
            // like argument registers above. depth==0: the callee is
            // declared directly in the function currently being
            // compiled, so the chain value is simply this frame's own
            // rbp. depth>0: the callee shares (or is) an ancestor's
            // scope one level up from here, so forward this function's
            // OWN incoming chain slot unchanged (covers self-recursion
            // and sibling nested functions calling each other).
            if (node->lhs->chain_depth == 0) {
                x86_mov_rr(cg_sec, 8, X86_R10, X86_RBP); // mov rbp, r10
            } else {
                X86Mem chain_m = {CG_X86_FP, X86_NOREG, 1, -CHAIN_SLOT_OFFSET};
                x86_mov_rm(cg_sec, 8, X86_R10, chain_m); // mov -CHAIN_SLOT_OFFSET(rbp), r10
            }
        }
        if (call_target == bi_s_alloca) {
            alloca_needed = true;
            fn_uses_alloca = true;
            emit_direct_call("__rcc_alloca", false);
        } else {
            emit_direct_call(call_target, is_asm_call);
        }
    } else {
        // callee_reg was computed before any argument (register or stack)
        // was evaluated and is held live across all of it; the same
        // register-stolen-by-a-later-alloc_reg() hazard fixed above for
        // arg_regs[] applies here too.
        materialize_reg(callee_reg);
        asm_call_reg(cg_sec, callee_reg); // call rcallee_reg
        free_reg(callee_reg);
    }

    if (stack_reserve > 0 && (!call_target || call_target != bi_s_alloca))
        x86_add_ri(cg_sec, 8, X86_RSP, stack_reserve); // addq $stack_reserve, %rsp

    if (saved_scratch & 2) {
        used_regs |= 2;
        asm_mov_rbp(cg_sec, X86_R11, 8, spill_offset(1)); // mov [rbp-spill_offset(1], X86_R11)
    }
    if (saved_scratch & 1) {
        used_regs |= 1;
        asm_mov_rbp(cg_sec, X86_R10, 8, spill_offset(0)); // mov [rbp-spill_offset(0], X86_R10)
    }

    if (has_hidden_retbuf) {
        if (need_temp_ret) {
            // Allocate the result register now that the staged arg registers
            // are free again, and derive the frame-relative buffer address.
            temp_ret_reg = alloc_reg();
            hidden_ret_reg = temp_ret_reg;
            asm_lea_rbp_reg(cg_sec, temp_ret_reg, 8, temp_ret_slot); // lea [rbp-8], rtemp_ret_reg
        }
        if ((saved_scratch & 1) && (!has_hidden_retbuf || hidden_ret_reg != 0)) {
            used_regs |= 1;
            asm_mov_rbp(cg_sec, X86_R10, 8, spill_offset(0)); // mov [rbp-spill_offset(0], X86_R10
        }
        int ret = temp_ret_reg != R_NONE ? temp_ret_reg : hidden_ret_reg;
#ifdef _WIN32
        // Win64: structs ≤8 bytes returned by value in RAX (not via pointer).
        // The callee returns the struct value in RAX; store it into the
        // destination buffer pointed to by ret.
        if (node->ty->size <= 8 && node->ty->size > 0) {
            int sz = node->ty->size <= 4 ? 4 : 8;
            x86_mov_mr(cg_sec, sz, x86_mem(REG(ret), 0), X86_RAX); // movq %rax, (ret)
        }
#endif
        return ret;
    }

#ifdef _WIN32
    // Win64: structs ≤8 bytes returned by value in RAX (no hidden retbuf).
    // The gen() caller expects an address; return a temp slot containing RAX.
    if (node->ty && (node->ty->kind == TY_STRUCT || node->ty->kind == TY_UNION || is_complex(node->ty)) && node->ty->size <= 8) {
        int addr = hidden_ret_reg != -1 ? hidden_ret_reg : alloc_int128_addr();
        int sz = node->ty->size <= 4 ? 4 : 8;
        x86_mov_mr(cg_sec, sz, x86_mem(REG(addr), 0), X86_RAX); // mov rax, (addr)
        return addr;
    }
#endif
#if !defined(_WIN32) && !defined(ARCH_ARM64)
    // Linux/SysV x86-64: a struct/union <=16 bytes with no floating member
    // anywhere (struct_returns_in_gp_regs) returns in RAX (first eightbyte)
    // and, if bigger than 8 bytes, RDX (second eightbyte) -- no hidden
    // pointer. Mirrors the has_hidden_retbuf classification above.
    if (node->ty && struct_returns_in_gp_regs(node->ty)) {
        int addr = hidden_ret_reg != -1 ? hidden_ret_reg : alloc_int128_addr();
        int lo_sz = node->ty->size <= 4 ? 4 : 8;
        x86_mov_mr(cg_sec, lo_sz, x86_mem(REG(addr), 0), X86_RAX); // mov %rax, (addr)
        if (node->ty->size > 8) {
            int hi_sz = node->ty->size - 8 <= 4 ? 4 : 8;
            x86_mov_mr(cg_sec, hi_sz, x86_mem(REG(addr), 8), X86_RDX); // mov %rdx, 8(addr)
        }
        return addr;
    }
#endif
#ifndef _WIN32
    // Linux SysV: _Complex double returns in xmm0:xmm1,
    // _Complex float returns in xmm0 (packed as two singles).
    if (node->ty && is_complex(node->ty)) {
        int addr = hidden_ret_reg != -1 ? hidden_ret_reg : alloc_int128_addr();
        int base_sz = node->ty->base ? node->ty->base->size : 8;
        if (node->ty->size <= 8) {
            x86_movsd_mr(cg_sec, x86_mem(REG(addr), 0), X86_XMM0);
        } else {
            x86_movsd_mr(cg_sec, x86_mem(REG(addr), 0), X86_XMM0);
            x86_movsd_mr(cg_sec, x86_mem(REG(addr), base_sz), X86_XMM1);
        }
        return addr;
    }
#endif
    // int128 / _BitInt(65..128) return: spill rax:rdx (x86-64) or x0:x1 (ARM64) to a 16-byte slot
    if (node->ty && is_int128_like(node->ty)) {
        int addr = alloc_int128_addr();
#ifdef ARCH_ARM64
        asm_str_reg_off_phy(cg_sec, ARM64_X0, addr, 8, 0); // str x0, [addr]
        asm_str_reg_off_phy(cg_sec, ARM64_X1, addr, 8, 1); // str x1, [addr, #8]
#else
        x86_mov_mr(cg_sec, 8, x86_mem(REG(addr), 0), X86_RAX); // movq %rax, (reg)
        x86_mov_mr(cg_sec, 8, x86_mem(REG(addr), 8), X86_RDX); // movq %rdx, 8(reg)
#endif
        return addr;
    }
    VReg r = alloc_reg();
    if (node->ty && is_flonum(node->ty)) {
#ifndef ARCH_ARM64
        if (node->ty->kind == TY_LDOUBLE) {
            // SysV x86-64 ABI: a `long double` (80-bit x87 extended)
            // return value comes back in ST0, never XMM0/XMM1 like
            // float/double -- the generic is_flonum path below (movq
            // %xmm0, rr) silently read whatever stale/unrelated value
            // happened to be sitting in XMM0 instead. Pop ST0 via
            // `fstpl`, rounding to double precision and writing through
            // a red-zone scratch slot -- matching rcc's own internal
            // GP-register representation for long double values
            // elsewhere (see the ND_ASSIGN long double store path,
            // "Store long double as 64-bit double (truncated)" above),
            // so the result composes correctly with every other
            // long-double consumer.
            X86Mem scratch = x86_mem(X86_RSP, -8);
            x86_fstpl_m(cg_sec, scratch); // fstpl -8(%rsp)
            x86_mov_rm(cg_sec, 8, REG(r), scratch); // movq -8(%rsp), rr
        } else {
            if (node->ty->kind == TY_FLOAT)
                asm_cvtss2sd(cg_sec); // cvtss2sd %xmm0, %xmm0
            asm_movq_xmm_r(cg_sec, r, X86_XMM0); // movq %xmm0, rr
        }
#endif
    } else {
        asm_mov_retval(cg_sec, r, 8); // movq %rax, rr
    }
    return r;
#endif
}


static int add_float_literal(double val, int size) {
    FloatLit *fl = arena_alloc(sizeof(FloatLit));
    if (cg_dry_run) return 0;
    fl->id = float_lit_count++;
    fl->val = val;
    fl->size = size;
    fl->next = float_lits;
    float_lits = fl;
    return fl->id;
}

// Names that GAS treats as keywords in Intel syntax (segment registers and
// expression operators).  Global symbols with these names need a safe alias.
static bool is_asm_reserved(const char *name) {
    // Dispatch on the first byte before comparing: this runs on every variable
    // reference and call target in codegen, and almost every real identifier is
    // rejected by its first byte alone.
    static const char *reg_kw[] = {
        "%al", "%ah", "%ax", "%eax", "%rax",
        "%bl", "%bh", "%bx", "%ebx", "%rbx",
        "%cl", "%ch", "%cx", "%ecx", "%rcx",
        "%dl", "%dh", "%dx", "%edx", "%rdx",
        "%sil", "%si", "%esi", "%rsi",
        "%dil", "%di", "%edi", "%rdi",
        "%bpl", "%bp", "%ebp", "%rbp",
        "%spl", "%sp", "%esp", "%rsp",
        "%r8", "%r8b", "%r8w", "%r8d",
        "%r9", "%r9b", "%r9w", "%r9d",
        "%r10", "%r10b", "%r10w", "%r10d",
        "%r11", "%r11b", "%r11w", "%r11d",
        "%r12", "%r12b", "%r12w", "%r12d",
        "%r13", "%r13b", "%r13w", "%r13d",
        "%r14", "%r14b", "%r14w", "%r14d",
        "%r15", "%r15b", "%r15w", "%r15d",
        NULL};
    switch (name[0]) {
    case '%':
        for (int i = 0; reg_kw[i]; i++)
            if (strcmp(name, reg_kw[i]) == 0) return true;
        return false;
    // segment registers
    case 'c': return name[1] == 's' && !name[2];
    case 'd': return name[1] == 's' && !name[2];
    case 'e': return name[1] == 's' && !name[2];
    case 'f': return name[1] == 's' && !name[2];
    case 'g': return name[1] == 's' && !name[2];
    // "ss", plus the "shl"/"shr" mnemonics
    case 's':
        if (name[1] == 's') return !name[2];
        return name[1] == 'h' && (name[2] == 'l' || name[2] == 'r') && !name[3];
    case 'a': return name[1] == 'n' && name[2] == 'd' && !name[3];
    case 'o': return name[1] == 'r' && !name[2];
    case 'n': return name[1] == 'o' && name[2] == 't' && !name[3];
    case 'x': return name[1] == 'o' && name[2] == 'r' && !name[3];
    default: return false;
    }
}

#ifndef ARCH_ARM64
static char *var_label(LVar *var) {
    if (var->asm_name) return var->asm_name;
    if (is_asm_reserved(var->name)) return format(".L_rcc_%s", var->name);
    return var->name;
}
#endif

static char *reg(VReg r, int size) {
#ifdef ARCH_ARM64
    if (r < 0 || r > 11)
        error("invalid register %d, arm64 has only 12", r);
    if (size == 8) return reg64[r];
    return reg32[r]; // arm64: wN for 1/2/4 bytes
#else
    if (r < 0 || r > 7)
        error("invalid register %d, x86_64 has only 8", r);
    if (size == 1) return reg8[r];
    if (size == 2) return reg16[r];
    if (size == 4) return reg32[r];
    return reg64[r];
#endif
}

#ifdef ARCH_ARM64
// Emit mov reg, #imm64 handling any size (movz + movk)
void emit_mov_imm64(Arm64Reg reg, uint64_t val) {
    int sf = 1; // always 64-bit for Arm64Reg
    asm_movz(cg_sec, reg, sf, (uint16_t)(val & 0xffff), 0); // movz x{reg}, #lo16
    for (int sh = 16; sh <= 48; sh += 16) {
        if ((val >> sh) != 0) // only emit if remaining bits non-zero
            asm_movk(cg_sec, reg, sf, (uint16_t)((val >> sh) & 0xffff), sh);
    }
}

// Emit mov reg, #imm for a signed 32-bit immediate, choosing 32- or 64-bit encoding
static void emit_mov_imm(const char *reg, int imm) {
    bool is_w = (reg[0] == 'w');
    int sf = is_w ? 0 : 1;
    int rd = atoi(reg + 1);
    uint64_t val = is_w ? (uint64_t)(uint32_t)imm : (uint64_t)(int64_t)(int32_t)imm;
    asm_movz(cg_sec, rd, sf, (uint16_t)(val & 0xffff), 0); // movz x{rd}, #val
    val >>= 16;
    int shift = 16;
    int max_shift = is_w ? 16 : 48;
    while (val && shift <= max_shift) {
        asm_movk(cg_sec, rd, sf, (uint16_t)(val & 0xffff), shift); // movk x{rd}, #val, lsl #shift
        val >>= 16;
        shift += 16;
    }
}

#endif // ARCH_ARM64

// Emit adrp+add pair for label address, with platform-appropriate relocations.
// Linux:         R_AARCH64_ADR_PREL_PG_HI21 + R_AARCH64_ADD_ABS_LO12_NC  (ADRP+ADD)
// Darwin local:  same (labels starting with '.' are always in this .o)
// Darwin C syms: R_AARCH64_ADR_GOT_PAGE + R_AARCH64_LD64_GOT_LO12_NC    (ADRP+LDR via GOT)
//   Labels starting with '.' are assembler-local (.LC*, .LF*, .L.label.*) — always
//   defined in the current .o, so ADRP+ADD is correct regardless of forward refs.
//   Labels starting with '_' are C-level symbols. They may be:
//     - undefined dylib symbols: ld64 rejects ARM64_RELOC_PAGEOFF12 for these
//     - locally defined C globals: ld64 synthesises a GOT entry when GOT relocs used
//   In both cases GOT access is correct on Darwin.
#ifdef ARCH_ARM64
static void emit_adrp_add(VReg r, const char *label) {
    if (cg_dry_run) return;
    Arm64Reg rd = REG(r);
    int sidx = objfile_find_sym(cg_obj, label);
    if (sidx < 0) {
        int sec = (label[0] == '.') ? SEC_RODATA : SEC_UNDEF;
        int bind = (label[0] == '.') ? SB_LOCAL : SB_GLOBAL;
        sidx = objfile_add_sym(cg_obj, label, sec, 0, 0, bind, ST_NOTYPE);
    }
#ifdef __APPLE__
    // Darwin uses GOT for data symbols.
    // Exception: .L. flow-control labels are local text labels, use direct access.
    if (label[0] != '.' || (label[1] != 'L' || label[2] != '.')) {
        size_t adrp_off = cg_sec->len;
        asm_adrp(cg_sec, rd); // adrp xrd, label@GOTPAGE
        objfile_add_reloc(cg_obj, SEC_TEXT, adrp_off, sidx, R_AARCH64_ADR_GOT_PAGE, 0);
        size_t ldr_off = cg_sec->len;
        asm_ldr_rd_rd(cg_sec, rd); // ldr xrd, [xrd, label@GOTPAGEOFF]
        objfile_add_reloc(cg_obj, SEC_TEXT, ldr_off, sidx, R_AARCH64_LD64_GOT_LO12_NC, 0);
        return;
    }
#endif
    size_t adrp_off = cg_sec->len;
    asm_adrp(cg_sec, rd); // adrp xrd, label
    objfile_add_reloc(cg_obj, SEC_TEXT, adrp_off, sidx, R_AARCH64_ADR_PREL_PG_HI21, 0);
    size_t add_off = cg_sec->len;
    asm_add_rd_rd_0(cg_sec, rd); // add x{rd}, x{rd}, #0 (reloc placeholder)
    objfile_add_reloc(cg_obj, SEC_TEXT, add_off, sidx, R_AARCH64_ADD_ABS_LO12_NC, 0);
}

// GOT-based address load: undefined weak → NULL, defined → address.
// On Darwin, emit_adrp_add already uses GOT for undefined external symbols.
static void emit_adrp_got(VReg r, const char *label) {
    if (cg_dry_run) return;
    Arm64Reg rd = REG(r);
    int sidx = objfile_find_sym(cg_obj, label);
    if (sidx < 0) {
        int sec = (label[0] == '.') ? SEC_RODATA : SEC_UNDEF;
        int bind = (label[0] == '.') ? SB_LOCAL : SB_GLOBAL;
        sidx = objfile_add_sym(cg_obj, label, sec, 0, 0, bind, ST_NOTYPE);
    }
    if (sidx >= 0 && cg_obj->syms[sidx].bind == SB_WEAK) {
        // Weak symbols: use GOT indirection so undefined weak resolves to NULL
        size_t adrp_off = cg_sec->len;
        asm_adrp(cg_sec, rd); // adrp x{rd}, :got:label
        objfile_add_reloc(cg_obj, SEC_TEXT, adrp_off, sidx, R_AARCH64_ADR_GOT_PAGE, 0);
        size_t ldr_off = cg_sec->len;
        asm_ldr_rd_rd(cg_sec, rd); // ldr x{rd}, [x{rd}, #:got_lo12:label]
        objfile_add_reloc(cg_obj, SEC_TEXT, ldr_off, sidx, R_AARCH64_LD64_GOT_LO12_NC, 0);
    } else {
        // Non-weak (global data): compute absolute address then load from it
        emit_adrp_add(r, label);
        asm_ldr_rd_rd(cg_sec, rd); // ldr x{rd}, [x{rd}] (load value from address)
    }
}

// TLS address load.
// TLS address load.
// NOTE: On Darwin/macOS, this uses the Linux local-exec model which produces
// output that the system linker cannot handle for TLS.  Proper Darwin TLS
// requires TLV descriptors in __thread_vars and TLVP relocations, which need
// Mach-O writer support not yet implemented.
// Darwin requires __thread_vars section with TLV descriptors, __thread_data
// for initial values, $tlv$init functions, and __tlv_bootstrap thunks.
static void emit_tls_addr(VReg r, LVar *var) {
    if (cg_dry_run) return;
    Arm64Reg rd = REG(r);
    const char *label = asm_sym_name(var_sym_label(var));
    int sidx = objfile_find_sym(cg_obj, label);
    if (sidx < 0)
        sidx = objfile_add_sym(cg_obj, label, SEC_UNDEF, 0, 0, SB_GLOBAL, ST_TLS);
#if defined(__APPLE__)
    // Darwin ARM64: TLVP (General Dynamic) model via TLV descriptors
    int tlv_boot = objfile_find_sym(cg_obj, "__tlv_bootstrap");
    if (tlv_boot < 0)
        tlv_boot = objfile_add_sym(cg_obj, "__tlv_bootstrap", SEC_UNDEF, 0, 0, SB_GLOBAL, ST_FUNC);
    size_t page_off = cg_sec->len;
    arm64_adrp(cg_sec, rd, 0);
    objfile_add_reloc(cg_obj, SEC_TEXT, page_off, sidx, R_AARCH64_TLSLE_ADD_TPREL_HI12, 0);
    size_t off_off = cg_sec->len;
    arm64_ldr_uoff(cg_sec, 3, rd, rd, 0);
    objfile_add_reloc(cg_obj, SEC_TEXT, off_off, sidx, R_AARCH64_TLSLE_ADD_TPREL_LO12, 0);
    arm64_ldr_uoff(cg_sec, 3, ARM64_X16, rd, 0);
    arm64_orr_reg(cg_sec, 1, ARM64_X0, ARM64_XZR, rd, ARM64_LSL, 0); // mov x0, rd (descriptor arg)
    arm64_blr(cg_sec, ARM64_X16);
    if (rd != ARM64_X0)
        arm64_add_reg(cg_sec, 1, rd, ARM64_X0, ARM64_XZR, ARM64_LSL, 0);
#else
    // Linux ARM64: Local Exec TLS model
    // mrs x{rd}, tpidr_el0
    arm64_mrs(cg_sec, rd, 0x5e82); // mrs x{rd}, tpidr_el0
    size_t hi_off = cg_sec->len;
    arm64_add_imm(cg_sec, 1, rd, rd, 0, 1); // add x{rd}, x{rd}, #:tprel_hi12:label, lsl #12
    objfile_add_reloc(cg_obj, SEC_TEXT, hi_off, sidx, R_AARCH64_TLSLE_ADD_TPREL_HI12, 0);
    size_t lo_off = cg_sec->len;
    arm64_add_imm(cg_sec, 1, rd, rd, 0, 0); // add x{rd}, x{rd}, #:tprel_lo12_nc:label
    objfile_add_reloc(cg_obj, SEC_TEXT, lo_off, sidx, R_AARCH64_TLSLE_ADD_TPREL_LO12, 0);
#endif
}
#endif

static int op_size(Type *ty) {
    // Arrays decay to pointers in rvalue context; the machine operand
    // for pointer subtraction (and any pointer arithmetic) is pointer-width
    // (8 on 64-bit).  Using the aggregate byte size (e.g. U8[6] → 6) as sz
    // corrupts the arithmetic size, causing rex_for_size(6, ...) to emit no
    // REX prefix, so the CPU decodes extended registers (r10/r11) as their
    // low-byte aliases (edx/ebx) and the subtraction operates on garbage.
    if (ty->kind == TY_ARRAY)
        return 8;
    if (is_integer(ty) && ty->size < 4)
        return 4;
    if (ty->size > 8)
        return 8;
    return ty->size;
}

static Type *promote_int(Type *ty) {
    if (ty && is_integer(ty) && ty->size < 4)
        return ty_int;
    return ty;
}

#ifndef ARCH_ARM64
static char size_suffix(int sz) {
    if (sz == 1) return 'b';
    if (sz == 2) return 'w';
    if (sz == 4) return 'l';
    return 'q';
}
#endif

static bool use_unsigned(Type *ty) {
    return ty && is_integer(ty) && ty->is_unsigned;
}

static bool use_unsigned_cmp(Node *node) {
    Type *lhs = promote_int(node->lhs->ty);
    Type *rhs = promote_int(node->rhs->ty);
    int size = lhs->size > rhs->size ? lhs->size : rhs->size;
    if (size < 4)
        size = 4;
    return get_integer_type(size, lhs->is_unsigned || rhs->is_unsigned)->is_unsigned;
}

static void sign_extend_to(VReg r, int from_size, int to_size) {
    if (to_size <= from_size)
        return;
    if (to_size == 8) {
        if (from_size == 4)
#ifdef ARCH_ARM64
            asm_movsx(cg_sec, r, r, 8, 4); // movsx8->r rr, rr
#else
            asm_movsx(cg_sec, r, r, 8, 4); // movsx8->r rr, rr
#endif
        else if (from_size == 2)
#ifdef ARCH_ARM64
            asm_movsx(cg_sec, r, r, 8, 2); // movsx8->r rr, rr
#else
            asm_movsx(cg_sec, r, r, 8, 2); // movsx8->r rr, rr
#endif
        else if (from_size == 1)
#ifdef ARCH_ARM64
            asm_movsx(cg_sec, r, r, 8, 1); // movsx8->r rr, rr
#else
            asm_movsx(cg_sec, r, r, 8, 1); // movsx8->r rr, rr
#endif
        else
#ifdef ARCH_ARM64
            asm_movsx(cg_sec, r, r, 8, 4); // movsx8->r rr, rr
#else
            asm_movsx(cg_sec, r, r, 8, 4); // movsx8->r rr, rr
#endif
    } else if (to_size == 4) {
        if (from_size == 2)
#ifdef ARCH_ARM64
            asm_movsx(cg_sec, r, r, 4, 2); // movsx4->r rr, rr
#else
            asm_movsx(cg_sec, r, r, 4, 2); // movsx4->r rr, rr
#endif
        else if (from_size == 1)
#ifdef ARCH_ARM64
            asm_movsx(cg_sec, r, r, 4, 1); // movsx4->r rr, rr
#else
            asm_movsx(cg_sec, r, r, 4, 1); // movsx4->r rr, rr
#endif
        else
            asm_mov_reg_reg(cg_sec, r, r, 4); // mov rr -> rr
    }
}

static void zero_extend_to(VReg r, int from_size, int to_size) {
    if (to_size <= from_size)
        return;
    if (to_size == 8) {
        if (from_size == 4)
            asm_mov_reg_reg(cg_sec, r, r, 4); // mov rr -> rr
        else if (from_size == 2)
#ifdef ARCH_ARM64
            asm_movzx(cg_sec, r, r, 8, 2); // movzx8->r rr, rr
#else
            asm_movzx(cg_sec, r, r, 4, 2); // movzx4->r rr, rr
#endif
        else if (from_size == 1)
#ifdef ARCH_ARM64
            asm_movzx(cg_sec, r, r, 8, 1); // movzx8->r rr, rr
#else
            asm_movzx(cg_sec, r, r, 4, 1); // movzx4->r rr, rr
#endif
        else
            asm_mov_reg_reg(cg_sec, r, r, 4); // mov rr -> rr
    } else if (to_size == 4) {
        if (from_size == 2)
#ifdef ARCH_ARM64
            asm_movzx(cg_sec, r, r, 4, 2); // movzx4->r rr, rr
#else
            asm_movzx(cg_sec, r, r, 4, 2); // movzx4->r rr, rr
#endif
        else if (from_size == 1)
#ifdef ARCH_ARM64
            asm_movzx(cg_sec, r, r, 4, 1); // movzx4->r rr, rr
#else
            asm_movzx(cg_sec, r, r, 4, 1); // movzx4->r rr, rr
#endif
        else
            asm_mov_reg_reg(cg_sec, r, r, 4); // mov rr -> rr
    }
}

// Convert the scalar value in register r (of type `from`, an integer or
// floating-point type) to the representation of complex base type `base`,
// store it as the real part at the buffer pointed to by register `addr`,
// and zero-fill the imaginary part at offset base->size.
static void emit_scalar_to_complex(int r, Type *from, Type *base, int addr) {
    int base_sz = base->size;
    bool base_flo = is_flonum(base);
#ifdef ARCH_ARM64
    if (base_flo) {
        if (is_flonum(from)) {
            asm_fmov_i2f(cg_sec, 0, r, 1); // fmov d0, x{r}
        } else if (from->is_unsigned) {
            asm_ucvtf(cg_sec, ARM64_D0, r, from->size == 8 ? 1 : 0); // ucvtf d0, w/x{r}
        } else {
            asm_scvtf(cg_sec, ARM64_D0, r, from->size == 8 ? 1 : 0); // scvtf d0, w/x{r}
        }
        if (base_sz == 4) {
            asm_fcvt(cg_sec, 0, 1, ARM64_S0, ARM64_D0); // fcvt s0, d0
            asm_str_fp(cg_sec, ARM64_S0, addr, 4); // str s0, [x{addr}]
            arm64_str_uoff(cg_sec, 2, ARM64_XZR, REG(addr), 1); // str wzr, [x{addr}, #4]
        } else {
            asm_str_fp(cg_sec, ARM64_D0, addr, 8); // str d0, [x{addr}]
            arm64_str_uoff(cg_sec, 3, ARM64_XZR, REG(addr), 1); // str xzr, [x{addr}, #8]
        }
    } else {
        if (is_flonum(from)) {
            asm_fmov_i2f(cg_sec, 0, r, 1); // fmov d0, x{r}
            asm_fcvtzs(cg_sec, r, base_sz); // fcvtzs w/x{r}, d0
        } else if (from->size < base_sz) {
            if (from->is_unsigned)
                zero_extend_to(r, from->size, base_sz);
            else
                sign_extend_to(r, from->size, base_sz);
        }
        if (base_sz == 4) {
            asm_str_reg_off(cg_sec, r, addr, 4, 0);
            arm64_str_uoff(cg_sec, 2, ARM64_XZR, REG(addr), 1); // str wzr, [x{addr}, #4]
        } else {
            asm_str_reg_off(cg_sec, r, addr, 8, 0);
            arm64_str_uoff(cg_sec, 3, ARM64_XZR, REG(addr), 1); // str xzr, [x{addr}, #8]
        }
    }
#else
    X86Mem m_addr0 = x86_mem(REG(addr), 0);
    X86Mem m_addr_base = x86_mem(REG(addr), base_sz);
    if (base_flo) {
        if (is_flonum(from)) {
            asm_movq_r_xmm(cg_sec, X86_XMM0, r); // movq reg64[r], %xmm0
        } else if (from->is_unsigned && from->size == 8) {
            int c = ++rcc_label_count;
            asm_test_reg_reg(cg_sec, r, r, 8);
            char *label_high = format(".L.u2cx.high.%d", c);
            char *label_end = format(".L.u2cx.end.%d", c);
            size_t off_js = asm_jcc_label(cg_sec, X86_S);
            asm_fixup_add(cg_sec, off_js, label_high, 1); // js .L.u2cx.high.%d
            x86_cvtsi2sd(cg_sec, 8, X86_XMM0, REG(r)); // cvtsi2sd reg64[r], %xmm0
            size_t off_jmp = asm_jmp_label(cg_sec);
            asm_fixup_add(cg_sec, off_jmp, label_end, 0); // jmp .L.u2cx.end.%d
            cg_def_label(label_high); // .L.u2cx.high.%d:
            x86_mov_rr(cg_sec, 8, X86_RCX, REG(r)); // movq reg64[r], %rcx
            x86_shr_ri(cg_sec, 8, X86_RCX, 1); // shrq %rcx
            x86_cvtsi2sd(cg_sec, 8, X86_XMM0, X86_RCX); // cvtsi2sd %rcx, %xmm0
            x86_addsd(cg_sec, X86_XMM0, X86_XMM0); // addsd %xmm0, %xmm0
            cg_def_label(label_end); // .L.u2cx.end.%d:
        } else if (from->is_unsigned && from->size == 4) {
            x86_cvtsi2sd(cg_sec, 8, X86_XMM0, REG(r)); // cvtsi2sd reg64[r], %xmm0
        } else {
            x86_cvtsi2sd(cg_sec, from->size < 4 ? 4 : from->size, X86_XMM0, REG(r)); // cvtsi2sd reg32/64[r], %xmm0
        }
        if (base_sz == 4) {
            x86_cvtsd2ss(cg_sec, X86_XMM0, X86_XMM0); // cvtsd2ss %xmm0, %xmm0
            x86_movss_mr(cg_sec, m_addr0, X86_XMM0); // movss %xmm0, (reg64[addr])
            x86_mov_mi(cg_sec, 4, m_addr_base, 0); // movl $0, base_sz(reg64[addr])
        } else {
            x86_movsd_mr(cg_sec, m_addr0, X86_XMM0); // movsd %xmm0, (reg64[addr])
            x86_mov_mi(cg_sec, 8, m_addr_base, 0); // movq $0, base_sz(reg64[addr])
        }
    } else {
        if (is_flonum(from)) {
            asm_movq_r_xmm(cg_sec, X86_XMM0, r); // movq reg64[r], %xmm0
            x86_cvttsd2si(cg_sec, base_sz, REG(r), X86_XMM0); // cvttsd2si %xmm0, reg32/64[r]
        } else if (from->size < base_sz) {
            if (from->is_unsigned)
                zero_extend_to(r, from->size, base_sz);
            else
                sign_extend_to(r, from->size, base_sz);
        }
        if (base_sz == 4) {
            x86_mov_mr(cg_sec, 4, m_addr0, REG(r)); // movl reg32[r], (reg64[addr])
            x86_mov_mi(cg_sec, 4, m_addr_base, 0); // movl $0, base_sz(reg64[addr])
        } else {
            x86_mov_mr(cg_sec, 8, m_addr0, REG(r)); // movq reg64[r], (reg64[addr])
            x86_mov_mi(cg_sec, 8, m_addr_base, 0); // movq $0, base_sz(reg64[addr])
        }
    }
#endif
}

// Convert a TY_COMPLEX value at address `src` (real-floating base type
// `from->base`, e.g. _Complex float) to a TY_COMPLEX value at address `dst`
// (real-floating base type `to->base`, e.g. _Complex double), narrowing or
// widening each component as needed (cvtss2sd/cvtsd2ss).  Both components
// are loaded before either is stored, so src == dst (in-place conversion,
// e.g. a smaller hidden-retbuf result being widened in place) is safe even
// when to->size > from->size.
static void emit_complex_convert_float(int src, int dst, Type *from, Type *to) {
    (void)to; // used via to->base->size in ARCH_ARM64 path
    int fsz = from->base->size;
#ifdef ARCH_ARM64
    int tsz = to->base->size;
    if (fsz == 4) {
        asm_ldr_fp(cg_sec, ARM64_S0, src, 4); // ldr s0, [x{src}]
        arm64_ldr_fp(cg_sec, 2, (Arm64Reg)(ARM64_S0 + 1), REG(src), 4); // ldr s1, [x{src}, #4]
        asm_fcvt(cg_sec, 1, 0, ARM64_D0, ARM64_S0); // fcvt d0, s0
        asm_fcvt(cg_sec, 1, 0, (Arm64Reg)(ARM64_D0 + 1), (Arm64Reg)(ARM64_S0 + 1)); // fcvt d1, s1
    } else {
        asm_ldr_fp(cg_sec, ARM64_D0, src, 8); // ldr d0, [x{src}]
        arm64_ldr_fp(cg_sec, 3, (Arm64Reg)(ARM64_D0 + 1), REG(src), 8); // ldr d1, [x{src}, #8]
        asm_fcvt(cg_sec, 0, 1, ARM64_S0, ARM64_D0); // fcvt s0, d0
        asm_fcvt(cg_sec, 0, 1, (Arm64Reg)(ARM64_S0 + 1), (Arm64Reg)(ARM64_D0 + 1)); // fcvt s1, d1
    }
    if (tsz == 4) {
        asm_str_fp(cg_sec, ARM64_S0, dst, 4); // str s0, [x{dst}]
        arm64_str_fp(cg_sec, 2, (Arm64Reg)(ARM64_S0 + 1), REG(dst), 4); // str s1, [x{dst}, #4]
    } else {
        asm_str_fp(cg_sec, ARM64_D0, dst, 8); // str d0, [x{dst}]
        arm64_str_fp(cg_sec, 3, (Arm64Reg)(ARM64_D0 + 1), REG(dst), 8); // str d1, [x{dst}, #8]
    }
#else
    if (fsz == 4) {
        // _Complex float → _Complex double: load singles, widen to doubles
        asm_mov_fp_rm(cg_sec, 4, X86_XMM0, x86_mem(REG(src), 0)); // movss (src), %xmm0
        asm_mov_fp_rm(cg_sec, 4, X86_XMM1, x86_mem(REG(src), 4)); // movss 4(src), %xmm1
        x86_cvtss2sd(cg_sec, X86_XMM0, X86_XMM0); // cvtss2sd %xmm0, %xmm0
        x86_cvtss2sd(cg_sec, X86_XMM1, X86_XMM1); // cvtss2sd %xmm1, %xmm1
        asm_mov_fp_mr(cg_sec, 8, x86_mem(REG(dst), 0), X86_XMM0); // movsd %xmm0, (dst)
        asm_mov_fp_mr(cg_sec, 8, x86_mem(REG(dst), 8), X86_XMM1); // movsd %xmm1, 8(dst)
    } else {
        // _Complex double → _Complex float: load doubles, narrow to singles
        asm_mov_fp_rm(cg_sec, 8, X86_XMM0, x86_mem(REG(src), 0)); // movsd (src), %xmm0
        asm_mov_fp_rm(cg_sec, 8, X86_XMM1, x86_mem(REG(src), 8)); // movsd 8(src), %xmm1
        x86_cvtsd2ss(cg_sec, X86_XMM0, X86_XMM0); // cvtsd2ss %xmm0, %xmm0
        x86_cvtsd2ss(cg_sec, X86_XMM1, X86_XMM1); // cvtsd2ss %xmm1, %xmm1
        asm_mov_fp_mr(cg_sec, 4, x86_mem(REG(dst), 0), X86_XMM0); // movss %xmm0, (dst)
        asm_mov_fp_mr(cg_sec, 4, x86_mem(REG(dst), 4), X86_XMM1); // movss %xmm1, 4(dst)
    }
#endif
}

// Convert between mixed int/float complex bases at the same address.
// Used for _Complex int ↔ _Complex double and similar.
static void emit_complex_convert_mixed(int src, int dst, Type *from, Type *to) {
    int fsz = from->base->size, tsz = to->base->size;
    bool from_int = is_integer(from->base);
    (void)tsz;
#ifdef ARCH_ARM64
    if (from_int) {
        // int complex → float complex: load from src, convert, store to dst
        if (fsz == 4) {
            if (from->base->is_unsigned) {
                arm64_ldr_uoff(cg_sec, 2, ARM64_X0, REG(src), 0);
                arm64_ldr_uoff(cg_sec, 2, ARM64_X1, REG(src), 1);
            } else {
                arm64_ldrsw_uoff(cg_sec, ARM64_X0, REG(src), 0);
                arm64_ldrsw_uoff(cg_sec, ARM64_X1, REG(src), 1);
            }
            arm64_scvtf(cg_sec, 0, 1, ARM64_D0, ARM64_X0);
            arm64_scvtf(cg_sec, 0, 1, ARM64_D1, ARM64_X1);
            arm64_str_fp(cg_sec, 3, ARM64_D0, REG(dst), 0);
            arm64_str_fp(cg_sec, 3, ARM64_D1, REG(dst), 8);
        }
    } else {
        // float complex → int complex: load from src, convert, store to dst
        if (fsz == 8) {
            arm64_ldr_fp(cg_sec, 3, ARM64_D0, REG(src), 0);
            arm64_ldr_fp(cg_sec, 3, ARM64_D1, REG(src), 8);
            if (to->base->is_unsigned) {
                arm64_fcvtzu(cg_sec, 0, 1, ARM64_X0, ARM64_D0);
                arm64_fcvtzu(cg_sec, 0, 1, ARM64_X1, ARM64_D1);
            } else {
                arm64_fcvtzs(cg_sec, 0, 1, ARM64_X0, ARM64_D0);
                arm64_fcvtzs(cg_sec, 0, 1, ARM64_X1, ARM64_D1);
            }
            arm64_str_uoff(cg_sec, 2, ARM64_X0, REG(dst), 0);
            arm64_str_uoff(cg_sec, 2, ARM64_X1, REG(dst), 1);
        } else if (fsz == 4) {
            arm64_ldr_fp(cg_sec, 2, ARM64_S0, REG(src), 0);
            arm64_ldr_fp(cg_sec, 2, ARM64_S1, REG(src), 4);
            if (to->base->is_unsigned) {
                arm64_fcvtzu(cg_sec, 0, 1, ARM64_X0, ARM64_S0);
                arm64_fcvtzu(cg_sec, 0, 1, ARM64_X1, ARM64_S1);
            } else {
                arm64_fcvtzs(cg_sec, 0, 1, ARM64_X0, ARM64_S0);
                arm64_fcvtzs(cg_sec, 0, 1, ARM64_X1, ARM64_S1);
            }
            arm64_str_uoff(cg_sec, 2, ARM64_X0, REG(dst), 0);
            arm64_str_uoff(cg_sec, 2, ARM64_X1, REG(dst), 1);
        }
    }
#else
    // x86-64: convert between int and float complex components
    if (from_int) {
        // int complex -> float complex: load ints, cvtsi2sd, store doubles
        if (fsz == 4) {
            VReg r1 = alloc_reg(), r2 = alloc_reg();
            if (from->base->is_unsigned) {
                x86_mov_rm(cg_sec, 4, REG(r1), x86_mem(REG(src), 0)); // movl (src), r1
                x86_mov_rm(cg_sec, 4, REG(r2), x86_mem(REG(src), 4)); // movl 4(src), r2
            } else {
                asm_movsx_base_off_reg(cg_sec, r1, src, 0, 8, 4); // movslq (src), r1
                asm_movsx_base_off_reg(cg_sec, r2, src, 4, 8, 4); // movslq 4(src), r2
            }
            // Save real to xmm0, then store
            asm_cvtsi2sd(cg_sec, r1, 4); // cvtsi2sd r1, %xmm0
            asm_mov_fp_mr(cg_sec, 8, x86_mem(REG(dst), 0), X86_XMM0); // movsd %xmm0, (dst)
            // Save imag to xmm0, then store
            asm_cvtsi2sd(cg_sec, r2, 4); // cvtsi2sd r2, %xmm0
            asm_mov_fp_mr(cg_sec, 8, x86_mem(REG(dst), 8), X86_XMM0); // movsd %xmm0, 8(dst)
            free_reg(r1);
            free_reg(r2);
        }
    } else {
        // float complex -> int complex: load doubles, cvttsd2si, store ints
        if (fsz == 8) {
            asm_mov_fp_rm(cg_sec, 8, X86_XMM0, x86_mem(REG(src), 0)); // movsd (src), %xmm0
            VReg r = alloc_reg();
            asm_cvttsd2si(cg_sec, r, 4); // cvttsd2si %xmm0, r (32-bit)
            x86_mov_mr(cg_sec, 4, x86_mem(REG(dst), 0), REG(r)); // movl r, (dst)
            asm_mov_fp_rm(cg_sec, 8, X86_XMM0, x86_mem(REG(src), 8)); // movsd 8(src), %xmm0
            asm_cvttsd2si(cg_sec, r, 4); // cvttsd2si %xmm0, r
            x86_mov_mr(cg_sec, 4, x86_mem(REG(dst), 4), REG(r)); // movl r, 4(dst)
            free_reg(r);
        } else if (fsz == 4) {
            asm_mov_fp_rm(cg_sec, 4, X86_XMM0, x86_mem(REG(src), 0)); // movss (src), %xmm0
            asm_cvtss2sd(cg_sec); // cvtss2sd %xmm0, %xmm0
            VReg r = alloc_reg();
            asm_cvttsd2si(cg_sec, r, 4); // cvttsd2si %xmm0, r
            x86_mov_mr(cg_sec, 4, x86_mem(REG(dst), 0), REG(r)); // movl r, (dst)
            asm_mov_fp_rm(cg_sec, 4, X86_XMM0, x86_mem(REG(src), 4)); // movss 4(src), %xmm0
            asm_cvtss2sd(cg_sec); // cvtss2sd %xmm0, %xmm0
            asm_cvttsd2si(cg_sec, r, 4); // cvttsd2si %xmm0, r
            x86_mov_mr(cg_sec, 4, x86_mem(REG(dst), 4), REG(r)); // movl r, 4(dst)
            free_reg(r);
        }
    }
#endif
}
// `from->base`, e.g. _Complex char) to a TY_COMPLEX value at address `dst`
// (integer base type `to->base`, e.g. _Complex int), sign- or
// zero-extending (per from->base->is_unsigned) or truncating each
// component as needed. Both components are loaded before either is
// stored, so src == dst is safe.
static void emit_complex_convert_int(int src, int dst, Type *from, Type *to) {
    int fsz = from->base->size, tsz = to->base->size;
    bool uns = from->base->is_unsigned;
#ifdef ARCH_ARM64
    if (fsz == 8) {
        arm64_ldr_uoff(cg_sec, 3, ARM64_X0, REG(src), 0); // ldr x0, [x{src}]
        arm64_ldr_uoff(cg_sec, 3, ARM64_X1, REG(src), 1); // ldr x1, [x{src}, #8]
    } else if (fsz == 4) {
        if (uns) {
            arm64_ldr_uoff(cg_sec, 2, ARM64_X0, REG(src), 0); // ldr w0, [x{src}]
            arm64_ldr_uoff(cg_sec, 2, ARM64_X1, REG(src), 1); // ldr w1, [x{src}, #4]
        } else {
            arm64_ldrsw_uoff(cg_sec, ARM64_X0, REG(src), 0); // ldrsw x0, [x{src}]
            arm64_ldrsw_uoff(cg_sec, ARM64_X1, REG(src), 1); // ldrsw x1, [x{src}, #4]
        }
    } else if (fsz == 2) {
        arm64_ldrh_uoff(cg_sec, ARM64_X0, REG(src), 0); // ldrh x0, [x{src}]
        arm64_ldrh_uoff(cg_sec, ARM64_X1, REG(src), 1); // ldrh x1, [x{src}, #2]
    } else {
        arm64_ldrb_uoff(cg_sec, ARM64_X0, REG(src), 0); // ldrb w0, [x{src}]
        arm64_ldrb_uoff(cg_sec, ARM64_X1, REG(src), 1); // ldrb w1, [x{src}, #1]
    }
    if (tsz == 8) {
        arm64_str_uoff(cg_sec, 3, ARM64_X0, REG(dst), 0); // str x0, [x{dst}]
        arm64_str_uoff(cg_sec, 3, ARM64_X1, REG(dst), 1); // str x1, [x{dst}, #8]
    } else if (tsz == 4) {
        arm64_str_uoff(cg_sec, 2, ARM64_X0, REG(dst), 0); // str w0, [x{dst}]
        arm64_str_uoff(cg_sec, 2, ARM64_X1, REG(dst), 1); // str w1, [x{dst}, #4]
    } else if (tsz == 2) {
        arm64_strh_uoff(cg_sec, ARM64_X0, REG(dst), 0); // strh w0, [x{dst}]
        arm64_strh_uoff(cg_sec, ARM64_X1, REG(dst), 1); // strh w1, [x{dst}, #2]
    } else {
        arm64_strb_uoff(cg_sec, ARM64_X0, REG(dst), 0); // strb w0, [x{dst}]
        arm64_strb_uoff(cg_sec, ARM64_X1, REG(dst), 1); // strb w1, [x{dst}, #1]
    }
#else
    // x86_64: load both components from src into RAX/RCX, then store to dst
    // Sign/zero-extend from source size, truncate to target size
    if (fsz == 8) {
        x86_mov_rm(cg_sec, 8, X86_RAX, x86_mem(REG(src), 0)); // movq (src), %rax
        x86_mov_rm(cg_sec, 8, X86_RCX, x86_mem(REG(src), 8)); // movq 8(src), %rcx
    } else if (fsz == 4) {
        if (uns) {
            x86_mov_rm(cg_sec, 4, X86_RAX, x86_mem(REG(src), 0)); // movl (src), %eax
            x86_mov_rm(cg_sec, 4, X86_RCX, x86_mem(REG(src), 4)); // movl 4(src), %ecx
        } else {
            x86_movsx_rm(cg_sec, 8, 4, X86_RAX, x86_mem(REG(src), 0)); // movslq (src), %rax
            x86_movsx_rm(cg_sec, 8, 4, X86_RCX, x86_mem(REG(src), 4)); // movslq 4(src), %rcx
        }
    } else if (fsz == 2) {
        if (uns) {
            x86_movzx_rm(cg_sec, 4, 2, X86_RAX, x86_mem(REG(src), 0)); // movzwl (src), %eax
            x86_movzx_rm(cg_sec, 4, 2, X86_RCX, x86_mem(REG(src), 2)); // movzwl 2(src), %ecx
        } else {
            x86_movsx_rm(cg_sec, 4, 2, X86_RAX, x86_mem(REG(src), 0)); // movswl (src), %eax
            x86_movsx_rm(cg_sec, 4, 2, X86_RCX, x86_mem(REG(src), 2)); // movswl 2(src), %ecx
        }
    } else {
        if (uns) {
            x86_movzx_rm(cg_sec, 4, 1, X86_RAX, x86_mem(REG(src), 0)); // movzbl (src), %eax
            x86_movzx_rm(cg_sec, 4, 1, X86_RCX, x86_mem(REG(src), 1)); // movzbl 1(src), %ecx
        } else {
            x86_movsx_rm(cg_sec, 4, 1, X86_RAX, x86_mem(REG(src), 0)); // movsbl (src), %eax
            x86_movsx_rm(cg_sec, 4, 1, X86_RCX, x86_mem(REG(src), 1)); // movsbl 1(src), %ecx
        }
    }
    // Store to dst at target size
    x86_mov_mr(cg_sec, tsz, x86_mem(REG(dst), 0), X86_RAX); // mov %al/%ax/%eax/%rax, (dst)
    x86_mov_mr(cg_sec, tsz, x86_mem(REG(dst), tsz), X86_RCX); // mov %cl/%cx/%ecx/%rcx, tsz(dst)
#endif
}

#ifndef ARCH_ARM64
// Given a 32-bit x86_64 register name (e.g. "%eax", "%r9d"), return the
// 8-bit ("%al", "%r9b") or 16-bit ("%ax", "%r9w") sub-register name, used
// to store a truncated _Complex char/short integer component.
static const char *x86_subreg(const char *reg32_name, int sz) {
    static const char *map8[] = {
        "%eax", "%al", "%ecx", "%cl", "%edx", "%dl", "%edi", "%dil",
        "%r8d", "%r8b", "%r9d", "%r9b", "%r10d", "%r10b", "%r11d", "%r11b", NULL};
    static const char *map16[] = {
        "%eax", "%ax", "%ecx", "%cx", "%edx", "%dx", "%edi", "%di",
        "%r8d", "%r8w", "%r9d", "%r9w", "%r10d", "%r10w", "%r11d", "%r11w", NULL};
    const char **map = sz == 1 ? map8 : map16;
    for (int i = 0; map[i]; i += 2)
        if (!strcmp(reg32_name, map[i]))
            return map[i + 1];
    return reg32_name;
}

// GCC extended-asm operand size-override modifiers (%b0/%w0/%k0/%q0/%h0):
// given a GP register operand string in *any* size class (e.g. "%rax",
// "%eax", "%al" — whatever the operand's natural width produced) and a
// requested size class, return the same physical register's name in that
// class. size_class: 0=64-bit, 1=32-bit, 2=16-bit, 3=8-bit low, 4=8-bit
// high (legacy A/B/C/D only). Returns NULL for a non-register operand
// (memory/immediate) or a class that register doesn't have (e.g. %h9),
// in which case the caller should fall back to the unmodified operand
// text, matching plain GCC's behavior of leaving it alone.
static const char *x86_reg_resize(const char *reg, int size_class) {
    static const char *const family[16][5] = {
        // 64,     32,      16,     8lo,    8hi
        {"%rax", "%eax", "%ax", "%al", "%ah"},
        {"%rcx", "%ecx", "%cx", "%cl", "%ch"},
        {"%rdx", "%edx", "%dx", "%dl", "%dh"},
        {"%rbx", "%ebx", "%bx", "%bl", "%bh"},
        {"%rsp", "%esp", "%sp", "%spl", NULL},
        {"%rbp", "%ebp", "%bp", "%bpl", NULL},
        {"%rsi", "%esi", "%si", "%sil", NULL},
        {"%rdi", "%edi", "%di", "%dil", NULL},
        {"%r8", "%r8d", "%r8w", "%r8b", NULL},
        {"%r9", "%r9d", "%r9w", "%r9b", NULL},
        {"%r10", "%r10d", "%r10w", "%r10b", NULL},
        {"%r11", "%r11d", "%r11w", "%r11b", NULL},
        {"%r12", "%r12d", "%r12w", "%r12b", NULL},
        {"%r13", "%r13d", "%r13w", "%r13b", NULL},
        {"%r14", "%r14d", "%r14w", "%r14b", NULL},
        {"%r15", "%r15d", "%r15w", "%r15b", NULL},
    };
    if (!reg || reg[0] != '%') return NULL;
    for (int i = 0; i < 16; i++)
        for (int c = 0; c < 5; c++)
            if (family[i][c] && !strcmp(reg, family[i][c]))
                return family[i][size_class];
    return NULL;
}
#endif

#ifdef ARCH_ARM64
// Sentinels for base register index in emit_store / emit_load:
// >= 0: virtual register index, REG maps to physical register
// ARM64_BASE_FP (-29): frame pointer (x29)
// ARM64_BASE_SP (-31): stack pointer (x31)
#define ARM64_BASE_FP (-29)
#define ARM64_BASE_SP (-31)
#define ARM64_BASE_X16 (-16)

// Get physical ARM64 register number from base index
static int arm64_base_phys(int base) {
    if (base == ARM64_BASE_FP || base == 29) return 29;
    if (base == ARM64_BASE_SP || base == 31) return 31;
    if (base == ARM64_BASE_X16 || base == 16) return 16;
    return REG(base);
}

// Check if base register matches destination virtual register
static bool arm64_base_conflicts(int base, VReg r) {
    return base >= 0 && base == r;
}

static void emit_store(Type *ty, VReg src, VReg base, int off) {
    int sz = ty->size;
    if (off == 0) {
        asm_str_reg_off(cg_sec, src, base, sz, 0); // str {src}, [base]
    } else if (off > -256 && off < 256) {
        asm_stur_sz(cg_sec, src, REG(base), sz, off); // stur {src}, [base, #off]
    } else {
        // Offset outside STUR range: use X17 scratch
        if (off < 0 && -off <= 4095) {
            arm64_sub_imm(cg_sec, 1, ARM64_X17, REG(base), -off, 0); // sub x17, base, #-off
        } else if (off >= 0 && off <= 4095) {
            arm64_add_imm(cg_sec, 1, ARM64_X17, REG(base), off, 0); // add x17, base, #off
        } else {
            int abs_off = off < 0 ? -off : off;
            emit_mov_imm64(ARM64_X17, (uint64_t)abs_off);
            if (off < 0)
                arm64_sub_reg(cg_sec, 1, ARM64_X17, REG(base), ARM64_X17, ARM64_LSL, 0);
            else
                arm64_add_reg(cg_sec, 1, ARM64_X17, ARM64_X17, REG(base), ARM64_LSL, 0);
        }
        switch (sz) {
        case 1: arm64_strb_uoff(cg_sec, REG(src), ARM64_X17, 0); break;
        case 2: arm64_strh_uoff(cg_sec, REG(src), ARM64_X17, 0); break;
        case 4: arm64_str_uoff(cg_sec, 2, REG(src), ARM64_X17, 0); break;
        default: arm64_str_uoff(cg_sec, 3, REG(src), ARM64_X17, 0); break;
        }
    }
}

static void emit_store_offset(Type *ty, VReg r, Arm64Reg base, int offset) {
    int sz = ty->size;
    int off = offset;
    if (off > -256 && off < 256) {
        asm_stur_sz(cg_sec, r, base, sz, off); // stur r, [base, #off]
        return;
    }
    int abs_off = off < 0 ? -off : off;
    // Use X17 scratch instead of alloc_reg to avoid spill pressure
    if (abs_off <= 4095) {
        if (off < 0)
            arm64_sub_imm(cg_sec, 1, ARM64_X17, base, abs_off, 0); // sub x17, base, #abs_off
        else
            arm64_add_imm(cg_sec, 1, ARM64_X17, base, abs_off, 0); // add x17, base, #abs_off
    } else {
        emit_mov_imm64(ARM64_X17, (uint64_t)abs_off);
        if (off < 0)
            arm64_sub_reg(cg_sec, 1, ARM64_X17, base, ARM64_X17, ARM64_LSL, 0);
        else
            arm64_add_reg(cg_sec, 1, ARM64_X17, ARM64_X17, base, ARM64_LSL, 0);
    }
    switch (sz) {
    case 1: arm64_strb_uoff(cg_sec, REG(r), ARM64_X17, 0); break;
    case 2: arm64_strh_uoff(cg_sec, REG(r), ARM64_X17, 0); break;
    case 4: arm64_str_uoff(cg_sec, 2, REG(r), ARM64_X17, 0); break;
    default: arm64_str_uoff(cg_sec, 3, REG(r), ARM64_X17, 0); break;
    }
}
#endif

static void emit_load(Type *ty, VReg r, int base, int off) {
#ifdef ARCH_ARM64
    // ARM64: ldr wN,[xN] is CONSTRAINED UNPREDICTABLE — avoid base==dest
    if (arm64_base_conflicts(base, r)) {
        asm_mov_x16_reg(cg_sec, base); // mov x16, x{base}
        base = ARM64_BASE_X16;
    }
    if (off > -256 && off < 256) {
        asm_ldur_sz(cg_sec, r, arm64_base_phys(base), ty->size, off); // ldur r, [base, #off]
        // Sign-extend narrow signed types (ARM64 ldur{b,h} zero-extend)
        if (!ty->is_unsigned) {
            if (ty->size == 1)
                arm64_sxtb(cg_sec, 0, REG(r), REG(r)); // sxtb w{r}, w{r}
            else if (ty->size == 2)
                arm64_sxth(cg_sec, 0, REG(r), REG(r)); // sxth w{r}, w{r}
        }
    } else {
        // Offset outside LDUR range: use X17 as scratch for address computation
        // (never in the register pool, avoids spill pressure)
        Arm64Reg bp = arm64_base_phys(base);
        if (off < 0 && -off <= 4095) {
            arm64_sub_imm(cg_sec, 1, ARM64_X17, bp, -off, 0); // sub x17, base, #-off
        } else if (off >= 0 && off <= 4095) {
            arm64_add_imm(cg_sec, 1, ARM64_X17, bp, off, 0); // add x17, base, #off
        } else {
            int abs_off = off < 0 ? -off : off;
            emit_mov_imm64(ARM64_X17, (uint64_t)abs_off);
            if (off < 0)
                arm64_sub_reg(cg_sec, 1, ARM64_X17, bp, ARM64_X17, ARM64_LSL, 0);
            else
                arm64_add_reg(cg_sec, 1, ARM64_X17, ARM64_X17, bp, ARM64_LSL, 0);
        }
        // Load from [x17] with correct size
        switch (ty->size) {
        case 1: arm64_ldrb_uoff(cg_sec, REG(r), ARM64_X17, 0); break;
        case 2: arm64_ldrh_uoff(cg_sec, REG(r), ARM64_X17, 0); break;
        case 4: arm64_ldr_uoff(cg_sec, 2, REG(r), ARM64_X17, 0); break;
        default: arm64_ldr_uoff(cg_sec, 3, REG(r), ARM64_X17, 0); break;
        }
        if (!ty->is_unsigned) {
            if (ty->size == 1)
                arm64_sxtb(cg_sec, 0, REG(r), REG(r));
            else if (ty->size == 2)
                arm64_sxth(cg_sec, 0, REG(r), REG(r));
        }
    }
#else
#ifndef X86_BASE_RBP
#define X86_BASE_RBP (-1)
#endif
    // x86_64: base >= 0 = virtual reg, X86_BASE_RBP = rbp-relative /* ldr %s, [%s]\n */
    int sz = op_size(ty);
    // Struct/union of 5-7 bytes: no x86 instruction loads exactly 5-7 bytes;
    // round up to 8 (64-bit load) so all bytes reach the register.
    if (sz > 4 && sz < 8)
        sz = 8;
    if (base == X86_BASE_RBP) {
        if (ty->size == 1) {
            if (ty->is_unsigned)
                asm_movzx_rbp_reg(cg_sec, r, 4, 1, off); // movzbl -off(%rbp), rr
            else
                asm_movsx_rbp_reg(cg_sec, r, 4, 1, off); // movsbl -off(%rbp), rr
        } else if (ty->size == 2) {
            if (ty->is_unsigned)
                asm_movzx_rbp_reg(cg_sec, r, 4, 2, off); // movzwl -off(%rbp), rr
            else
                asm_movsx_rbp_reg(cg_sec, r, 4, 2, off); // movswl -off(%rbp), rr
        } else {
            asm_mov_rbp_reg(cg_sec, r, sz, off); // mov -off(%rbp), rr
        }
    } else {
        if (ty->size == 1) {
            if (ty->is_unsigned)
                asm_movzx_base_off_reg(cg_sec, r, base, (int64_t)off, 4, 1); // movzbl off(base), rr
            else
                asm_movsx_base_off_reg(cg_sec, r, base, (int64_t)off, 4, 1); // movsbl off(base), rr
        } else if (ty->size == 2) {
            if (ty->is_unsigned)
                asm_movzx_base_off_reg(cg_sec, r, base, (int64_t)off, 4, 2); // movzwl off(base), rr
            else
                asm_movsx_base_off_reg(cg_sec, r, base, (int64_t)off, 4, 2); // movswl off(base), rr
        } else {
            asm_mov_base_off_reg(cg_sec, r, base, (int64_t)off, sz); // movl/movq off(base), rr
        }
    }
    if (sz == 8 && ty->size < 4)
        asm_movzx(cg_sec, r, r, 4, ty->size); // movzx4->r rr, rr
#endif
}

// Number of currently-free allocatable registers. Used to decide whether an
// inline builtin expansion (memcmp/memset/memcpy/str*) can hold all of its
// operand VRegs live simultaneously WITHOUT alloc_reg() having to spill one
// of them. alloc_reg() hands out the lowest free index and spills the highest
// used index, so operand results (allocated first, at low indices) are never
// spill victims as long as we entered the expansion with at least as many
// free registers as operands held at once. When headroom is insufficient we
// fall through to the generic call path, which stages arguments correctly
// under arbitrary register pressure.
static int free_reg_count(void) {
    int n = 0;
    for (int i = 0; i < NUM_REGS; i++)
        if ((used_regs & (1 << i)) == 0)
            n++;
    return n;
}

VReg alloc_reg(void) {
    for (int i = 0; i < NUM_REGS; i++) {
        if ((used_regs & (1 << i)) == 0) {
            used_regs |= (1 << i);
            ever_used_regs |= (1 << i);
            return (VReg)i;
        }
    }
    // All registers are in use. Spill the register with the highest index
    // (least likely to be referenced by outer callers right now).
    //
    // Must skip a register that is ALREADY spilled in the first pass: a
    // still-outstanding spill's slot address (spill_offset(i)) must not
    // change out from under any read-only call site elsewhere in this
    // file that reads it directly (comparisons/arithmetic operating on a
    // spilled operand's backing memory without going through free_reg()/
    // materialize_reg() first) -- picking one of those as a victim here
    // would still be correct now that spills are a per-index STACK
    // (push_spill_slot() below never clobbers an outstanding spill), but
    // preferring a not-yet-spilled register first keeps the common case
    // spill-free-with-headroom fast path unchanged and the generated code
    // no noisier than before.
    //
    // First pass: prefer a used-but-not-yet-spilled register.
    for (int i = NUM_REGS - 1; i >= 0; i--) {
        if ((used_regs & (1 << i)) && !(spilled_regs & (1 << i))) {
            if (opt_W) {
                if (reg_owner[i])
                    fprintf(stderr, "\033[1;33mwarning:\033[0m spilling %s (%s) to stack in %s\n", reg64[i], reg_owner[i], current_fn);
                else
                    fprintf(stderr, "\033[1;33mwarning:\033[0m spilling %s to stack in %s\n", reg64[i], current_fn);
            }
#ifdef ARCH_ARM64
            asm_stur_fp(cg_sec, i, push_spill_slot(i)); // str x(i), [x29, #-push_spill_slot(i)]
#else
            asm_mov_reg_rbp(cg_sec, i, 8, push_spill_slot(i)); // mov ri, [rbp-8]
#endif
            spilled_regs |= (1 << i);
            spill_count++;
            used_regs &= ~(1 << i);
            used_regs |= (1 << i); // reclaim for new value
            ever_used_regs |= (1 << i);
            return (VReg)i;
        }
    }
    // Fallback: every used register is already spilled. With a per-index
    // spill STACK (push_spill_slot() below), re-spilling one here is safe
    // -- it pushes a fresh slot rather than clobbering the outstanding
    // one -- so this is purely a "pick a victim" tier, not a correctness
    // hazard; it only differs from the first pass in also considering
    // already-spilled registers as candidates.
    for (int i = NUM_REGS - 1; i >= 0; i--) {
        if (used_regs & (1 << i)) {
            if (opt_W) {
                if (reg_owner[i])
                    fprintf(stderr, "\033[1;33mwarning:\033[0m spilling %s (%s) to stack in %s\n", reg64[i], reg_owner[i], current_fn);
                else
                    fprintf(stderr, "\033[1;33mwarning:\033[0m spilling %s to stack in %s\n", reg64[i], current_fn);
            }
#ifdef ARCH_ARM64
            asm_stur_fp(cg_sec, i, push_spill_slot(i)); // str x(i), [x29, #-push_spill_slot(i)]
#else
            asm_mov_reg_rbp(cg_sec, i, 8, push_spill_slot(i)); // mov ri, [rbp-8]
#endif
            spilled_regs |= (1 << i);
            spill_count++;
            used_regs &= ~(1 << i);
            used_regs |= (1 << i); // reclaim for new value
            ever_used_regs |= (1 << i);
            return (VReg)i;
        }
    }
    error("Register exhaustion");
    return 0;
}

// Like alloc_reg(), but never returns `avoid1` or `avoid2` (pass -1 for
// either slot to not exclude it). Needed wherever a VReg is computed
// once and then a SECOND VReg is allocated that must remain
// concurrently live with one or more earlier ones -- e.g. a bitfield
// read-modify-write's "old value, masked" register (`rt`) and its
// address register (`ra`, still needed for the final store) both have
// to survive allocating the "new value, masked+shifted" register
// (`rv`) that gets ORed into `rt` and written back through `ra`.
// alloc_reg()'s ordinary spill selection has no notion of "this used
// register is needed again in a few instructions, don't touch it", so
// under register pressure it can legitimately pick either one as the
// spill victim for the new allocation. That doesn't just move the
// victim's value to memory the way it does for a genuinely outer/
// enclosing VReg -- since VReg identity IS the physical register
// index, the new allocation then returns the exact same index as the
// victim, so the two "different" VRegs alias the same physical
// register while both are supposedly independently live. Every
// subsequent instruction issued against either one hits the same 8
// bytes: `asm_or_reg_reg(rt, rv, ...)` degenerates into a self-`or
// reg,reg` no-op that silently drops the just-computed value, or a
// `mov rv,ra`-style clobber corrupts the address the final store
// writes through. materialize_reg() can't fix this after the fact --
// reloading the victim back into its slot would just as destructively
// clobber whatever the new VReg (which now shares that same slot) had
// already computed into it. Found via a qbe-style `(Ref){RCon, 1}`-
// shaped positional compound literal (two sibling bitfields packed
// into one word) used as the last of 5 register-class call arguments:
// rcc's own `noimm()` classification read back `.type` as `.val`'s bit
// pattern because the bitfield merge OR'd a register with itself; a
// second, narrower repro (no intervening struct args before the
// literal) instead clobbered the store's own address register and
// segfaulted on the write-through.
VReg alloc_reg_avoid2(VReg avoid1, VReg avoid2) {
    for (int i = 0; i < NUM_REGS; i++) {
        if (i == avoid1 || i == avoid2) continue;
        if ((used_regs & (1 << i)) == 0) {
            used_regs |= (1 << i);
            ever_used_regs |= (1 << i);
            return (VReg)i;
        }
    }
    for (int i = NUM_REGS - 1; i >= 0; i--) {
        if (i == avoid1 || i == avoid2) continue;
        if ((used_regs & (1 << i)) && !(spilled_regs & (1 << i))) {
            if (opt_W) {
                if (reg_owner[i])
                    fprintf(stderr, "\033[1;33mwarning:\033[0m spilling %s (%s) to stack in %s\n", reg64[i], reg_owner[i], current_fn);
                else
                    fprintf(stderr, "\033[1;33mwarning:\033[0m spilling %s to stack in %s\n", reg64[i], current_fn);
            }
#ifdef ARCH_ARM64
            asm_stur_fp(cg_sec, i, push_spill_slot(i));
#else
            asm_mov_reg_rbp(cg_sec, i, 8, push_spill_slot(i));
#endif
            spilled_regs |= (1 << i);
            spill_count++;
            ever_used_regs |= (1 << i);
            return (VReg)i;
        }
    }
    for (int i = NUM_REGS - 1; i >= 0; i--) {
        if (i == avoid1 || i == avoid2) continue;
        if (used_regs & (1 << i)) {
            if (opt_W) {
                if (reg_owner[i])
                    fprintf(stderr, "\033[1;33mwarning:\033[0m spilling %s (%s) to stack in %s\n", reg64[i], reg_owner[i], current_fn);
                else
                    fprintf(stderr, "\033[1;33mwarning:\033[0m spilling %s to stack in %s\n", reg64[i], current_fn);
            }
#ifdef ARCH_ARM64
            asm_stur_fp(cg_sec, i, push_spill_slot(i));
#else
            asm_mov_reg_rbp(cg_sec, i, 8, push_spill_slot(i));
#endif
            spilled_regs |= (1 << i);
            spill_count++;
            ever_used_regs |= (1 << i);
            return (VReg)i;
        }
    }
    // Every OTHER register is exhausted too (needs >= NUM_REGS
    // concurrently-live values beyond the two avoided ones) -- fall
    // back to the ordinary allocator; a self-aliasing collision is
    // still better than an outright crash, and this is far beyond any
    // register pressure real call sites of this helper produce today.
    return alloc_reg();
}

void free_reg(VReg i) {
    if (spilled_regs & (1 << i)) {
#ifdef ARCH_ARM64
        asm_ldur_fp(cg_sec, i, pop_spill_slot(i)); // ldr x(i), [x29, #-pop_spill_slot(i)]
#else
        asm_mov_rbp_reg(cg_sec, i, 8, pop_spill_slot(i)); // mov [rbp-8], ri
#endif
        spilled_regs &= ~(1 << i);
    }
    used_regs &= ~(1 << i);
    reg_owner[i] = NULL;
}

// Ensure a VReg's value is actually sitting in its physical register,
// reloading from its spill slot first if a later alloc_reg() call stole
// the register out from under it. Needed anywhere a VReg is computed once
// (e.g. a call's register-passed argument, evaluated before the call's
// other arguments) and then read again after intervening codegen — the
// intervening code may run alloc_reg() under register pressure and spill
// this VReg's register to make room, and unlike free_reg() (which is only
// called when the VReg's owner is actually done with it), nothing else
// notices that this VReg's physical register now holds someone else's
// value. A plain `REG(r)` read at that point returns a stranger's data,
// not r's. Idempotent: a no-op when r isn't currently spilled.

static void materialize_reg(VReg r) {
    if (spilled_regs & (1 << r)) {
#ifdef ARCH_ARM64
        asm_ldur_fp(cg_sec, r, pop_spill_slot(r)); // ldr x(r), [x29, #-pop_spill_slot(r)]
#else
        asm_mov_rbp_reg(cg_sec, r, 8, pop_spill_slot(r)); // mov [rbp-pop_spill_slot(r)], rr
#endif
        spilled_regs &= ~(1 << r);
    }
}

#ifndef ARCH_ARM64
#ifdef _WIN32
// MinGW emulated TLS: load the address of a thread-local variable by calling
// __emutls_get_address with a pointer to the variable's control block.
static void emit_emutls_addr(VReg dst, const char *label) {
    bool save0 = (used_regs & (1 << 0)) && dst != 0;
    bool save1 = (used_regs & (1 << 1)) && dst != 1;
    if (save0) asm_mov_phyreg_rbp(cg_sec, X86_R10, 8, spill_offset(0));
    if (save1) asm_mov_phyreg_rbp(cg_sec, X86_R11, 8, spill_offset(1));

    VReg tmp = alloc_reg();
    const char *ctrl = format("__emutls_v.%s", label);
    asm_lea_rip_reg(cg_sec, tmp, ctrl);
    x86_mov_rr(cg_sec, 8, X86_RCX, REG(tmp));
    free_reg(tmp);

    x86_sub_ri(cg_sec, 8, X86_RSP, 32);
    size_t call_off = asm_call_label(cg_sec);
    int sidx = objfile_find_sym(cg_obj, "__emutls_get_address");
    if (sidx < 0)
        sidx = objfile_add_sym(cg_obj, "__emutls_get_address", SEC_UNDEF, 0, 0, SB_GLOBAL, ST_FUNC);
    objfile_add_reloc(cg_obj, SEC_TEXT, call_off + 1, sidx, R_X86_64_PC32, -4);
    x86_add_ri(cg_sec, 8, X86_RSP, 32);

    x86_mov_rr(cg_sec, 8, REG(dst), X86_RAX);
    if (save1) asm_mov_rbp_reg(cg_sec, 1, 8, spill_offset(1));
    if (save0) asm_mov_rbp_reg(cg_sec, 0, 8, spill_offset(0));
}
#endif
#endif

#ifdef ARCH_ARM64
// Minimal ARM64 inline-asm template validator.
// Reports rcc-style errors for invalid mnemonics and range violations
// so that test 139 gets our messages instead of gas messages.
static void arm64_validate_asm_template(const char *tmpl, Token *tok) {
    // Known ARM64 mnemonics (comprehensive but not exhaustive)
    static const char *const known[] = {
        "add", "adds", "sub", "subs", "neg", "negs", "mul", "madd", "msub", "mneg",
        "div", "udiv", "sdiv", "smull", "umull", "smulh", "umulh",
        "and", "ands", "orr", "eor", "bic", "bics", "orn", "eon",
        "lsl", "lsr", "asr", "ror", "extr",
        "mov", "movz", "movk", "movn",
        "adrp", "adr",
        "ldr", "ldrb", "ldrh", "ldrsb", "ldrsh", "ldrsw",
        "ldur", "ldurb", "ldurh", "ldursb", "ldursh", "ldursw",
        "str", "strb", "strh", "stur", "sturb", "sturh",
        "ldp", "stp", "ldnp", "stnp",
        "b", "bl", "blr", "br", "ret",
        "cbz", "cbnz", "tbz", "tbnz",
        "b.eq", "b.ne", "b.lt", "b.le", "b.gt", "b.ge",
        "b.cc", "b.cs", "b.hi", "b.ls", "b.mi", "b.pl", "b.vs", "b.vc",
        "b.lo", "b.hs", "b.al", "b.nv", "b.eq", "beq", "bne", "blt", "ble", "bgt", "bge",
        "mrs", "msr", "nop", "brk", "svc", "hlt",
        "dmb", "dsb", "isb",
        "csel", "cset", "csetm", "csinc", "csinv", "csneg",
        "clz", "cls", "rbit", "rev", "rev16", "rev32",
        "sbfm", "ubfm", "bfm", "sbfx", "ubfx", "bfi", "bfxil",
        "sxth", "sxtw", "sxtb", "uxtb", "uxth",
        "fmov", "fadd", "fsub", "fmul", "fdiv", "fneg", "fabs", "fsqrt",
        "fcmp", "fccmp", "fcmpe", "fccmpe",
        "fcvt", "fcvtzu", "fcvtzs", "scvtf", "ucvtf", "fcvtas", "fcvtau",
        "fcvtms", "fcvtmu", "fcvtns", "fcvtnu", "fcvtps", "fcvtpu",
        "fmax", "fmin", "fmaxnm", "fminnm",
        "paciasp", "autiasp", "pacibsp", "autibsp", "xpaclri",
        "prfm", "prefetch",
        "eret", "drps", "eor",
        "lda", "ldaex", "stl", "stlex", "stlr", "ldar", "ldaxr", "stlxr",
        "ldxr", "stxr", "ldaxp", "stlxp", "ldxp", "stxp",
        "clrex", "yield", "wfe", "wfi", "sev", "sevl",
        "hint", "sys", "sysl", "at", "dc", "ic", "tlbi", "msr", "mrs",
        NULL};

    const char *p = tmpl;
    while (*p) {
        // skip whitespace / separators
        while (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r' ||
               *p == ';') p++;
        if (!*p) break;

        // skip operand substitutions (%N, %w0, %[name], etc.)
        if (*p == '%') {
            while (*p && *p != ';' && *p != '\n') p++;
            continue;
        }

        // Extract mnemonic (first word)
        char mnem[64];
        int mlen = 0;
        while (*p && *p != ' ' && *p != '\t' && *p != ';' && *p != '\n' && *p != ':' && mlen < 63)
            mnem[mlen++] = tolower((unsigned char)*p++);
        mnem[mlen] = '\0';
        if (!mlen) continue;

        // Label definition (e.g. ".L_skip:") — not an instruction, skip it.
        if (*p == ':') {
            p++;
            continue;
        }

        // Skip to rest of instruction
        while (*p == ' ' || *p == '\t') p++;
        const char *operands = p;

        // Validate mnemonic
        bool found = false;
        for (int j = 0; known[j]; j++) {
            if (strcmp(mnem, known[j]) == 0) {
                found = true;
                break;
            }
        }
        if (!found) {
            // skip rest of line and continue (unknown mnemonic — let the
            // assembler handle it, matching GCC's dg-do compile behavior)
            while (*p && *p != ';' && *p != '\n') p++;
            continue;
        }

        // --- Specific instruction validation ---

        // LSL/LSR/ASR immediate shift range
        if ((strcmp(mnem, "lsl") == 0 || strcmp(mnem, "lsr") == 0 || strcmp(mnem, "asr") == 0) &&
            operands[0] && operands[0] != '%') {
            bool is32 = (tolower((unsigned char)operands[0]) == 'w');
            const char *hash = NULL;
            for (const char *q = operands; *q && *q != ';' && *q != '\n'; q++)
                if (*q == '#') {
                    hash = q + 1;
                    break;
                }
            if (hash) {
                long val = strtol(hash, NULL, 0);
                int maxshift = is32 ? 31 : 63;
                if (val < 0 || val > maxshift)
                    error_tok_simple(tok, "shift immediate out of range");
            }
        }

        // MRS: validate system register name
        if (strcmp(mnem, "mrs") == 0) {
            static const char *const sysregs[] = {
                "fpcr", "fpsr", "nzcv", "daif", "currentel", "spsel",
                "midr_el1", "mpidr_el1", "id_aa64pfr0_el1", "id_aa64pfr1_el1",
                "id_aa64dfr0_el1", "id_aa64dfr1_el1", "id_aa64isar0_el1",
                "id_aa64isar1_el1", "id_aa64mmfr0_el1", "id_aa64mmfr1_el1",
                "sctlr_el1", "tcr_el1", "ttbr0_el1", "ttbr1_el1", "mair_el1",
                "tpidr_el0", "tpidrro_el0", "tpidr_el1",
                "sp_el0", "elr_el1", "spsr_el1",
                "cntkctl_el1", "cntpct_el0", "cntvct_el0", "cntfrq_el0",
                "pmcr_el0", "pmcntenset_el0", "pmccntr_el0", "pmintenset_el1",
                "esr_el1", "far_el1", "par_el1", "afsr0_el1", "afsr1_el1",
                "revidr_el1", "aidr_el1", "csselr_el1", "clidr_el1", "ctr_el0",
                "dczid_el0", "isr_el1", "vbar_el1", "rvbar_el1", "rmr_el1",
                "contextidr_el1", "tpidr_el0", "tpidrro_el0", "amair_el1",
                NULL};
            const char *comma = strchr(operands, ',');
            if (comma) {
                const char *r = comma + 1;
                while (*r == ' ' || *r == '\t') r++;
                char sysreg[64];
                int sl = 0;
                while (*r && *r != ' ' && *r != '\t' && *r != '\n' && *r != ';' && sl < 63)
                    sysreg[sl++] = tolower((unsigned char)*r++);
                sysreg[sl] = '\0';
                if (*sysreg && *sysreg != '%') {
                    bool ok = false;
                    for (int j = 0; sysregs[j]; j++)
                        if (strcmp(sysreg, sysregs[j]) == 0) {
                            ok = true;
                            break;
                        }
                    if (!ok)
                        error_tok_simple(tok, "unsupported system register");
                }
            }
        }

        // MSR: validate system register name (first operand)
        if (strcmp(mnem, "msr") == 0) {
            static const char *const sysregs_msr[] = {
                "fpcr", "fpsr", "nzcv", "daif", "currentel", "spsel",
                "sctlr_el1", "tcr_el1", "ttbr0_el1", "ttbr1_el1", "mair_el1",
                "tpidr_el0", "tpidr_el1", "vbar_el1", "contextidr_el1", "amair_el1",
                "cntkctl_el1", "pmcr_el0", "pmcntenset_el0", "pmintenset_el1",
                NULL};
            char sysreg[64];
            int sl = 0;
            const char *r = operands;
            while (*r == ' ' || *r == '\t') r++;
            while (*r && *r != ',' && *r != ' ' && *r != '\t' && *r != '\n' && sl < 63)
                sysreg[sl++] = tolower((unsigned char)*r++);
            sysreg[sl] = '\0';
            (void)sysregs_msr;
            (void)sysreg; // validated by gas; don't double-error
        }

        // DMB/DSB: validate barrier option
        if (strcmp(mnem, "dmb") == 0 || strcmp(mnem, "dsb") == 0) {
            static const char *const opts[] = {
                "sy", "st", "ld", "ish", "ishst", "ishld",
                "nsh", "nshst", "nshld", "osh", "oshst", "oshld",
                "full", NULL};
            char opt[32];
            int ol = 0;
            const char *r = operands;
            while (*r == ' ' || *r == '\t') r++;
            while (*r && *r != ' ' && *r != '\t' && *r != '\n' && *r != ';' && ol < 31)
                opt[ol++] = tolower((unsigned char)*r++);
            opt[ol] = '\0';
            if (*opt && *opt != '%') {
                // allow numeric
                bool ok = (*opt >= '0' && *opt <= '9');
                for (int j = 0; !ok && opts[j]; j++)
                    if (strcmp(opt, opts[j]) == 0) ok = true;
                if (!ok)
                    error_tok_simple(tok, "invalid operand '%s'", opt);
            }
        }

        // ADD/SUB: check for missing third operand (when operands are literal)
        if ((strcmp(mnem, "add") == 0 || strcmp(mnem, "sub") == 0) &&
            operands[0] && operands[0] != '%') {
            int ncommas = 0;
            for (const char *q = operands; *q && *q != ';' && *q != '\n'; q++)
                if (*q == ',') ncommas++;
            if (ncommas < 2)
                error_tok_simple(tok, "missing third operand");
        }

        // MOVZ/MOVK: check immediate value and shift
        if (strcmp(mnem, "movz") == 0 || strcmp(mnem, "movk") == 0) {
            const char *hash = NULL;
            for (const char *q = operands; *q && *q != ';' && *q != '\n'; q++)
                if (*q == '#') {
                    hash = q + 1;
                    break;
                }
            if (hash) {
                long val = strtol(hash, NULL, 0);
                if (val < 0 || val > 65535)
                    error_tok_simple(tok, "move wide immediate out of range");
                // Check for lsl shift
                const char *comma2 = NULL;
                {
                    // Find comma after the first arg (dest reg)
                    const char *q = operands;
                    int nc = 0;
                    while (*q && *q != ';' && *q != '\n') {
                        if (*q == ',') {
                            nc++;
                            if (nc == 2) {
                                comma2 = q;
                                break;
                            }
                        }
                        q++;
                    }
                }
                if (comma2) {
                    const char *h2 = strchr(comma2, '#');
                    if (h2) {
                        long shift = strtol(h2 + 1, NULL, 0);
                        if (shift != 0 && shift != 16 && shift != 32 && shift != 48)
                            error_tok_simple(tok, "move wide shift out of range");
                    }
                }
            }
        }

        // Skip rest of this instruction
        while (*p && *p != ';' && *p != '\n') p++;
    }
}
#endif // ARCH_ARM64

// Try to extract the symbol name of an address-of-global expression
// (traversing casts), e.g. "i" (key) where an arch_static_branch() caller
// passed &some_static_key — GCC accepts this as an "i" (immediate) operand
// since the address is a link-time constant, emitted as "$symbol" (an
// immediate holding the symbol's address) the same way a numeric
// immediate is emitted as "$N".
#ifndef ARCH_ARM64 // only used by the x86-64 inline-asm "i"/"n" operand path
static bool try_const_addr_sym(Node *n, const char **sym_out) {
    while (n && n->kind == ND_CAST)
        n = n->lhs;
    if (n && n->kind == ND_ADDR && n->lhs && n->lhs->kind == ND_LVAR && !n->lhs->var->is_local) {
        *sym_out = asm_sym_name(var_sym_label(n->lhs->var));
        return true;
    }
    return false;
}
#endif // !ARCH_ARM64
// Try to extract an integer constant from a node (traversing casts).
// Returns true and sets *val if the node reduces to a compile-time constant.
static bool try_const_int(Node *n, int64_t *val) {
    while (n && n->kind == ND_CAST)
        n = n->lhs;
    if (n && n->kind == ND_NUM) {
        *val = n->val;
        return true;
    }
    return false;
}
// Resolve a GCC "global register variable" asm(...) register name (e.g.
// "rsp" on x86-64, "sp" on arm64) to the physical register it pins, or -1
// if unrecognized. Only the names the kernel headers we build actually use
// need to resolve (x86-64's `current_stack_pointer asm("rsp")`, arm64's
// `asm("sp")`) — anything else returns -1 so the caller can fall back
// instead of silently mis-decoding a register it doesn't understand.
#ifdef ARCH_ARM64
static int global_reg_lookup(const char *name) {
    if (!strcmp(name, "sp")) return ARM64_SP;
    if (!strcmp(name, "fp") || !strcmp(name, "x29")) return ARM64_X29;
    if (!strcmp(name, "lr") || !strcmp(name, "x30")) return ARM64_X30;
    if (name[0] == 'x' && name[1] >= '0' && name[1] <= '9') {
        int n = atoi(name + 1);
        if (n >= 0 && n <= 30) return n; // Arm64Reg Xn is numbered n
    }
    return -1;
}
#else
static int global_reg_lookup(const char *name) {
    static const struct {
        const char *name;
        X86Reg reg;
    } tab[] = {
        {"rax", X86_RAX},
        {"rbx", X86_RBX},
        {"rcx", X86_RCX},
        {"rdx", X86_RDX},
        {"rsp", X86_RSP},
        {"rbp", X86_RBP},
        {"rsi", X86_RSI},
        {"rdi", X86_RDI},
        {"r8", X86_R8},
        {"r9", X86_R9},
        {"r10", X86_R10},
        {"r11", X86_R11},
        {"r12", X86_R12},
        {"r13", X86_R13},
        {"r14", X86_R14},
        {"r15", X86_R15},
    };
    for (size_t i = 0; i < sizeof(tab) / sizeof(tab[0]); i++)
        if (!strcmp(name, tab[i].name)) return tab[i].reg;
    return -1;
}
#endif

// Materialize `r = <hardware register named by var->global_reg_name>` for a
// read of a GCC global register variable (LVar.is_global_reg — see rcc.h).
// Such a variable has no address and no backing memory at all in
// conforming GCC/Clang; taking one is already rejected elsewhere (the same
// `is_register` check that rejects `&register int x`). Returns false
// (emits nothing) on an unrecognized register name so the caller can fall
// back to its ordinary global-variable path rather than miscompile it.
static bool gen_global_reg_read(VReg r, LVar *var) {
    int reg = global_reg_lookup(var->global_reg_name);
    if (reg < 0) return false;
#ifdef ARCH_ARM64
    if (reg == ARM64_SP)
        arm64_add_imm(cg_sec, 1, REG(r), ARM64_SP, 0, 0); // mov r, sp  (add r, sp, #0)
    else
        arm64_orr_reg(cg_sec, 1, REG(r), ARM64_XZR, (Arm64Reg)reg, ARM64_LSL, 0); // mov r, xN
#else
    x86_mov_rr(cg_sec, 8, REG(r), (X86Reg)reg);
#endif
    return true;
}
// GNU nested functions: walk `depth` levels up through saved static-chain
// pointers, leaving the ancestor frame's own frame-pointer value in `r`.
// depth==1 is a single load of the CURRENT function's own chain slot (its
// incoming static chain, spilled at prologue entry — see the
// `fn->is_nested` prologue block below); depth>1 repeats the load through
// each intermediate ancestor's own chain slot, which lives at the same
// CHAIN_SLOT_OFFSET in every nested function's frame by construction (see
// parser.c's push_fn_ctx). Caller combines the result with the variable's
// own `->offset` in that ancestor's frame. Static-chain register is %r10
// (x86-64 SysV convention, also used unmodified on the MS x64 ABI since
// r10 is scratch/volatile there too) or x18 (AArch64: GCC's
// STATIC_CHAIN_REGNUM, already excluded from this file's ARM64 register
// pool — see the "x16-x18: reserved" comment above).
static void gen_chain_walk_into(VReg r, int depth) {
#ifdef ARCH_ARM64
    arm64_ldur(cg_sec, 1, REG(r), ARM64_X29, -CHAIN_SLOT_OFFSET); // ldur r, [x29, #-CHAIN_SLOT_OFFSET]
    for (int i = 1; i < depth; i++)
        arm64_ldur(cg_sec, 1, REG(r), REG(r), -CHAIN_SLOT_OFFSET); // ldur r, [r, #-CHAIN_SLOT_OFFSET]
#else
    asm_mov_rbp_reg(cg_sec, r, 8, CHAIN_SLOT_OFFSET); // r = [rbp - CHAIN_SLOT_OFFSET]
    for (int i = 1; i < depth; i++)
        x86_mov_rm(cg_sec, 8, REG(r), x86_mem(REG(r), -CHAIN_SLOT_OFFSET)); // r = [r - CHAIN_SLOT_OFFSET]
#endif
}

VReg gen_addr(Node *node) {
    // A vector arithmetic/compare/unary result is materialized into a 16-byte
    // slot; its "address" is that slot (used when a vector value is returned by
    // value or otherwise needs an address).
    if (node->ty && node->ty->is_vector) {
        switch (node->kind) {
        case ND_ADD:
        case ND_SUB:
        case ND_MUL:
        case ND_DIV:
        case ND_MOD:
        case ND_BITAND:
        case ND_BITOR:
        case ND_BITXOR:
        case ND_EQ:
        case ND_NE:
        case ND_LT:
        case ND_LE:
        case ND_NEG:
        case ND_BITNOT:
#ifndef ARCH_ARM64
            if (node->ty->size == 64)
                return gen_vector64_x86(node);
            if (node->ty->size == 32)
                return gen_vector32_x86(node);
#endif
            return gen_vector(node);
        default:
            break;
        }
    }
    // codeql[cpp/long-switch]: central AST-node-kind dispatch; splitting cases into helpers is a large, purely-cosmetic refactor of core compiler internals, not attempted here.
    switch (node->kind) {
    case ND_LVAR: {
        //fprintf(stderr, "DEBUG ND_LVAR: %s is_local=%d is_function=%d is_weak=%d\n",
        //        node->var->name, node->var->is_local, node->var->is_function, node->var->is_weak);
        VReg r = alloc_reg();
        if (opt_W) reg_owner[r] = node->var->name;
        if (node->var->is_local) {
            bool is_vla = node->var->ty->kind == TY_VLA || ((node->var->ty->kind == TY_STRUCT || node->var->ty->kind == TY_UNION) && node->var->ty->vla_len_expr);
            if (node->chain_depth > 0) {
                VReg base = alloc_reg();
                gen_chain_walk_into(base, node->chain_depth);
                if (is_vla) {
                    // VLA locals are stored as fat pointers: the "address"
                    // used by callers (e.g. array subscript lowering) is
                    // the pointer *value* at ancestor_frame-(offset-8),
                    // not the address of that pointer's own storage slot.
#ifdef ARCH_ARM64
                    arm64_load_from_reg_minus(REG(base), node->var->offset - 8, REG(r));
#else
                    x86_mov_rm(cg_sec, 8, REG(r), x86_mem(REG(base), -(node->var->offset - 8))); // mov -(offset-8)(%rN), r
#endif
                } else {
#ifdef ARCH_ARM64
                    if (node->var->offset <= 4095)
                        arm64_sub_imm(cg_sec, 1, REG(r), REG(base), node->var->offset, 0); // sub r, base, #offset
                    else {
                        emit_mov_imm64(REG(r), (uint64_t)node->var->offset); // mov r, #offset
                        arm64_sub_reg(cg_sec, 1, REG(r), REG(base), REG(r), ARM64_LSL, 0); // sub r, base, r
                    }
#else
                    x86_lea(cg_sec, 8, REG(r), x86_mem(REG(base), -node->var->offset)); // lea -offset(%rN), r
#endif
                }
                free_reg(base);
            } else if (is_vla) {
#ifdef ARCH_ARM64
                arm64_load_from_fp_minus(node->var->offset - 8, REG(r));
#else
                asm_mov_rbp_reg(cg_sec, r, 8, node->var->offset - 8); // mov [rbp-8], rr
#endif
            } else {
#ifdef ARCH_ARM64
                if (node->var->offset <= 4095)
                    asm_sub_reg_fp_imm(cg_sec, r, node->var->offset); // sub r, x29, #node->var->offset
                else {
                    int v = node->var->offset;
                    asm_mov_imm(cg_sec, r, 8, v & 0xffff); // mov r, #v
                    v >>= 16;
                    int s = 16;
                    while (v) {
                        asm_movk(cg_sec, REG(r), 1, (uint16_t)(v & 0xffff), s); // movk r, #v, lsl #s
                        v >>= 16;
                        s += 16;
                    }
                    asm_sub_reg_fp_reg(cg_sec, r, r, 8); // sub r, x29, r
                }
#else
                asm_lea_rbp_reg(cg_sec, r, 8, node->var->offset); // lea [rbp-8], rr
#endif
            }
        } else {
            if (node->var->is_weak)
                cg_weak_declare(asm_sym_name(var_sym_label(node->var)));
#ifdef ARCH_ARM64
            if (node->var->is_tls)
                emit_tls_addr(r, node->var);
            else if (node->var->is_weak)
                emit_adrp_got(r, asm_sym_name(var_sym_label(node->var)));
            else
                emit_adrp_add(r, asm_sym_name(var_sym_label(node->var)));
#else
            if (node->var->is_tls) {
#ifdef _WIN32
                emit_emutls_addr(r, var_label(node->var));
#else
                VReg base = alloc_reg();
                asm_mov_fs0_reg(cg_sec, base);
                asm_lea_tpoff_base_reg(cg_sec, r, base, var_sym_label(node->var));
                free_reg(base);
#endif
            } else {
#ifdef _WIN32
                if (var_needs_refptr(node->var))
                    asm_mov_refptr_rip_reg(cg_sec, r, var_sym_label(node->var));
                else
#endif
                    if (node->var->is_weak || var_needs_got(node->var))
                    asm_mov_got_rip_reg(cg_sec, r, var_sym_label(node->var)); // mov sym@GOTPCREL(%rip), r
                else
                    asm_lea_rip_reg(cg_sec, r, var_sym_label(node->var)); // lea rip, rr
            }
#endif
        }
        return r;
    }
    case ND_DEREF:
        return gen(node->lhs);
    case ND_REAL:
    case ND_IMAG: {
        int r = gen_addr(node->lhs);
        if (r < 0) {
            /* rvalue: eval compound expression, extract real/imag from slot */
            r = gen_to_int128(node->lhs);
        }
        int offset = (node->kind == ND_IMAG) ? node->lhs->ty->base->size : 0;
        if (offset > 0) {
#ifdef ARCH_ARM64
            if (offset <= 4095)
                asm_add_imm(cg_sec, r, 8, offset); // add r, r, #offset
            else {
                VReg ti = alloc_reg();
                emit_mov_imm64(REG(ti), (uint64_t)offset); // mov ti, #offset
                asm_add_reg_reg(cg_sec, r, ti, 8); // add r, r, ti
                free_reg(ti);
            }
#else
            if (offset <= 4095)
                asm_add_imm(cg_sec, r, 8, offset); // add r, r, #offset
            else {
                VReg ti = alloc_reg();
                asm_mov_imm(cg_sec, ti, 8, offset); // mov $offset, ti
                asm_add_reg_reg(cg_sec, r, ti, 8); // add r, r, ti
                free_reg(ti);
            }
#endif
        }
        return r;
    }
    case ND_MEMBER: {
        VReg r = gen_addr(node->lhs);
        if (node->member->offset_expr) {
            VReg o = gen(node->member->offset_expr);
#ifdef ARCH_ARM64
            asm_add_reg_reg(cg_sec, r, o, 8); // add r, r, o
#else
            asm_add_reg_reg(cg_sec, r, o, 8); // add r, r, o
#endif
            free_reg(o);
            return r;
        }
#ifdef ARCH_ARM64
        if (node->member->offset > 0 && node->member->offset <= 4095)
            asm_add_imm(cg_sec, r, 8, node->member->offset); // add r, r, #node->member->offset
        else if (node->member->offset > 0) {
            VReg ti = alloc_reg();
            emit_mov_imm64(REG(ti), (uint64_t)node->member->offset); // mov ti, #node->member->offset
            asm_add_reg_reg(cg_sec, r, ti, 8); // add r, r, ti
            free_reg(ti);
        } else if (node->member->offset < 0 && -node->member->offset <= 4095)
            asm_sub_imm(cg_sec, r, 8, -node->member->offset); // sub r, r, #-node->member->offset
        else if (node->member->offset < 0) {
            VReg ti = alloc_reg();
            emit_mov_imm64(REG(ti), (uint64_t)(-node->member->offset)); // mov ti, #-node->member->offset
            asm_sub_reg_reg(cg_sec, r, ti, 8); // sub r, r, ti
            free_reg(ti);
        }
#else
        asm_add_imm(cg_sec, r, 8, node->member->offset); // add $node->member->offset, rr
#endif
        return r;
    }
    case ND_COND: {
        // Struct/union ternary lvalue: (cond ? a : b).member
        int c = ++rcc_label_count;
        VReg r = alloc_reg();
        VReg cond = gen(node->cond);
        if (node->cond->ty && is_flonum(node->cond->ty)) {
            // IEEE truthiness: -0.0 == 0.0 (false), NaN truthy.
            gen_flonum_branch_if_zero(cond, NULL, format(".L.else.%d", c));
            free_reg(cond);
            VReg then_r = gen_addr(node->then);
            asm_mov_reg_reg(cg_sec, r, then_r, 8); // mov rthen_r -> rr
            free_reg(then_r);
            {
                size_t o = asm_jmp_label(cg_sec); // b .L.end.%d
                asm_fixup_add(cg_sec, o, format(".L.end.%d", c), 0);
            }
            cg_def_label(format(".L.else.%d", c));
            VReg else_r = gen_addr(node->els);
            asm_mov_reg_reg(cg_sec, r, else_r, 8); // mov relse_r -> rr
            free_reg(else_r);
            cg_def_label(format(".L.end.%d", c));
            return r;
        }
        int cond_sz = node->cond->ty->size;
#ifdef ARCH_ARM64
        asm_cmp_zero(cg_sec, cond, cond_sz); // cmp $0, rcond
        {
            size_t o = asm_jcc_label(cg_sec, ARM64_EQ); // jcc label
            asm_fixup_add(cg_sec, o, format(".L.else.%d", c), 1);
        }
        free_reg(cond);
        int then_r = gen_addr(node->then);
        asm_mov_reg_reg(cg_sec, r, then_r, 8); // mov rthen_r -> rr
        free_reg(then_r);
        {
            size_t o = asm_jmp_label(cg_sec); // b .L.end.%d
            asm_fixup_add(cg_sec, o, format(".L.end.%d", c), 0);
        }
        cg_def_label(format(".L.else.%d", c));
        VReg else_r = gen_addr(node->els);
        asm_mov_reg_reg(cg_sec, r, else_r, 8); // mov relse_r -> rr
        free_reg(else_r);
        cg_def_label(format(".L.end.%d", c));
#else
        asm_cmp_zero(cg_sec, cond, cond_sz); // cmp $0, rcond
        {
            size_t o = asm_jcc_label(cg_sec, X86_E); // jcc label
            asm_fixup_add(cg_sec, o, format(".L.else.%d", c), 1);
        }
        free_reg(cond);
        VReg then_r = gen_addr(node->then);
        asm_mov_reg_reg(cg_sec, r, then_r, 8); // mov rthen_r -> rr
        free_reg(then_r);
        {
            size_t o = asm_jmp_label(cg_sec); // je .L.else.%d
            asm_fixup_add(cg_sec, o, format(".L.end.%d", c), 0);
        }
        cg_def_label(format(".L.else.%d", c));
        VReg else_r = gen_addr(node->els);
        asm_mov_reg_reg(cg_sec, r, else_r, 8); // mov relse_r -> rr
        free_reg(else_r);
        cg_def_label(format(".L.end.%d", c));
#endif
        return r;
    }
    case ND_CAST:
        if (node->ty && node->ty->is_vector) {
            // Vector cast: same-size vector -> vector is a bitcast (the
            // address is unchanged); scalar -> vector broadcasts the scalar
            // to every lane (GCC semantics), materialized in a slot.
            if (node->lhs && node->lhs->ty && node->lhs->ty->is_vector)
                return gen_addr(node->lhs);
#ifdef ARCH_ARM64
            // NEON DUP broadcast. Float scalars are double bit patterns in
            // GP registers — narrow to single first (arm64 dup takes the
            // low bytes of the GP register).
            Type *elem = node->ty ? node->ty->base : NULL;
            int esz = elem ? (int)elem->size : 4;
            VReg r = gen(node->lhs);
            VReg dst = alloc_int128_addr();
            if (elem && is_flonum(elem) && esz == 4) {
                asm_fmov_i2f(cg_sec, 0, r, 1); // fmov d0, x{r}
                asm_fcvt(cg_sec, 0, 1, ARM64_S0, ARM64_D0); // fcvt s0, d0
                asm_fmov_f2i(cg_sec, r, ARM64_S0, 0); // fmov w{r}, s0
            }
            // DUP (general): broadcast the low esz bytes of w{r} to every
            // lane. arm64_dup_gen only supports 32/64-bit; byte and
            // halfword elements need imm5 = esz directly.
            if (esz == 8 || esz == 4)
                arm64_dup_gen(cg_sec, esz == 8 ? 3 : 2, ASM_Q0, REG(r));
            else
                secbuf_emit32le(cg_sec, 0x4E000C00u | ((uint32_t)esz << 16) | ((uint32_t)(REG(r) & 31) << 5) | (uint32_t)(ASM_Q0 & 31));
            asm_str_q(cg_sec, ASM_Q0, dst); // str q0, [x{dst}]
            free_reg(r);
            return dst;
#else
            return gen_vector_splat(node->lhs, node->ty);
#endif
        }
        if (node->ty && (node->ty->kind == TY_STRUCT || node->ty->kind == TY_UNION || node->ty->kind == TY_COMPLEX || node->ty->kind == TY_ARRAY)) {
            // Complex -> complex cast with a differently-sized base type of
            // the same kind (e.g. _Complex char -> _Complex int, from the
            // usual arithmetic conversions). The source storage doesn't
            // match node->ty's size, so handing back gen_addr(node->lhs)'s
            // address as-is would make callers read/write past the smaller
            // object. Materialize a converted copy on the stack instead.
            if (node->ty->kind == TY_COMPLEX && node->lhs->ty && node->lhs->ty->kind == TY_COMPLEX &&
                (node->lhs->ty->base->size != node->ty->base->size ||
                 is_flonum(node->lhs->ty->base) != is_flonum(node->ty->base))) {
                VReg src = gen_addr(node->lhs);
                if (src < 0)
                    src = gen(node->lhs); // gen() for complex exprs also returns an address
                int alloc = (node->ty->size + 7) & ~7;
                fn_struct_ret_off += alloc;
                if (fn_struct_ret_off > fn_struct_ret_total) fn_struct_ret_total = fn_struct_ret_off;
                int result_off = current_fn_stack_size + fn_struct_ret_off;
                VReg dst = alloc_reg();
#ifdef ARCH_ARM64
                asm_sub_reg_fp_imm(cg_sec, dst, result_off); // sub dst, x29, #result_off
#else
                asm_lea_rbp_reg(cg_sec, dst, 8, result_off); // lea -result_off(%rbp), dst
#endif
                if (is_flonum(node->lhs->ty->base) && is_flonum(node->ty->base))
                    emit_complex_convert_float(src, dst, node->lhs->ty, node->ty);
                else if (!is_flonum(node->lhs->ty->base) && !is_flonum(node->ty->base))
                    emit_complex_convert_int(src, dst, node->lhs->ty, node->ty);
                else {
#ifndef ARCH_ARM64
                    // Mixed int/float complex: convert each component
                    // int complex -> float complex: load ints, convert to float
                    if (is_flonum(node->ty->base)) {
                        int fsz = node->lhs->ty->base->size;
                        int tsz = node->ty->base->size;
                        bool uns = node->lhs->ty->base->is_unsigned;
                        if (fsz == 8) {
                            x86_mov_rm(cg_sec, 8, X86_RAX, x86_mem(REG(src), 0));
                            x86_mov_rm(cg_sec, 8, X86_RCX, x86_mem(REG(src), 8));
                        } else {
                            // Sign/zero-extend narrower int to 32/64-bit before cvtsi2sd
                            int load_sz = fsz <= 4 ? 4 : 8;
                            if (uns) {
                                x86_mov_rm(cg_sec, fsz, X86_RAX, x86_mem(REG(src), 0));
                                x86_mov_rm(cg_sec, fsz, X86_RCX, x86_mem(REG(src), fsz));
                            } else {
                                x86_movsx_rm(cg_sec, load_sz, fsz, X86_RAX, x86_mem(REG(src), 0));
                                x86_movsx_rm(cg_sec, load_sz, fsz, X86_RCX, x86_mem(REG(src), fsz));
                            }
                        }
                        x86_cvtsi2sd(cg_sec, fsz <= 4 ? 4 : 8, X86_XMM0, X86_RAX);
                        x86_cvtsi2sd(cg_sec, fsz <= 4 ? 4 : 8, X86_XMM1, X86_RCX);
                        if (tsz == 4) {
                            x86_cvtsd2ss(cg_sec, X86_XMM0, X86_XMM0);
                            x86_cvtsd2ss(cg_sec, X86_XMM1, X86_XMM1);
                            x86_movss_mr(cg_sec, x86_mem(REG(dst), 0), X86_XMM0);
                            x86_movss_mr(cg_sec, x86_mem(REG(dst), 4), X86_XMM1);
                        } else {
                            x86_movsd_mr(cg_sec, x86_mem(REG(dst), 0), X86_XMM0);
                            x86_movsd_mr(cg_sec, x86_mem(REG(dst), 8), X86_XMM1);
                        }
                    } else {
                        // float complex -> int complex: load doubles, convert to ints
                        int tsz = node->ty->base->size;
                        x86_movsd_rm(cg_sec, X86_XMM0, x86_mem(REG(src), 0));
                        x86_movsd_rm(cg_sec, X86_XMM1, x86_mem(REG(src), 8));
                        x86_cvttsd2si(cg_sec, tsz <= 4 ? 4 : 8, X86_RAX, X86_XMM0);
                        x86_cvttsd2si(cg_sec, tsz <= 4 ? 4 : 8, X86_RCX, X86_XMM1);
                        if (tsz == 8) {
                            x86_mov_mr(cg_sec, 8, x86_mem(REG(dst), 0), X86_RAX);
                            x86_mov_mr(cg_sec, 8, x86_mem(REG(dst), 8), X86_RCX);
                        } else if (tsz == 4) {
                            x86_mov_mr(cg_sec, 4, x86_mem(REG(dst), 0), X86_RAX);
                            x86_mov_mr(cg_sec, 4, x86_mem(REG(dst), 4), X86_RCX);
                        } else {
                            x86_mov_mr(cg_sec, tsz, x86_mem(REG(dst), 0), X86_RAX);
                            x86_mov_mr(cg_sec, tsz, x86_mem(REG(dst), tsz), X86_RCX);
                        }
                    }
#else
                    // ARM64: fall through to gen() for mixed complex conversion
                    free_reg(src);
                    free_reg(dst);
                    return gen(node);
#endif
                }
                free_reg(src);
                return dst;
            }
            VReg r = gen_addr(node->lhs);
            if (r >= 0) return r;
            // Non-lvalue cast to complex/struct: materialize on stack
            if (node->ty->kind == TY_COMPLEX) {
                Type *from = node->lhs->ty;
                if (from && (is_flonum(from) || is_integer(from)))
                    // Scalar -> complex: delegate to gen(), which performs the
                    // proper int/float conversion to the complex base type and
                    // materializes the {real, imag} pair on the stack.
                    return gen(node);
                return -1;
            }
            return -1;
        }
        // Scalar cast: not an lvalue, caller can fall back to gen()
        return -1;
    case ND_COMMA: {
        // Compound literal: evaluate LHS side effects, return address of RHS
        VReg r1 = gen(node->lhs);
        if (r1 != -1) free_reg(r1);
        return gen_addr(node->rhs);
    }
    case ND_FUNCALL:
        if (node->ty && (node->ty->kind == TY_STRUCT || node->ty->kind == TY_UNION || node->ty->kind == TY_COMPLEX))
            return gen_funcall(node, -1);
        error_tok(node->tok, "lvalue required as left operand of assignment");
        return -1;
    case ND_ASSIGN: {
        // Assignment expression used as an lvalue (e.g. (a = b).member,
        // &(a = b), or an outer struct-assign whose RHS is itself an
        // assignment): the store must actually happen here, not just the
        // address of its target -- returning gen_addr(node->lhs) alone
        // (the old behavior) silently dropped the assignment's side effect,
        // leaving `lhs` unmodified/uninitialized while callers read through
        // its address as if the store had occurred. For struct/union/array/
        // complex targets, gen() already performs the copy and returns the
        // destination address directly, so use that; for scalar targets
        // gen() returns the assigned *value*, so discard it and take the
        // (side-effect-free re-read of the) target's address afterward.
        Type *lt = node->lhs->ty;
        if (lt && (lt->kind == TY_STRUCT || lt->kind == TY_UNION || lt->kind == TY_ARRAY || lt->kind == TY_COMPLEX))
            return gen(node);
        VReg r = gen(node);
        if (r != -1) free_reg(r);
        return gen_addr(node->lhs);
    }
    case ND_STMT_EXPR: {
        // Statement expression used as lvalue (e.g. d = ({ bar(); }))
        // Evaluate the block and return address of the last expression
        VReg result = -1;
        for (Node *n = node->body; n; n = n->next) {
            VReg r = gen(n);
            if (node->stmt_expr_result && n->kind == ND_EXPR_STMT && n->lhs == node->stmt_expr_result) {
                result = gen_addr(node->stmt_expr_result);
            }
            if (r != -1) free_reg(r);
        }
        if (result != -1)
            return result;
        error_tok(node->tok, "lvalue required as left operand of assignment");
        return -1;
    }
    case ND_NUM:
    case ND_FNUM:
        return -1; // not an lvalue; caller should handle via temp allocation
    case ND_ADD:
    case ND_SUB:
    case ND_MUL:
    case ND_DIV:
    case ND_MOD:
        return -1; // expression result, not an lvalue
    case ND_BITNOT:
    case ND_NEG:
        return -1; // expression result, not an lvalue
    default:
        error_tok(node->tok, "lvalue required as left operand of assignment");
        return -1;
    }
}

// Store a forward branch fixup. Uses fwd_off (direct patch) if non-NULL,
// otherwise shared_label (hash table) if non-NULL,
// otherwise auto-generates and resolves immediately.
static void cg_add_fwd(size_t instr_off, int type, size_t *fwd_off, const char *shared_label) {
    if (fwd_off) {
        *fwd_off = instr_off;
    } else if (shared_label) {
        asm_fixup_add(cg_sec, instr_off, shared_label, type);
    } else {
        int c = ++rcc_label_count;
        const char *lbl = format(".L.gcbi.%d", c);
        asm_fixup_add(cg_sec, instr_off, lbl, type);
        cg_def_label(lbl);
    }
}

// Emit a branch to the false-target when the flonum value held as a
// double bit pattern in GP register `r` compares equal to 0.0 (i.e. the
// C condition is FALSE). IEEE semantics: -0.0 compares equal to +0.0
// (condition false) even though its bit pattern is nonzero; NaN is
// unordered, hence truthy (never branches). A plain bitwise `cmp $0, r`
// would misclassify -0.0 as truthy.
// Target resolution mirrors cg_add_fwd: fwd_off (direct patch) else
// shared_label (fixup) else auto-label defined in place.
static void gen_flonum_branch_if_zero(VReg r, size_t *fwd_off, const char *shared_label) {
#ifdef ARCH_ARM64
    asm_fmov_i2f(cg_sec, 0, r, 1); // fmov d0, x{r}
    asm_fmov_d1_xzr(cg_sec); // fmov d1, xzr
    asm_fcmp(cg_sec, 1); // fcmp d0, d1
    int c = ++rcc_label_count;
    const char *skip = format(".L.fz.skip.%d", c);
    {
        size_t o = asm_jcc_label(cg_sec, ARM64_VS); // b.vs skip (NaN truthy)
        asm_fixup_add(cg_sec, o, skip, 1);
    }
    {
        size_t o = asm_jcc_label(cg_sec, ARM64_EQ); // b.eq target (zero incl. -0.0)
        cg_add_fwd(o, 1, fwd_off, shared_label);
    }
    cg_def_label(skip);
#else
    asm_movq_r_xmm(cg_sec, X86_XMM0, r); // movq %s, %xmm0
    x86_pxor(cg_sec, X86_XMM1, X86_XMM1); // xorpd %xmm1, %xmm1
    asm_ucomisd(cg_sec); // ucomisd %xmm1, %xmm0
    int c = ++rcc_label_count;
    const char *skip = format(".L.fz.skip.%d", c);
    {
        size_t o = asm_jcc_label(cg_sec, X86_P); // jp skip (NaN truthy)
        asm_fixup_add(cg_sec, o, skip, 1);
    }
    {
        size_t o = asm_jcc_label(cg_sec, X86_E); // je target (zero incl. -0.0)
        cg_add_fwd(o, 1, fwd_off, shared_label);
    }
    cg_def_label(skip);
#endif
}

// Branch to label when the flonum value in GP register `r` is truthy
// (compares unequal to 0.0, or is NaN). Inverse of
// gen_flonum_branch_if_zero with a plain-label target.
static void gen_flonum_branch_if_nonzero(VReg r, const char *label) {
#ifdef ARCH_ARM64
    asm_fmov_i2f(cg_sec, 0, r, 1); // fmov d0, x{r}
    asm_fmov_d1_xzr(cg_sec); // fmov d1, xzr
    asm_fcmp(cg_sec, 1); // fcmp d0, d1
    {
        size_t o = asm_jcc_label(cg_sec, ARM64_VS); // b.vs label (NaN truthy)
        asm_fixup_add(cg_sec, o, label, 1);
    }
    {
        size_t o = asm_jcc_label(cg_sec, ARM64_NE); // b.ne label (ordered unequal)
        asm_fixup_add(cg_sec, o, label, 1);
    }
#else
    asm_movq_r_xmm(cg_sec, X86_XMM0, r); // movq %s, %xmm0
    x86_pxor(cg_sec, X86_XMM1, X86_XMM1); // xorpd %xmm1, %xmm1
    asm_ucomisd(cg_sec); // ucomisd %xmm1, %xmm0
    {
        size_t o = asm_jcc_label(cg_sec, X86_P); // jp label (NaN truthy)
        asm_fixup_add(cg_sec, o, label, 1);
    }
    {
        size_t o = asm_jcc_label(cg_sec, X86_NE); // jne label (ordered unequal)
        asm_fixup_add(cg_sec, o, label, 1);
    }
#endif
}

// Normalize the flonum value in GP register `r` (double bit pattern) to a
// C truthiness 0/1: 1 if the value compares unequal to 0.0 (or is NaN),
// 0 if it compares equal to 0.0 (including -0.0). Mirrors gen_cast_reg's
// _Bool conversion path.
static void gen_flonum_truthiness(VReg r) {
#ifdef ARCH_ARM64
    asm_fmov_i2f(cg_sec, 0, r, 1); // fmov d0, x{r}
    asm_fmov_d1_xzr(cg_sec); // fmov d1, xzr
    asm_fcmp(cg_sec, 1); // fcmp d0, d1
    asm_cset(cg_sec, r, ARM64_NE); // cset r, ne (ordered unequal)
    // fcmp is unordered (NaN): Z=1 -> cset ne gave 0, but NaN is truthy.
    // csinc r, r, xzr, vc: if not-unordered keep r, else r = xzr+1 = 1.
    arm64_csinc(cg_sec, 0, REG(r), REG(r), ARM64_XZR, ARM64_VC);
#else
    asm_movq_r_xmm(cg_sec, X86_XMM0, r); // movq r, %xmm0
    x86_pxor(cg_sec, X86_XMM1, X86_XMM1); // xorpd %xmm1, %xmm1
    asm_ucomisd(cg_sec); // ucomisd %xmm1, %xmm0
    asm_setcc(cg_sec, X86_RAX, X86_NE); // setne %al (ordered, unequal)
    asm_setcc(cg_sec, X86_RCX, X86_P); // setp %cl (unordered: NaN)
    x86_or_rr(cg_sec, 1, X86_RAX, X86_RCX); // orb %cl, %al
    asm_movzx_phys(cg_sec, r, X86_RAX, 4, 1); // movzbl %al, r
#endif
}

// Emit a conditional branch to label when the condition is FALSE (inverted sense).
// If shared_label != NULL: all branches fix up to this label (via asm_fixup_add).
// If shared_label == NULL:
//   fwd_off != NULL → store single branch offset for direct patching via asm_patch_branch.
//   fwd_off == NULL → auto-generate label and resolve via cg_def_label (hash table fallback).
static void gen_cond_branch_inv(Node *cond, size_t *fwd_off, const char *shared_label) {
    if (cond->kind == ND_LOGAND) {
        // Two recursive calls share the same label.
        // Only call cg_def_label when we own the label;
        // if the caller provided shared_label, defer to the caller.
        bool own_label = (shared_label == NULL);
        if (own_label) {
            int c = ++rcc_label_count;
            shared_label = format(".L.gcbi.%d", c);
        }
        gen_cond_branch_inv(cond->lhs, NULL, shared_label);
        gen_cond_branch_inv(cond->rhs, NULL, shared_label);
        if (own_label)
            cg_def_label(shared_label);
        return;
    }
    if (cond->kind == ND_LOGOR) {
        VReg lhs = gen(cond->lhs);
        if (cond->lhs->ty && is_flonum(cond->lhs->ty)) {
            // Truthy (nonzero or NaN) lhs skips the rhs check. A bitwise
            // GP-register test would misclassify -0.0 (nonzero bit
            // pattern, value == 0.0) as truthy.
            int c2 = ++rcc_label_count;
            const char *skip = format(".L.glor.fskip.%d", c2);
            gen_flonum_branch_if_nonzero(lhs, skip);
            free_reg(lhs);
            // If lhs was false, check rhs; if rhs is also false, branch to target
            gen_cond_branch_inv(cond->rhs, fwd_off, shared_label);
            cg_def_label(skip);
            return;
        }
        int lhs_sz = cond->lhs->ty->size;
        asm_cmp_zero(cg_sec, lhs, lhs_sz); // cmp $0, rlhs
        free_reg(lhs);
#ifdef ARCH_ARM64
        size_t o = asm_jcc_label(cg_sec, ARM64_NE); // b.ne skip
#else
        size_t o = asm_jcc_label(cg_sec, X86_NE); // jne skip
#endif
        // If lhs was false, check rhs; if rhs is also false, branch to target
        gen_cond_branch_inv(cond->rhs, fwd_off, shared_label);
        asm_patch_branch(cg_sec, o, cg_sec->len, 1); // patch skip to here
        return;
    }
    if (cond->kind == ND_EQ || cond->kind == ND_NE || cond->kind == ND_LT || cond->kind == ND_LE) {
        if (cond->lhs->ty && cond->lhs->ty->kind == TY_INT128) {
            // __int128 comparison: gen_int128 returns a boolean VReg (0 or 1)
            VReg r = gen_int128(cond);
            asm_cmp_zero(cg_sec, r, 4); // cmpl $0, rr
            free_reg(r);
#ifdef ARCH_ARM64
            size_t o = asm_jcc_label(cg_sec, ARM64_EQ); // b.eq target
#else
            size_t o = asm_jcc_label(cg_sec, X86_E); // je target
#endif
            cg_add_fwd(o, 1, fwd_off, shared_label);
            return;
        }
        if ((cond->lhs->ty && is_wide_bitint(cond->lhs->ty)) ||
            (cond->rhs && cond->rhs->ty && is_wide_bitint(cond->rhs->ty))) {
            // wide _BitInt comparison: gen_bitint returns a boolean VReg (0 or 1)
            VReg r = gen_bitint(cond);
            asm_cmp_zero(cg_sec, r, 4); // cmpl $0, rr
            free_reg(r);
#ifdef ARCH_ARM64
            size_t o = asm_jcc_label(cg_sec, ARM64_EQ); // b.eq target
#else
            size_t o = asm_jcc_label(cg_sec, X86_E); // je target
#endif
            cg_add_fwd(o, 1, fwd_off, shared_label);
            return;
        }
        if ((cond->lhs->ty && is_decimal(cond->lhs->ty)) ||
            (cond->rhs && cond->rhs->ty && is_decimal(cond->rhs->ty))) {
            // decimal comparison: gen_decimal returns a boolean VReg (0 or 1)
            VReg r = gen_decimal(cond);
            asm_cmp_zero(cg_sec, r, 4); // cmpl $0, rr
            free_reg(r);
#ifdef ARCH_ARM64
            size_t o = asm_jcc_label(cg_sec, ARM64_EQ); // b.eq target
#else
            size_t o = asm_jcc_label(cg_sec, X86_E); // je target
#endif
            cg_add_fwd(o, 1, fwd_off, shared_label);
            return;
        }
        if (cond->lhs->ty && is_complex(cond->lhs->ty)) {
            // Complex EQ/NE: compare both parts, branch if not equal
            int r = gen(cond); // use the expression handler which now handles complex
#ifdef ARCH_ARM64
            asm_cmp_zero(cg_sec, r, 4);
            {
                size_t o = asm_jcc_label(cg_sec, ARM64_EQ); // b.eq target
                cg_add_fwd(o, 1, fwd_off, shared_label);
            }
#else
            asm_cmp_zero(cg_sec, r, 4); // testl r, r
            {
                size_t o = asm_jcc_label(cg_sec, X86_E); // je target
                cg_add_fwd(o, 1, fwd_off, shared_label);
            }
#endif
            return;
        }
        if (is_flonum(cond->lhs->ty)) {
            VReg r_lhs = gen(cond->lhs);
            VReg r_rhs = gen(cond->rhs);
#ifdef ARCH_ARM64
            asm_fmov_i2f(cg_sec, 0, r_lhs, 1); // fmov d0, x{r_lhs}
            asm_fmov_i2f(cg_sec, 1, r_rhs, 1); // fmov d1, x{r_rhs}
            asm_fcmp(cg_sec, 1); // fcmp d0, d1
            if (cond->kind == ND_EQ) {
                // 2 branches to same target
                const char *lbl = shared_label;
                bool own_lbl = false;
                if (!lbl) {
                    int c = ++rcc_label_count;
                    lbl = format(".L.gcbi.%d", c);
                    own_lbl = true;
                }
                {
                    size_t o = asm_jcc_label(cg_sec, ARM64_NE); // b.ne label
                    asm_fixup_add(cg_sec, o, lbl, 1);
                }
                {
                    size_t o = asm_jcc_label(cg_sec, ARM64_VS); // b.vs label
                    asm_fixup_add(cg_sec, o, lbl, 1);
                }
                if (own_lbl) cg_def_label(lbl);
            } else if (cond->kind == ND_NE) {
                int c = ++rcc_label_count;
                {
                    size_t _cj = asm_jcc_label(cg_sec, ARM64_VS);
                    asm_fixup_add(cg_sec, _cj, format(".L.fc.skip.%d", c), 1);
                }
                {
                    size_t o = asm_jcc_label(cg_sec, ARM64_EQ); // b.eq target
                    cg_add_fwd(o, 1, fwd_off, shared_label);
                }
                cg_def_label(format(".L.fc.skip.%d", c));
            } else if (cond->kind == ND_LT) {
                size_t o = asm_jcc_label(cg_sec, ARM64_PL); // b.pl target
                cg_add_fwd(o, 1, fwd_off, shared_label);
            } else {
                size_t o = asm_jcc_label(cg_sec, ARM64_HI); // b.hi target
                cg_add_fwd(o, 1, fwd_off, shared_label);
            }
#else
            asm_movq_r_xmm(cg_sec, X86_XMM0, r_lhs); // movq X86_XMM0, r_lhs
            asm_movq_r_xmm(cg_sec, X86_XMM1, r_rhs); // movq X86_XMM1, r_rhs
            asm_ucomisd(cg_sec); // ucomisd
            if (cond->kind == ND_EQ) {
                // 2 branches to same target
                const char *lbl = shared_label;
                bool own_lbl = false;
                if (!lbl) {
                    int c = ++rcc_label_count;
                    lbl = format(".L.gcbi.%d", c);
                    own_lbl = true;
                }
                {
                    size_t o = asm_jcc_label(cg_sec, X86_NE); // jne label
                    asm_fixup_add(cg_sec, o, lbl, 1);
                }
                {
                    size_t o = asm_jcc_label(cg_sec, X86_P); // jp label
                    asm_fixup_add(cg_sec, o, lbl, 1);
                }
                if (own_lbl) cg_def_label(lbl);
            } else if (cond->kind == ND_NE) {
                int c = ++rcc_label_count;
                {
                    size_t o = asm_jcc_label(cg_sec, X86_P); // jp .L.fc.skip.c
                    asm_fixup_add(cg_sec, o, format(".L.fc.skip.%d", c), 1);
                }
                {
                    size_t o = asm_jcc_label(cg_sec, X86_E); // je target
                    cg_add_fwd(o, 1, fwd_off, shared_label);
                }
                cg_def_label(format(".L.fc.skip.%d", c));
            } else if (cond->kind == ND_LT) {
                const char *lbl = shared_label;
                bool own_lbl = false;
                if (!lbl) {
                    int c = ++rcc_label_count;
                    lbl = format(".L.gcbi.%d", c);
                    own_lbl = true;
                }
                {
                    size_t o = asm_jcc_label(cg_sec, X86_AE); // jae label
                    asm_fixup_add(cg_sec, o, lbl, 1);
                }
                {
                    size_t o = asm_jcc_label(cg_sec, X86_P); // jp label
                    asm_fixup_add(cg_sec, o, lbl, 1);
                }
                if (own_lbl) cg_def_label(lbl);
            } else {
                const char *lbl = shared_label;
                bool own_lbl = false;
                if (!lbl) {
                    int c = ++rcc_label_count;
                    lbl = format(".L.gcbi.%d", c);
                    own_lbl = true;
                }
                {
                    size_t o = asm_jcc_label(cg_sec, X86_A); // ja label
                    asm_fixup_add(cg_sec, o, lbl, 1);
                }
                {
                    size_t o = asm_jcc_label(cg_sec, X86_P); // jp label
                    asm_fixup_add(cg_sec, o, lbl, 1);
                }
                if (own_lbl) cg_def_label(lbl);
            }
#endif
            free_reg(r_rhs);
            free_reg(r_lhs);
            return;
        }
        // Commute a constant-lhs equality so the constant uses the immediate
        // compare path (see the matching comment in the value path); avoids a
        // self-compare when the rhs reuses the spilled lhs register.
        if ((cond->kind == ND_EQ || cond->kind == ND_NE) && cond->lhs->kind == ND_NUM && cond->rhs->kind != ND_NUM) {
            Node *swap_tmp = cond->lhs;
            cond->lhs = cond->rhs;
            cond->rhs = swap_tmp;
        }
        VReg r_lhs = gen(cond->lhs);
        int sz = op_size(cond->lhs->ty);
        if (sz < op_size(cond->rhs->ty))
            sz = op_size(cond->rhs->ty);
        // Fold small integer constants, including -C from ND_NEG(ND_NUM(C))
        int64_t rhs_val;
        bool rhs_is_imm = false;
        if (cond->rhs->kind == ND_NUM) {
            rhs_val = cond->rhs->val;
            rhs_is_imm = true;
        } else if (cond->rhs->kind == ND_NEG && cond->rhs->lhs &&
                   cond->rhs->lhs->kind == ND_NUM) {
            rhs_val = -(cond->rhs->lhs->val);
            rhs_is_imm = true;
        }
        if (rhs_is_imm && rhs_val == (int32_t)rhs_val) {
#ifdef ARCH_ARM64
            int32_t imm = (int32_t)rhs_val;
            if (imm >= 0 && imm <= 4095) {
                asm_cmp_imm(cg_sec, r_lhs, sz, imm); // cmp $imm, rr_lhs
            } else if (imm < 0 && imm >= -4095) {
                asm_cmn_vreg_imm(cg_sec, r_lhs, sz, -imm); // cmn x{r_lhs}, #(-imm)
            } else {
                emit_mov_imm64(ARM64_X16, (uint64_t)(int64_t)imm);
                asm_cmp_reg_phy(cg_sec, r_lhs, ARM64_X16, sz); // cmp r16, rr_lhs
            }
#else
            asm_cmp_imm(cg_sec, r_lhs, sz, (int32_t)rhs_val); // cmp $imm, rr_lhs
#endif
        } else {
            VReg r_rhs = gen(cond->rhs);
#ifndef ARCH_ARM64
            // When r_lhs was spilled during gen(rhs) and rhs reused the same
            // physical register, a plain reg-reg cmp would self-compare.
            // Compare against the spilled lhs slot instead (flags: lhs - rhs).
            if (r_lhs == r_rhs && (spilled_regs & (1 << r_lhs))) {
                asm_cmp_spill_reg(cg_sec, r_lhs, sz, spill_offset(r_lhs));
                spilled_regs &= ~(1 << r_lhs);
            } else
#endif
                asm_cmp_reg_reg(cg_sec, r_lhs, r_rhs, sz); // cmp rr_rhs, rr_lhs
            free_reg(r_rhs);
        }
        free_reg(r_lhs);

#ifdef ARCH_ARM64
        if (cond->kind == ND_EQ) {
            size_t o = asm_jcc_label(cg_sec, ARM64_NE); // b.ne target
            cg_add_fwd(o, 1, fwd_off, shared_label);
        } else if (cond->kind == ND_NE) {
            size_t o = asm_jcc_label(cg_sec, ARM64_EQ); // b.eq target
            cg_add_fwd(o, 1, fwd_off, shared_label);
        } else if (cond->kind == ND_LT) {
            size_t o = asm_jcc_label(cg_sec, use_unsigned_cmp(cond) ? ARM64_HS : ARM64_GE); // b.hs/b.ge target
            cg_add_fwd(o, 1, fwd_off, shared_label);
        } else {
            size_t o = asm_jcc_label(cg_sec, use_unsigned_cmp(cond) ? ARM64_HI : ARM64_GT); // b.hi/b.gt target
            cg_add_fwd(o, 1, fwd_off, shared_label);
        }
#else
        if (cond->kind == ND_EQ) {
            size_t o = asm_jcc_label(cg_sec, X86_NE); // jne target
            cg_add_fwd(o, 1, fwd_off, shared_label);
        } else if (cond->kind == ND_NE) {
            size_t o = asm_jcc_label(cg_sec, X86_E); // je target
            cg_add_fwd(o, 1, fwd_off, shared_label);
        } else if (cond->kind == ND_LT) {
            size_t o = asm_jcc_label(cg_sec, use_unsigned_cmp(cond) ? X86_AE : X86_GE); // jae/jge target
            cg_add_fwd(o, 1, fwd_off, shared_label);
        } else {
            size_t o = asm_jcc_label(cg_sec, use_unsigned_cmp(cond) ? X86_A : X86_G); // ja/jg target
            cg_add_fwd(o, 1, fwd_off, shared_label);
        }
#endif
        return;
    }

    // Complex types: gen() returns address, need to load and OR components.
    // Exception: ND_ASSIGN with complex funcall RHS already returns a
    // boolean truthiness value from its own inline check.
    if (cond->ty && is_complex(cond->ty) && cond->kind != ND_ASSIGN) {
        VReg addr = gen(cond);
        int base_sz = cond->ty->base ? cond->ty->base->size : 8;
#ifdef ARCH_ARM64
        VReg rt = alloc_reg();
        asm_ldr_reg_off(cg_sec, rt, addr, 8, 0);
        if (cond->ty->size > base_sz) {
            VReg rt2 = alloc_reg();
            asm_ldr_reg_off(cg_sec, rt2, addr, 8, base_sz);
            arm64_orr_reg(cg_sec, 1, REG(rt), REG(rt), REG(rt2), ARM64_LSL, 0);
            free_reg(rt2);
        }
        asm_cmp_zero(cg_sec, rt, 8);
        size_t cj = asm_jcc_label(cg_sec, ARM64_EQ);
        cg_add_fwd(cj, 1, fwd_off, shared_label);
        free_reg(rt);
        free_reg(addr);
#else
        VReg rt = alloc_reg();
        x86_mov_rm(cg_sec, 8, REG(rt), x86_mem(REG(addr), 0));
        if (cond->ty->size > 8)
            x86_or_rm(cg_sec, 8, REG(rt), x86_mem(REG(addr), base_sz));
        asm_cmp_zero(cg_sec, rt, 8);
        size_t cj = asm_jcc_label(cg_sec, X86_E);
        cg_add_fwd(cj, 1, fwd_off, shared_label);
        free_reg(rt);
        free_reg(addr);
#endif
        return;
    }

    VReg r = gen(cond);
    if (cond->ty && is_flonum(cond->ty)) {
        // IEEE truthiness: -0.0 == 0.0 (false), NaN truthy. A bitwise
        // `cmp $0` on the double bit pattern would treat -0.0 as truthy.
        gen_flonum_branch_if_zero(r, fwd_off, shared_label);
        free_reg(r);
        return;
    }
    if (cond->ty && is_wide_bitint(cond->ty)) {
        // wide _BitInt truthiness: r holds the slot ADDRESS, not the value.
        // Call the to_bool helper and branch on its result.
        emit_bitint_bits_arg(cond->ty->bitint_width);
        emit_bitint_addr_arg(r, 0);
        free_reg(r);
        emit_bitint_call((char *)"rcc_bitint_to_bool");
        VReg br = alloc_reg();
        asm_mov_retval(cg_sec, br, 4); // movl %eax, br
        asm_cmp_zero(cg_sec, br, 4); // cmpl $0, br
        free_reg(br);
#ifdef ARCH_ARM64
        size_t bo = asm_jcc_label(cg_sec, ARM64_EQ); // b.eq target (false)
#else
        size_t bo = asm_jcc_label(cg_sec, X86_E); // je target (false)
#endif
        cg_add_fwd(bo, 1, fwd_off, shared_label);
        return;
    }
    int sz = cond->ty->size > 8 ? 8 : (cond->ty->size > 0 ? cond->ty->size : 4);
#ifdef ARCH_ARM64
    asm_cmp_zero(cg_sec, r, sz);
    free_reg(r);
    size_t cond_off = asm_jcc_label(cg_sec, ARM64_EQ);
    cg_add_fwd(cond_off, 1, fwd_off, shared_label);
#else
    asm_cmp_zero(cg_sec, r, sz);
    free_reg(r);
    size_t cond_off = asm_jcc_label(cg_sec, X86_E);
    cg_add_fwd(cond_off, 1, fwd_off, shared_label);
#endif
}

// ===== __int128 support =====
// Convention: all TY_INT128 values live in a 16-byte stack slot.
// gen_int128(node) returns a GP register holding the slot address.
// lo word is at [addr+0], hi word at [addr+8].

// Allocate a fresh 16-byte-aligned int128 scratch slot in the struct-ret area.
int alloc_int128_slot(void) {
    return alloc_wide_slot(16);
}

// Allocate a fresh aligned scratch slot of `size` bytes (>= 8, rounded to
// 16) in the struct-ret area. Used for __int128 (16), _BitInt(N>64)
// (8*ceil(N/64)), and 32-byte vector operands.
int alloc_wide_slot(int size) {
    if (size < 8) size = 8;
    size = (size + 15) & ~15;
    // The hidden struct-return buffer var and 32-byte vector params belong
    // to the local pool but can extend past fn->stack_size (the frame sizes
    // those slots via struct_ret_total only). Never hand out struct-ret-buf
    // slots that overlap the vars, or the result store clobbers e.g. the
    // saved struct-return pointer (a 16-byte slot next to a 32-byte vector
    // param pair already did before this guard).
    int deep = current_fn_def ? current_fn_def->stack_size : current_fn_stack_size;
    for (LVar *var = current_fn_def ? current_fn_def->locals : NULL; var; var = var->next) {
        int end = var->offset + (var->ty ? (int)var->ty->size : 8);
        if (end > deep)
            deep = end;
    }
    if (deep > current_fn_stack_size) {
        int need = deep - current_fn_stack_size + size;
        while (fn_struct_ret_off < need) {
            fn_struct_ret_off = (fn_struct_ret_off + 15) & ~15;
            fn_struct_ret_off += size;
        }
    }
    fn_struct_ret_off = (fn_struct_ret_off + 15) & ~15;
    fn_struct_ret_off += size;
    if (fn_struct_ret_off > fn_struct_ret_total)
        fn_struct_ret_total = fn_struct_ret_off;
    return current_fn_stack_size + fn_struct_ret_off;
}

// Return a GP register holding the address of a fresh 16-byte scratch slot.
VReg alloc_int128_addr(void) {
    return alloc_wide_addr(16);
}

// Return a GP register holding the address of a fresh `size`-byte slot.
VReg alloc_wide_addr(int size) {
    int slot = alloc_wide_slot(size);
    VReg r = alloc_reg();
#ifdef ARCH_ARM64
    if (slot <= 4095) {
        asm_sub_reg_fp_imm(cg_sec, r, slot); // sub r, x29, #slot
    } else {
        int v = slot;
        asm_mov_imm(cg_sec, r, 8, v & 0xffff);
        v >>= 16;
        int s = 16;
        while (v) {
            asm_movk(cg_sec, REG(r), 1, (uint16_t)(v & 0xffff), s);
            v >>= 16;
            s += 16;
        }
        asm_sub_reg_fp_reg(cg_sec, r, r, 8); // sub ta, x29, ta
    }
#else
    asm_lea_rbp_reg(cg_sec, r, 8, slot);
#endif
    return r;
}

// Widen val (64-bit GP register) to a fresh 128-bit slot; frees val.
static VReg widen_to_int128(VReg val, bool is_unsigned) {
    VReg addr = alloc_int128_addr();
#ifdef ARCH_ARM64
    asm_str_reg_off(cg_sec, val, addr, 8, 0);
    if (is_unsigned) {
        asm_str_xzr_uoff(cg_sec, REG(addr), 1); // str xzr, [%s, #8]
    } else {
        asm_asr_x16_src_63(cg_sec, val); // asr x16, %s, #63
        asm_str_x16_uoff(cg_sec, addr, 1); // str x16, [%s, #8]
    }
#else
    asm_mov_reg_to_retval(cg_sec, val, 8); // movq val, %rax
    asm_mov_rax_mem(cg_sec, addr); // movq %rax, (addr)
    if (is_unsigned) {
        asm_movq_zero_mem8(cg_sec, addr); // movq $0, 8(addr)
    } else {
        asm_mov_reg_to_retval(cg_sec, val, 8); // movq val, %rax
        asm_shift_rax_imm(cg_sec, false, 63); // sarq $63, %rax
        asm_movq_rax_mem8(cg_sec, addr); // movq %rax, 8(addr)
    }
#endif
    free_reg(val);
    return addr;
}

// Ensure operand is available as an int128 slot address.
static VReg gen_to_int128(Node *operand) {
    if (operand->ty && operand->ty->kind == TY_INT128)
        return gen_int128(operand);
    int val = gen(operand);
    Type *ty = operand->ty ? operand->ty : ty_llong;
    // Extend to 64-bit first if narrower
    if (ty->size < 8) {
        if (ty->is_unsigned)
            zero_extend_to(val, ty->size, 8);
        else
            sign_extend_to(val, ty->size, 8);
    }
    return widen_to_int128(val, ty->is_unsigned);
}


static VReg gen_int128(Node *node) {
    if (!node) return R_NONE;
    if (opt_g) emit_loc(node);

    // codeql[cpp/long-switch]: central AST-node-kind dispatch; splitting cases into helpers is a large, purely-cosmetic refactor of core compiler internals, not attempted here.
    switch (node->kind) {

    case ND_NUM: {
        long long lo = node->val;
        long long hi = (lo < 0 && !(node->ty && node->ty->is_unsigned)) ? -1LL : 0LL;
        int addr = alloc_int128_addr();
#ifdef ARCH_ARM64
        int t = alloc_reg();
        if (lo >= -65536 && lo <= 65535) {
            asm_mov_imm(cg_sec, t, 8, lo); // mov %s, #%lld
        } else {
            uint64_t v = (uint64_t)lo;
            asm_mov_imm(cg_sec, t, 8, v & 0xffff);
            v >>= 16;
            int s = 16;
            while (v) {
                asm_movk(cg_sec, t, 1, (uint16_t)(v & 0xffff), s);
                v >>= 16;
                s += 16;
            }
        }
        asm_str_reg_off(cg_sec, t, addr, 8, 0);
        if (hi == 0) {
            asm_str_xzr_uoff(cg_sec, REG(addr), 1); // str xzr, [%s, #8]
        } else {
            asm_mov_imm(cg_sec, t, 8, -1); // mov %s, #-1
            asm_str_reg_off(cg_sec, t, addr, 8, 8); // store hi to [%s, #8]
        }
        free_reg(t);
#else
        if (lo == (int32_t)lo) {
            X86Mem m = {REG(addr), X86_NOREG, 1, 0};
            x86_mov_mi(cg_sec, 8, m, lo); // movq $%lld, (%s)
        } else {
            asm_movabs_phy(cg_sec, X86_RAX, lo); // movabsq $%lld, %%rax
            asm_mov_rax_mem(cg_sec, addr); // movq %%rax, (%s)
        }
        if (hi == 0) {
            asm_movq_zero_mem8(cg_sec, addr); // movq $0, 8(%s)
        } else {
            asm_movabs_phy(cg_sec, X86_RAX, hi); // movabsq $%lld, %%rax
            asm_movq_rax_mem8(cg_sec, addr); // movq %%rax, 8(%s)
        }
#endif
        return addr;
    }

    case ND_LVAR:
    case ND_MEMBER:
        return gen_addr(node);

    case ND_DEREF:
        // *ptr where ptr → 128-bit value: the pointer value IS the address
        return gen(node->lhs);

    case ND_ADDR:
        // & of something: gen_addr gives the address (result is pointer, not int128)
        return gen_addr(node->lhs);

    case ND_COMMA:
    case ND_CHAIN: {
        int r = gen(node->lhs);
        if (r >= 0) free_reg(r);
        return gen_int128(node->rhs);
    }

    case ND_ASSIGN: {
#ifdef ARCH_ARM64
        int rhs_a = gen_to_int128(node->rhs);
        int lhs_a = gen_addr(node->lhs);
        int t1 = alloc_reg(), t2 = alloc_reg();
        asm_ldr_reg_off(cg_sec, t1, rhs_a, 8, 0);
        asm_ldr_reg_off(cg_sec, t2, rhs_a, 8, 8); // ldr %s, [%s, #8]
        asm_str_reg_off(cg_sec, t1, lhs_a, 8, 0);
        asm_str_reg_off(cg_sec, t2, lhs_a, 8, 8); // str %s, [%s, #8]
        free_reg(t1);
        free_reg(t2);
#else
        // Compute rhs first to avoid volatile-register clobber on Windows:
        // __udivti3 etc. use r10 internally, which may hold lhs_a if evaluated first.
        int rhs_a = gen_to_int128(node->rhs);
        VReg lhs_a = gen_addr(node->lhs);
        x86_mov_rm(cg_sec, 8, X86_RAX, x86_mem(REG(rhs_a), 0)); // movq (%rs), %rax
        x86_mov_rm(cg_sec, 8, X86_RDX, x86_mem(REG(rhs_a), 8)); // movq 8(%rs), %rdx
        x86_mov_mr(cg_sec, 8, x86_mem(REG(lhs_a), 0), X86_RAX); // movq %rax, (%rl)
        x86_mov_mr(cg_sec, 8, x86_mem(REG(lhs_a), 8), X86_RDX); // movq %rdx, 8(%rl)
#endif
        free_reg(rhs_a);
        return lhs_a;
    }

    case ND_CAST: {
        Type *from = node->lhs->ty;
        Type *to = node->ty;
        // from int128 to int128 (e.g. signed ↔ unsigned, or same)
        if (from && from->kind == TY_INT128)
            return gen_int128(node->lhs);
        // from float to int128
        if (from && is_flonum(from)) {
            int r = gen(node->lhs);
            int addr = alloc_int128_addr();
#ifdef ARCH_ARM64
            asm_fmov_d0_x(cg_sec, r); // fmov d0, x{r}
            free_reg(r);
            if (to->is_unsigned)
                asm_fcvtzu_x16(cg_sec); // fcvtzu x16, d0
            else
                asm_fcvtzs_x16(cg_sec); // fcvtzs x16, d0
            asm_str_x16_uoff(cg_sec, addr, 0); // str x16, [%s]
            if (to->is_unsigned) {
                asm_str_xzr_uoff(cg_sec, REG(addr), 1); // str xzr, [%s, #8]
            } else {
                asm_asr_x16_63(cg_sec); // asr x16, x16, #63
                asm_str_x16_uoff(cg_sec, addr, 1); // str x16, [%s, #8]
            }
#else
            (void)0 /* TODO: movq to/from xmm */;
            free_reg(r);
            if (to->is_unsigned) {
                // Unsigned float→int128: handle values >= 2^63 via the 2-step trick
                int c = ++rcc_label_count;
                asm_movabs_phy(cg_sec, X86_RAX, 0x43e0000000000000LL); // movq $0x43e0000000000000, %%rax (2^63 as double)
                x86_movq_r_xmm(cg_sec, X86_XMM1, X86_RAX); // movq %%rax, %%xmm1
                x86_comisd(cg_sec, X86_XMM0, X86_XMM1); // comisd %%xmm1, %%xmm0
                emit_jcc_fixup(cg_sec, X86_B, format(".L.f2i128u.%d", c)); // jb .L.f2i128u.%d
                x86_subsd(cg_sec, X86_XMM0, X86_XMM1); // subsd %%xmm1, %%xmm0
                x86_cvttsd2si(cg_sec, 8, X86_RAX, X86_XMM0); // cvttsd2si %%xmm0, %%rax
                asm_movabs_phy(cg_sec, X86_RDX, 0x8000000000000000LL); // movq $0x8000000000000000, %%rdx
                x86_xor_rr(cg_sec, 8, X86_RAX, X86_RDX); // xorq %%rdx, %%rax (add 2^63 back)
                emit_jmp_fixup(cg_sec, format(".L.f2i128u_lo.%d", c)); // jmp .L.f2i128u_lo.%d
                cg_def_label(format(".L.f2i128u.%d", c)); // .L.xxx.c
                x86_cvttsd2si(cg_sec, 8, X86_RAX, X86_XMM0); // cvttsd2si %%xmm0, %%rax
                cg_def_label(format(".L.f2i128u_lo.%d", c)); // .L.xxx.c
                asm_mov_rax_mem(cg_sec, addr); // movq %%rax, (%s)
                asm_movq_zero_mem8(cg_sec, addr); // movq $0, 8(%s)
            } else {
                x86_cvttsd2si(cg_sec, 8, X86_RAX, X86_XMM0); // cvttsd2si %%xmm0, %%rax
                asm_mov_rax_mem(cg_sec, addr); // movq %%rax, (%s)
                asm_shift_rax_imm(cg_sec, false, 63); // sarq $63, %%rax
                asm_movq_rax_mem8(cg_sec, addr); // movq %%rax, 8(%s)
            }
#endif
            return addr;
        }
        // Complex-to-complex cast: delegate to gen_addr which handles conversion
        if (from && is_complex(from) && is_complex(to))
            return gen_addr(node);
        int val = gen(node->lhs);
        if (from && from->size < 8) {
            if (from->is_unsigned)
                zero_extend_to(val, from->size, 8);
            else
                sign_extend_to(val, from->size, 8);
        }
        return widen_to_int128(val, from ? from->is_unsigned : false);
    }

    case ND_NEG: {
        int src = gen_to_int128(node->lhs);
        int dst = alloc_int128_addr();
#ifdef ARCH_ARM64
        int t = alloc_reg();
        asm_ldr_reg_off(cg_sec, t, src, 8, 0);
        asm_negs(cg_sec, t, t); // negs %s, %s
        asm_str_reg_off(cg_sec, t, dst, 8, 0);
        asm_ldr_reg_off(cg_sec, t, src, 8, 8);
        asm_ngc(cg_sec, t, t); // ngc %s, %s
        asm_str_reg_off(cg_sec, t, dst, 8, 8);
        free_reg(t);
#else
        asm_mov_mem_rax(cg_sec, src);
        asm_mov_mem8_rdx(cg_sec, src);
        asm_mov_rax_mem(cg_sec, dst);
        asm_mov_rdx_mem8(cg_sec, dst);
        asm_negq_mem(cg_sec, dst); // negq (%s)
        asm_adcq_mem8_0(cg_sec, dst); // adcq $0, 8(%s)
        asm_negq_mem8(cg_sec, dst); // negq 8(%s)
#endif
        free_reg(src);
        return dst;
    }

    case ND_BITNOT: {
        int src = gen_to_int128(node->lhs);
        int dst = alloc_int128_addr();
#ifdef ARCH_ARM64
        int t = alloc_reg();
        asm_ldr_reg_off(cg_sec, t, src, 8, 0);
        asm_not(cg_sec, t, 8);
        asm_str_reg_off(cg_sec, t, dst, 8, 0);
        asm_ldr_reg_off(cg_sec, t, src, 8, 8);
        asm_not(cg_sec, t, 8);
        asm_str_reg_off(cg_sec, t, dst, 8, 8);
        free_reg(t);
#else
        asm_mov_mem_rax(cg_sec, src);
        asm_mov_mem8_rdx(cg_sec, src);
        asm_notq_rax(cg_sec);
        asm_notq_rdx(cg_sec);
        asm_mov_rax_mem(cg_sec, dst);
        asm_mov_rdx_mem8(cg_sec, dst);
#endif
        free_reg(src);
        return dst;
    }

    case ND_ADD: {
        int lhs = gen_to_int128(node->lhs);
        int rhs = gen_to_int128(node->rhs);
        int dst = alloc_int128_addr();
#ifdef ARCH_ARM64
        int t1 = alloc_reg(), t2 = alloc_reg(), t3 = alloc_reg(), t4 = alloc_reg();
        asm_ldr_reg_off(cg_sec, t1, lhs, 8, 0);
        asm_ldr_reg_off(cg_sec, t2, lhs, 8, 8);
        asm_ldr_reg_off(cg_sec, t3, rhs, 8, 0);
        asm_ldr_reg_off(cg_sec, t4, rhs, 8, 8);
        asm_adds_phy(cg_sec, t1, t1, REG(t3));
        asm_adc_phy(cg_sec, t2, t2, REG(t4)); // adc %s, %s, %s
        asm_str_reg_off(cg_sec, t1, dst, 8, 0);
        asm_str_reg_off(cg_sec, t2, dst, 8, 8);
        free_reg(t1);
        free_reg(t2);
        free_reg(t3);
        free_reg(t4);
#else
        asm_mov_mem_rax(cg_sec, lhs);
        asm_mov_mem8_rdx(cg_sec, lhs);
        asm_addq_mem_rax(cg_sec, rhs);
        asm_adcq_mem8_rdx(cg_sec, rhs);
        asm_mov_rax_mem(cg_sec, dst);
        asm_mov_rdx_mem8(cg_sec, dst);
#endif
        free_reg(lhs);
        free_reg(rhs);
        return dst;
    }

    case ND_SUB: {
        int lhs = gen_to_int128(node->lhs);
        int rhs = gen_to_int128(node->rhs);
        int dst = alloc_int128_addr();
#ifdef ARCH_ARM64
        int t1 = alloc_reg(), t2 = alloc_reg(), t3 = alloc_reg(), t4 = alloc_reg();
        asm_ldr_reg_off(cg_sec, t1, lhs, 8, 0);
        asm_ldr_reg_off(cg_sec, t2, lhs, 8, 8);
        asm_ldr_reg_off(cg_sec, t3, rhs, 8, 0);
        asm_ldr_reg_off(cg_sec, t4, rhs, 8, 8);
        asm_subs_phy(cg_sec, t1, t1, REG(t3));
        asm_sbc_phy(cg_sec, t2, t2, REG(t4)); // sbc %s, %s, %s
        asm_str_reg_off(cg_sec, t1, dst, 8, 0);
        asm_str_reg_off(cg_sec, t2, dst, 8, 8);
        free_reg(t1);
        free_reg(t2);
        free_reg(t3);
        free_reg(t4);
#else
        asm_mov_mem_rax(cg_sec, lhs);
        asm_mov_mem8_rdx(cg_sec, lhs);
        asm_op_mem_rax(cg_sec, "subq", rhs);
        asm_sbbq_mem8_rdx(cg_sec, rhs);
        asm_mov_rax_mem(cg_sec, dst);
        asm_mov_rdx_mem8(cg_sec, dst);
#endif
        free_reg(lhs);
        free_reg(rhs);
        return dst;
    }

    case ND_MUL: {
        int lhs = gen_to_int128(node->lhs);
        int rhs = gen_to_int128(node->rhs);
#ifdef ARCH_ARM64
        int dst = alloc_int128_addr();
        // (alo:ahi) * (blo:bhi): result_lo = alo*blo.lo; result_hi = alo*blo.hi + ahi*blo.lo + alo*bhi.lo
        int a_lo = alloc_reg(), a_hi = alloc_reg(), b_lo = alloc_reg(), b_hi = alloc_reg();
        asm_ldr_reg_off(cg_sec, a_lo, lhs, 8, 0);
        asm_ldr_reg_off(cg_sec, a_hi, lhs, 8, 8);
        asm_ldr_reg_off(cg_sec, b_lo, rhs, 8, 0);
        asm_ldr_reg_off(cg_sec, b_hi, rhs, 8, 8);
        int r_lo = alloc_reg(), r_hi = alloc_reg(), tmp = alloc_reg();
        asm_mul_rd_rn_rm(cg_sec, r_lo, a_lo, b_lo, 8); // mul r_lo, a_lo, b_lo
        asm_umulh(cg_sec, r_hi, a_lo, b_lo); // umulh r_hi, a_lo, b_lo
        asm_mul_rd_rn_rm(cg_sec, tmp, a_hi, b_lo, 8); // mul tmp, a_hi, b_lo
        asm_add_rd_rn_rm(cg_sec, r_hi, r_hi, tmp, 8); // add r_hi, r_hi, tmp
        asm_mul_rd_rn_rm(cg_sec, tmp, a_lo, b_hi, 8); // mul tmp, a_lo, b_hi
        asm_add_rd_rn_rm(cg_sec, r_hi, r_hi, tmp, 8); // add r_hi, r_hi, tmp
        free_reg(tmp);
        free_reg(a_lo);
        free_reg(a_hi);
        free_reg(b_lo);
        free_reg(b_hi);
        asm_str_reg_off(cg_sec, r_lo, dst, 8, 0);
        asm_str_reg_off(cg_sec, r_hi, dst, 8, 8);
        free_reg(r_lo);
        free_reg(r_hi);
#else
        // Load all values first using alloc_reg(), then release lhs/rhs, then allocate dst.
        // This avoids using hardcoded registers that might alias lhs/rhs address regs.
        int a_lo = alloc_reg(), a_hi = alloc_reg(), b_lo = alloc_reg(), b_hi = alloc_reg();
        asm_ldr_reg_off(cg_sec, a_lo, lhs, 8, 0);
        asm_ldr_reg_off(cg_sec, a_hi, lhs, 8, 8);
        asm_ldr_reg_off(cg_sec, b_lo, rhs, 8, 0);
        asm_ldr_reg_off(cg_sec, b_hi, rhs, 8, 8);
        free_reg(lhs);
        free_reg(rhs);
        int dst = alloc_int128_addr();
        asm_mov_reg_to_retval(cg_sec, a_lo, 8);
        asm_mulq(cg_sec, b_lo);
        asm_mov_rax_mem(cg_sec, dst);
        asm_mov_rdx_mem8(cg_sec, dst);
        asm_imulq_reg_reg(cg_sec, a_hi, b_lo);
        asm_add_reg_mem8(cg_sec, dst, a_hi);
        asm_imulq_reg_reg(cg_sec, a_lo, b_hi);
        asm_add_reg_mem8(cg_sec, dst, a_lo);
        free_reg(a_lo);
        free_reg(a_hi);
        free_reg(b_lo);
        free_reg(b_hi);
#endif
        // lhs and rhs freed inside the #else block above (x86-64) or after ARM64 block
#ifdef ARCH_ARM64
        free_reg(lhs);
        free_reg(rhs);
#endif
        return dst;
    }

    case ND_DIV:
    case ND_MOD: {
        bool is_mod = (node->kind == ND_MOD);
        bool is_unsigned = node->ty && node->ty->is_unsigned;
        char *fn = (char *)(is_unsigned ? (is_mod ? "__umodti3" : "__udivti3")
                                        : (is_mod ? "__modti3" : "__divti3"));
        (void)fn; // used in x86 path
        int lhs = gen_to_int128(node->lhs);
        int rhs = gen_to_int128(node->rhs);
        int dst = alloc_int128_addr();
#ifdef ARCH_ARM64
        // Pass args in x0-x3: x0=lhs.lo, x1=lhs.hi, x2=rhs.lo, x3=rhs.hi
        arm64_ldr_uoff(cg_sec, 3, ARM64_X0, REG(lhs), 0); // ldr x0, [lhs]
        arm64_ldr_uoff(cg_sec, 3, ARM64_X1, REG(lhs), 1); // ldr x1, [lhs, #8]
        arm64_ldr_uoff(cg_sec, 3, ARM64_X2, REG(rhs), 0); // ldr x2, [rhs]
        arm64_ldr_uoff(cg_sec, 3, ARM64_X3, REG(rhs), 1); // ldr x3, [rhs, #8]
        arm64_sub_imm(cg_sec, 1, ARM64_SP, ARM64_SP, 32, 0); // sub sp, sp, #32
        arm64_str_uoff(cg_sec, 3, REG(dst), ARM64_SP, 2); // str dst, [sp, #16]
        emit_direct_call(fn, false); // bl fn
        arm64_ldr_uoff(cg_sec, 3, ARM64_X9, ARM64_SP, 2); // ldr x9, [sp, #16]
        arm64_str_uoff(cg_sec, 3, ARM64_X0, ARM64_X9, 0); // str x0, [x9]
        arm64_str_uoff(cg_sec, 3, ARM64_X1, ARM64_X9, 1); // str x1, [x9, #8]
        arm64_ldr_uoff(cg_sec, 3, REG(dst), ARM64_SP, 2); // ldr dst, [sp, #16]
        arm64_add_imm(cg_sec, 1, ARM64_SP, ARM64_SP, 32, 0); // add sp, sp, #32
#else
#ifdef _WIN32
        // Windows x64: __udivti3 etc. take &a in rcx, &b in rdx, return in xmm0
        x86_mov_rr(cg_sec, 8, X86_RCX, REG(lhs)); // movq lhs, %rcx
        x86_mov_rr(cg_sec, 8, X86_RDX, REG(rhs)); // movq rhs, %rdx
        x86_sub_ri(cg_sec, 8, X86_RSP, 32); // subq $32, %rsp
        emit_direct_call(fn, false);
        x86_add_ri(cg_sec, 8, X86_RSP, 32); // addq $32, %rsp
        x86_movdqu_mr(cg_sec, x86_mem(REG(dst), 0), X86_XMM0); // movdqu %xmm0, (dst)
#else
        x86_mov_rm(cg_sec, 8, X86_RDI, x86_mem(REG(lhs), 0)); // movq (lhs), %rdi
        x86_mov_rm(cg_sec, 8, X86_RSI, x86_mem(REG(lhs), 8)); // movq 8(lhs), %rsi
        x86_mov_rm(cg_sec, 8, X86_RDX, x86_mem(REG(rhs), 0)); // movq (rhs), %rdx
        x86_mov_rm(cg_sec, 8, X86_RCX, x86_mem(REG(rhs), 8)); // movq 8(rhs), %rcx
        emit_direct_call(fn, false);
        asm_mov_rax_mem(cg_sec, dst);
        asm_mov_rdx_mem8(cg_sec, dst);
#endif
#endif
        free_reg(lhs);
        free_reg(rhs);
        return dst;
    }

    case ND_SHL:
    case ND_SHR: {
        bool is_shl = (node->kind == ND_SHL);
        bool is_unsigned = node->ty && node->ty->is_unsigned;
        int lhs = gen_to_int128(node->lhs);
        int dst = alloc_int128_addr();
#ifdef ARCH_ARM64
        bool is_rhs_const = (node->rhs->kind == ND_NUM);
        if (is_rhs_const) {
            int shift = (int)(node->rhs->val & 127);
            int t1 = alloc_reg(), t2 = alloc_reg();
            asm_ldr_reg_off(cg_sec, t1, lhs, 8, 0);
            asm_ldr_reg_off(cg_sec, t2, lhs, 8, 8);
            free_reg(lhs);
            if (shift == 0) {
                // no-op
            } else if (is_shl) {
                if (shift < 64) {
                    // hi = (hi<<shift) | (lo>>(64-shift)), lo = lo<<shift
                    arm64_lsl_imm(cg_sec, 1, ARM64_X16, REG(t2), shift);
                    arm64_lsr_imm(cg_sec, 1, ARM64_X17, REG(t1), 64 - shift);
                    arm64_orr_reg(cg_sec, 1, REG(t2), ARM64_X16, ARM64_X17, ARM64_LSL, 0); // orr t2, x16, x17
                    asm_shl_imm(cg_sec, t1, 8, (uint8_t)shift); // shl t1, #shift
                } else if (shift == 64) {
                    asm_mov_reg_reg(cg_sec, t2, t1, 8);
                    asm_mov_xzr_phy(cg_sec, REG(t1)); // mov %s, xzr
                } else {
                    arm64_lsl_imm(cg_sec, 1, REG(t2), REG(t1), shift - 64); // lsl t2, t1, #(shift-64)
                    asm_mov_xzr_phy(cg_sec, REG(t1)); // mov %s, xzr
                }
            } else {
                if (shift < 64) {
                    // lo = (lo>>shift) | (hi<<(64-shift)), hi >>= shift
                    arm64_lsr_imm(cg_sec, 1, ARM64_X16, REG(t1), shift);
                    arm64_lsl_imm(cg_sec, 1, ARM64_X17, REG(t2), 64 - shift);
                    arm64_orr_reg(cg_sec, 1, REG(t1), ARM64_X16, ARM64_X17, ARM64_LSL, 0); // orr t1, x16, x17
                    asm_shift_imm(cg_sec, t2, 8, is_unsigned, shift); // lsr/asr t2, t2, #shift
                } else if (shift == 64) {
                    asm_mov_reg_reg(cg_sec, t1, t2, 8);
                    if (is_unsigned)
                        asm_mov_xzr_phy(cg_sec, REG(t2)); // mov t2, xzr
                    else
                        asm_sar_imm(cg_sec, t2, 8, 63);
                } else {
                    if (is_unsigned)
                        arm64_lsr_imm(cg_sec, 1, REG(t1), REG(t2), shift - 64); // lsr t1, t2, #(shift-64)
                    else
                        arm64_asr_imm(cg_sec, 1, REG(t1), REG(t2), shift - 64); // asr t1, t2, #(shift-64)
                    if (is_unsigned)
                        asm_mov_xzr_phy(cg_sec, REG(t2)); // mov t2, xzr
                    else
                        asm_sar_imm(cg_sec, t2, 8, 63);
                }
            }
            asm_str_reg_off(cg_sec, t1, dst, 8, 0);
            asm_str_reg_off(cg_sec, t2, dst, 8, 8);
            free_reg(t1);
            free_reg(t2);
        } else {
            // Variable shift: call libgcc
            char *fn = is_shl ? "__ashlti3" : (is_unsigned ? "__lshrti3" : "__ashrti3");
            int r_shift = gen(node->rhs);
            arm64_ldr_uoff(cg_sec, 3, ARM64_X0, REG(lhs), 0); // ldr x0, [lhs]
            arm64_ldr_uoff(cg_sec, 3, ARM64_X1, REG(lhs), 1); // ldr x1, [lhs, #8]
            arm64_orr_reg(cg_sec, 1, ARM64_X2, ARM64_XZR, REG(r_shift), ARM64_LSL, 0); // mov x2, r_shift
            free_reg(r_shift);
            free_reg(lhs);
            arm64_sub_imm(cg_sec, 1, ARM64_SP, ARM64_SP, 32, 0); // sub sp, sp, #32
            arm64_str_uoff(cg_sec, 3, REG(dst), ARM64_SP, 2); // str dst, [sp, #16]
            emit_direct_call(fn, false); // bl fn
            arm64_ldr_uoff(cg_sec, 3, ARM64_X9, ARM64_SP, 2); // ldr x9, [sp, #16]
            arm64_str_uoff(cg_sec, 3, ARM64_X0, ARM64_X9, 0); // str x0, [x9]
            arm64_str_uoff(cg_sec, 3, ARM64_X1, ARM64_X9, 1); // str x1, [x9, #8]
            arm64_ldr_uoff(cg_sec, 3, REG(dst), ARM64_SP, 2); // ldr dst, [sp, #16]
            arm64_add_imm(cg_sec, 1, ARM64_SP, ARM64_SP, 32, 0); // add sp, sp, #32
        }
#else
        // x86-64: rax=lo, rdx=hi; use shldq/shrdq
        asm_mov_mem_rax(cg_sec, lhs); // movq (lhs), %%rax
        asm_mov_mem8_rdx(cg_sec, lhs); // movq 8(lhs), %%rdx
        free_reg(lhs);
        if (node->rhs->kind == ND_NUM) {
            int shift = (int)node->rhs->val;
            if (shift == 0) {
                // no-op
            } else if (is_shl) {
                if (shift < 64) {
                    asm_shldq_imm(cg_sec, shift); // shldq $%d, %%rax, %%rdx
                    asm_shl_rax_imm(cg_sec, shift); // shlq $%d, %%rax
                } else if (shift == 64) {
                    asm_mov_rax_rdx(cg_sec); // movq %%rax, %%rdx
                    asm_xor_rax_rax(cg_sec); // xorq %%rax, %%rax
                } else if (shift < 128) {
                    asm_shl_rax_imm(cg_sec, shift - 64); // shlq $%d, %%rax
                    asm_mov_rax_rdx(cg_sec); // movq %%rax, %%rdx
                    asm_xor_rax_rax(cg_sec); // xorq %%rax, %%rax
                } else {
                    asm_xor_rax_rax(cg_sec); // xorq %%rax, %%rax
                    asm_xor_rdx_rdx(cg_sec); // xorq %%rdx, %%rdx
                }
            } else {
                if (shift < 64) {
                    asm_shrdq_imm(cg_sec, shift); // shrdq $%d, %%rdx, %%rax
                    asm_shift_rdx_imm(cg_sec, is_unsigned, shift); // %s $%d, %%rdx
                } else if (shift == 64) {
                    asm_mov_rdx_rax(cg_sec); // movq %%rdx, %%rax
                    if (is_unsigned)
                        asm_xor_rdx_rdx(cg_sec); // xorq %%rdx, %%rdx
                    else
                        asm_shift_rdx_imm(cg_sec, false, 63); // sarq $63, %%rdx
                } else if (shift < 128) {
                    asm_mov_rdx_rax(cg_sec); // movq %%rdx, %%rax
                    asm_shift_rax_imm(cg_sec, is_unsigned, shift - 64); // %s $%d, %%rax
                    if (is_unsigned)
                        asm_xor_rdx_rdx(cg_sec); // xorq %%rdx, %%rdx
                    else
                        asm_shift_rax_imm(cg_sec, false, 63); // sarq $63, %%rax
                } else {
                    if (is_unsigned) {
                        asm_xor_rax_rax(cg_sec); // xorq %%rax, %%rax
                        asm_xor_rdx_rdx(cg_sec); // xorq %%rdx, %%rdx
                    } else {
                        asm_shift_rax_imm(cg_sec, false, 63); // sarq $63, %%rax
                        asm_mov_rax_rdx(cg_sec); // movq %%rax, %%rdx
                    }
                }
            }
        } else {
            int r_shift = gen(node->rhs);
            asm_mov_reg_ecx(cg_sec, r_shift); // movl %r_shift, %ecx
            free_reg(r_shift);
            if (is_shl) {
                asm_shldq_cl(cg_sec); // shldq %%cl, %%rax, %%rdx
                asm_shl_rax_cl(cg_sec); // shlq %%cl, %%rax
                // shld/shl only use count mod 64. For count >= 64:
                // rdx gets the shifted lo, rax becomes 0.
                x86_test_ri(cg_sec, 4, X86_RCX, 0x40); // testl $0x40, %ecx
                size_t o = asm_jcc_label(cg_sec, X86_Z);
                int c = ++rcc_label_count;
                asm_fixup_add(cg_sec, o, format(".L.shl128_%d", c), 1);
                asm_mov_rax_rdx(cg_sec); // movq %%rax, %%rdx
                asm_xor_rax_rax(cg_sec); // xorq %%rax, %%rax
                cg_def_label(format(".L.shl128_%d", c));
            } else {
                asm_shrdq_cl(cg_sec); // shrdq %%cl, %%rdx, %%rax
                asm_shift_rdx_cl(cg_sec, is_unsigned); // %s %%cl, %%rdx
                int c = ++rcc_label_count;
                x86_test_ri(cg_sec, 4, X86_RCX, 0x40); // testl $0x40, %ecx
                size_t o = asm_jcc_label(cg_sec, X86_Z);
                asm_fixup_add(cg_sec, o, format(".L.shr128_%d", c), 1);
                asm_mov_rdx_rax(cg_sec); // movq %%rdx, %%rax
                if (is_unsigned)
                    asm_xor_rdx_rdx(cg_sec); // xorq %%rdx, %%rdx
                else
                    asm_shift_rdx_imm(cg_sec, false, 63); // sarq $63, %%rdx
                cg_def_label(format(".L.shr128_%d", c));
            }
        }
        asm_mov_rax_mem(cg_sec, dst); // movq %%rax, (dst)
        asm_mov_rdx_mem8(cg_sec, dst); // movq %%rdx, 8(dst)
#endif
        return dst;
    }

    case ND_BITAND:
    case ND_BITOR:
    case ND_BITXOR: {
        int lhs = gen_to_int128(node->lhs);
        int rhs = gen_to_int128(node->rhs);
        int dst = alloc_int128_addr();
#ifdef ARCH_ARM64
        int t1 = alloc_reg(), t2 = alloc_reg(), t3 = alloc_reg(), t4 = alloc_reg();
        asm_ldr_reg_off(cg_sec, t1, lhs, 8, 0);
        asm_ldr_reg_off(cg_sec, t2, lhs, 8, 8);
        asm_ldr_reg_off(cg_sec, t3, rhs, 8, 0);
        asm_ldr_reg_off(cg_sec, t4, rhs, 8, 8);
        if (node->kind == ND_BITAND) {
            arm64_and_reg(cg_sec, 1, REG(t1), REG(t1), REG(t3), ARM64_LSL, 0); // and t1, t1, t3
            arm64_and_reg(cg_sec, 1, REG(t2), REG(t2), REG(t4), ARM64_LSL, 0); // and t2, t2, t4
        } else if (node->kind == ND_BITOR) {
            arm64_orr_reg(cg_sec, 1, REG(t1), REG(t1), REG(t3), ARM64_LSL, 0); // orr t1, t1, t3
            arm64_orr_reg(cg_sec, 1, REG(t2), REG(t2), REG(t4), ARM64_LSL, 0); // orr t2, t2, t4
        } else {
            arm64_eor_reg(cg_sec, 1, REG(t1), REG(t1), REG(t3), ARM64_LSL, 0); // eor t1, t1, t3
            arm64_eor_reg(cg_sec, 1, REG(t2), REG(t2), REG(t4), ARM64_LSL, 0); // eor t2, t2, t4
        }
        asm_str_reg_off(cg_sec, t1, dst, 8, 0);
        asm_str_reg_off(cg_sec, t2, dst, 8, 8);
        free_reg(t1);
        free_reg(t2);
        free_reg(t3);
        free_reg(t4);
#else
        const char *op = node->kind == ND_BITAND ? "andq"
            : node->kind == ND_BITOR             ? "orq"
                                                 : "xorq";
        asm_mov_mem_rax(cg_sec, lhs);
        asm_mov_mem8_rdx(cg_sec, lhs);
        asm_op_mem_rax(cg_sec, op, rhs);
        asm_op_mem8_rdx(cg_sec, op, rhs);
        asm_mov_rax_mem(cg_sec, dst);
        asm_mov_rdx_mem8(cg_sec, dst);
#endif
        free_reg(lhs);
        free_reg(rhs);
        return dst;
    }

    // Comparisons: result is int (0 or 1)
    case ND_EQ:
    case ND_NE:
    case ND_LT:
    case ND_LE: {
        int lhs = gen_to_int128(node->lhs);
        int rhs = gen_to_int128(node->rhs);
        bool is_unsigned_cmp = node->lhs->ty && node->lhs->ty->is_unsigned;
        int result = alloc_reg();
        int c = ++rcc_label_count;
#ifdef ARCH_ARM64
        int t1 = alloc_reg(), t2 = alloc_reg(), t3 = alloc_reg(), t4 = alloc_reg();
        asm_ldr_reg_off(cg_sec, t1, lhs, 8, 0);
        asm_ldr_reg_off(cg_sec, t2, lhs, 8, 8);
        asm_ldr_reg_off(cg_sec, t3, rhs, 8, 0);
        asm_ldr_reg_off(cg_sec, t4, rhs, 8, 8);
        free_reg(lhs);
        free_reg(rhs);
        if (node->kind == ND_EQ || node->kind == ND_NE) {
            asm_cmp_reg_reg(cg_sec, t1, t3, 8);
            asm_ccmp_eq(cg_sec, t2, t4); // ccmp %s, %s, #0, eq
            asm_cset(cg_sec, result, node->kind == ND_EQ ? ARM64_EQ : ARM64_NE);
        } else {
            // LT or LE: compare hi first (signed or unsigned), then lo (unsigned)
            int hicond_lt = is_unsigned_cmp ? ARM64_LO : ARM64_LT;
            int hicond_gt = is_unsigned_cmp ? ARM64_HI : ARM64_GT;
            int locond = node->kind == ND_LT ? ARM64_LO : ARM64_LS;
            asm_movq_zero(cg_sec, result);
            asm_cmp_reg_reg(cg_sec, t2, t4, 8);
            emit_jcc_fixup(cg_sec, hicond_lt, format(".L.cmp128y.%d", c));
            emit_jcc_fixup(cg_sec, hicond_gt, format(".L.cmp128n.%d", c));
            asm_cmp_phy_phy(cg_sec, t1, t3); // cmp t1, t3 (lo compare, always unsigned)
            emit_jcc_fixup(cg_sec, locond, format(".L.cmp128y.%d", c));
            emit_jmp_fixup(cg_sec, format(".L.cmp128n.%d", c));
            cg_def_label(format(".L.cmp128y.%d", c));
            asm_mov_imm(cg_sec, result, 4, 1);
            cg_def_label(format(".L.cmp128n.%d", c));
        }
        free_reg(t1);
        free_reg(t2);
        free_reg(t3);
        free_reg(t4);
#else
        if (node->kind == ND_EQ || node->kind == ND_NE) {
            int t1 = alloc_reg(), t2 = alloc_reg(), t3 = alloc_reg(), t4 = alloc_reg();
            asm_ldr_reg_off(cg_sec, t1, lhs, 8, 0);
            asm_ldr_reg_off(cg_sec, t2, lhs, 8, 8);
            asm_ldr_reg_off(cg_sec, t3, rhs, 8, 0);
            asm_ldr_reg_off(cg_sec, t4, rhs, 8, 8);
            asm_xor_reg_reg(cg_sec, t1, t3, 8);
            asm_xor_reg_reg(cg_sec, t2, t4, 8);
            asm_or_reg_reg(cg_sec, t1, t2, 8);
            asm_setcc(cg_sec, X86_RAX, node->kind == ND_EQ ? X86_E : X86_NE);
            asm_movzx_phys(cg_sec, result, X86_RAX, 4, 1);
            free_reg(t1);
            free_reg(t2);
            free_reg(t3);
            free_reg(t4);
        } else {
            asm_mov_mem8_rdx(cg_sec, lhs);
            asm_op_mem8_rdx(cg_sec, "cmpq", rhs);
            if (is_unsigned_cmp) {
                emit_jcc_fixup(cg_sec, X86_B, format(".L.cmp128y.%d", c));
                emit_jcc_fixup(cg_sec, X86_A, format(".L.cmp128n.%d", c));
            } else {
                emit_jcc_fixup(cg_sec, X86_L, format(".L.cmp128y.%d", c));
                emit_jcc_fixup(cg_sec, X86_G, format(".L.cmp128n.%d", c));
            }
            asm_mov_mem_rax(cg_sec, lhs);
            asm_op_mem_rax(cg_sec, "cmpq", rhs);
            if (node->kind == ND_LT)
                emit_jcc_fixup(cg_sec, X86_B, format(".L.cmp128y.%d", c));
            else
                emit_jcc_fixup(cg_sec, X86_BE, format(".L.cmp128y.%d", c));
            cg_def_label(format(".L.cmp128n.%d", c));
            asm_xorl_reg_reg(cg_sec, result, result);
            emit_jmp_fixup(cg_sec, format(".L.cmp128e.%d", c));
            cg_def_label(format(".L.cmp128y.%d", c));
            asm_movl_imm(cg_sec, result, 1);
            cg_def_label(format(".L.cmp128e.%d", c));
        }
        free_reg(lhs);
        free_reg(rhs);
#endif
        return result;
    }

    case ND_COND: {
        // ternary with int128 result
        int c = ++rcc_label_count;
        int cond_r = gen(node->cond);
        int dst = alloc_int128_addr();
        if (node->cond->ty && is_flonum(node->cond->ty)) {
            // IEEE truthiness: -0.0 == 0.0 (false), NaN truthy.
            gen_flonum_branch_if_zero(cond_r, NULL, format(".L.else.%d", c));
        } else {
#ifdef ARCH_ARM64
            asm_cmp_zero(cg_sec, cond_r, node->cond->ty ? node->cond->ty->size : 4);
            emit_jcc_fixup(cg_sec, ARM64_EQ, format(".L.else.%d", c));
#else
            asm_cmp_zero(cg_sec, cond_r, node->cond->ty ? node->cond->ty->size : 4);
            emit_jcc_fixup(cg_sec, X86_E, format(".L.else.%d", c));
#endif
        }
        free_reg(cond_r);
        int then_a = gen_to_int128(node->then);
#ifdef ARCH_ARM64
        {
            int t1 = alloc_reg(), t2 = alloc_reg();
            asm_ldr_reg_off(cg_sec, t1, then_a, 8, 0);
            asm_ldr_reg_off(cg_sec, t2, then_a, 8, 8);
            asm_str_reg_off(cg_sec, t1, dst, 8, 0);
            asm_str_reg_off(cg_sec, t2, dst, 8, 8);
            free_reg(t1);
            free_reg(t2);
        }
        emit_jmp_fixup(cg_sec, format(".L.end.%d", c));
#else
        asm_mov_mem_rax(cg_sec, then_a);
        asm_mov_mem8_rdx(cg_sec, then_a);
        asm_mov_rax_mem(cg_sec, dst);
        asm_mov_rdx_mem8(cg_sec, dst);
        emit_jmp_fixup(cg_sec, format(".L.end.%d", c));
#endif
        free_reg(then_a);
        cg_def_label(format(".L.else.%d", c));
        int else_a = gen_to_int128(node->els);
#ifdef ARCH_ARM64
        {
            int t1 = alloc_reg(), t2 = alloc_reg();
            asm_ldr_reg_off(cg_sec, t1, else_a, 8, 0);
            asm_ldr_reg_off(cg_sec, t2, else_a, 8, 8);
            asm_str_reg_off(cg_sec, t1, dst, 8, 0);
            asm_str_reg_off(cg_sec, t2, dst, 8, 8);
            free_reg(t1);
            free_reg(t2);
        }
#else
        asm_mov_mem_rax(cg_sec, else_a);
        asm_mov_mem8_rdx(cg_sec, else_a);
        asm_mov_rax_mem(cg_sec, dst);
        asm_mov_rdx_mem8(cg_sec, dst);
#endif
        free_reg(else_a);
        cg_def_label(format(".L.end.%d", c));
        return dst;
    }

    case ND_FUNCALL:
        return gen_funcall(node, -1);

#ifdef ARCH_ARM64
    case ND_VA_ARG: {
        // va_arg(ap, __int128) on ARM64: 2 consecutive GP register slots
        int r = gen(node->lhs); // va_list pointer
        int addr = alloc_int128_addr();
        int c = ++rcc_label_count;
        // Check for 2 register slots available (gr_offs <= -16 = at least 2 slots)
        arm64_ldr_uoff(cg_sec, 2, ARM64_X16, REG(r), 6); // ldr w16, [x{r}, #24] (__gr_offs)
        asm_cmp_w16_imm(cg_sec, -16); // cmp w16, #-16
        emit_jcc_fixup(cg_sec, ARM64_GT, format(".L.va128_stk.%d", c)); // b.gt .L.va128_stk.%d
        // Register save area: load both 64-bit halves
        arm64_add_imm(cg_sec, 0, ARM64_X17, ARM64_X16, 16, 0); // add w17, w16, #16
        arm64_str_uoff(cg_sec, 2, ARM64_X17, REG(r), 6); // str w17, [x{r}, #24]
        arm64_ldr_uoff(cg_sec, 3, ARM64_X9, REG(r), 1); // ldr x9, [x{r}, #8] (__gr_top)
        asm_sxtw_x17_w16(cg_sec); // sxtw x17, w16
        arm64_add_reg(cg_sec, 1, ARM64_X9, ARM64_X9, ARM64_X17, ARM64_LSL, 0); // add x9, x9, x17
        arm64_ldp(cg_sec, 1, ARM64_X16, ARM64_X17, ARM64_X9, 0, false, false); // ldp x16, x17, [x9]
        arm64_str_uoff(cg_sec, 3, ARM64_X16, REG(addr), 0); // str x16, [x{addr}]
        arm64_str_uoff(cg_sec, 3, ARM64_X17, REG(addr), 1); // str x17, [x{addr}, #8]
        emit_jmp_fixup(cg_sec, format(".L.va128_d.%d", c)); // b .L.va128_d.%d
        // Overflow stack: 16-byte aligned
        cg_def_label(format(".L.va128_stk.%d", c)); // .L.xxx.c
        arm64_ldr_uoff(cg_sec, 3, ARM64_X16, REG(r), 0); // ldr x16, [x{r}] (__stack)
        arm64_add_imm(cg_sec, 1, ARM64_X17, ARM64_X16, 15, 0); // add x17, x16, #15
        arm64_and_imm(cg_sec, 1, ARM64_X17, ARM64_X17, (uint64_t)-16); // and x17, x17, #-16 (align to 16)
        arm64_str_uoff(cg_sec, 3, ARM64_X17, REG(r), 0); // str x17, [x{r}]
        arm64_ldp(cg_sec, 1, ARM64_X16, ARM64_X17, ARM64_X17, 0, false, false); // ldp x16, x17, [x17]
        arm64_str_uoff(cg_sec, 3, ARM64_X16, REG(addr), 0); // str x16, [x{addr}]
        arm64_str_uoff(cg_sec, 3, ARM64_X17, REG(addr), 1); // str x17, [x{addr}, #8]
        cg_def_label(format(".L.va128_d.%d", c)); // .L.xxx.c
        free_reg(r);
        return addr;
    }
#elif defined(_WIN32)
    case ND_VA_ARG: {
        // Windows x64: va_list is char*. __int128 is passed by pointer (8-byte slot).
        // The shadow space slot holds a pointer to the 16-byte int128 value.
        int r = gen_addr(node->lhs); // address of the char* ap variable
        int addr = alloc_int128_addr();
        x86_mov_rm(cg_sec, 8, X86_RCX, x86_mem(REG(r), 0)); // movq (r), %rcx  (rcx = ap)
        x86_add_mi(cg_sec, 8, x86_mem(REG(r), 0), 8); // addq $8, (r)  (ap += 8)
        x86_mov_rm(cg_sec, 8, X86_RCX, x86_mem(X86_RCX, 0)); // movq (%rcx), %rcx  (rcx = *slot)
        x86_mov_rm(cg_sec, 8, X86_RAX, x86_mem(X86_RCX, 0)); // movq (%rcx), %rax  (rax = lo)
        x86_mov_mr(cg_sec, 8, x86_mem(REG(addr), 0), X86_RAX); // movq %rax, (addr)
        x86_mov_rm(cg_sec, 8, X86_RAX, x86_mem(X86_RCX, 8)); // movq 8(%rcx), %rax  (rax = hi)
        x86_mov_mr(cg_sec, 8, x86_mem(REG(addr), 8), X86_RAX); // movq %rax, 8(addr)
        free_reg(r);
        return addr;
    }
#else
    case ND_VA_ARG: {
        // va_arg(ap, __int128) on x86-64: 2 consecutive GP register slots
        int r = gen(node->lhs); // va_list pointer
        int addr = alloc_int128_addr();
        int c = ++rcc_label_count;
        // gp_offset + 16 <= 48 means <= 32 (2+ register slots available)
        x86_cmp_mi(cg_sec, 4, (X86Mem){REG(r), X86_NOREG, 1, 0}, 32); // cmpl $32, (r) (gp_offset)
        emit_jcc_fixup(cg_sec, X86_A, format(".L.va128_stk.%d", c));
        // Register save area: load both 64-bit halves
        x86_mov_rm(cg_sec, 4, X86_RCX, (X86Mem){REG(r), X86_NOREG, 1, 0}); // movl (r), %ecx (gp_offset)
        x86_add_rm(cg_sec, 8, X86_RCX, (X86Mem){REG(r), X86_NOREG, 1, 16}); // addq 16(r), %rcx (reg_save_area)
        x86_add_mi(cg_sec, 4, (X86Mem){REG(r), X86_NOREG, 1, 0}, 16); // addl $16, (r)
        x86_mov_rm(cg_sec, 8, X86_RAX, (X86Mem){X86_RCX, X86_NOREG, 1, 0}); // movq (%rcx), %rax (lo)
        asm_mov_rax_mem(cg_sec, addr); // movq %rax, (addr)
        x86_mov_rm(cg_sec, 8, X86_RAX, (X86Mem){X86_RCX, X86_NOREG, 1, 8}); // movq 8(%rcx), %rax (hi)
        x86_mov_mr(cg_sec, 8, x86_mem(REG(addr), 8), X86_RAX); // movq %rax, 8(addr)
        emit_jmp_fixup(cg_sec, format(".L.va128_d.%d", c));
        // Overflow stack: 16-byte aligned
        cg_def_label(format(".L.va128_stk.%d", c));
        x86_mov_rm(cg_sec, 8, X86_RAX, (X86Mem){REG(r), X86_NOREG, 1, 8}); // movq 8(r), %rax (overflow_arg_area)
        VReg t = alloc_reg();
        asm_mov_reg_reg(cg_sec, t, r, 8); // t = r (copy va_list ptr)
        x86_lea(cg_sec, 8, REG(t), (X86Mem){X86_RAX, X86_NOREG, 1, 15}); // leaq 15(%rax), %rt
        x86_and_ri(cg_sec, 8, REG(t), -16); // andq $-16, %rt (align)
        x86_lea(cg_sec, 8, REG(t), (X86Mem){REG(t), X86_NOREG, 1, 16}); // leaq 16(%rt), %rt (skip aligned block)
        x86_mov_mr(cg_sec, 8, (X86Mem){REG(r), X86_NOREG, 1, 8}, REG(t)); // movq %rt, 8(r) (update overflow)
        x86_mov_rm(cg_sec, 8, X86_RAX, (X86Mem){REG(t), X86_NOREG, 1, -16}); // movq -16(%rt), %rax (lo)
        asm_mov_rax_mem(cg_sec, addr); // movq %rax, (addr)
        x86_mov_rm(cg_sec, 8, X86_RAX, (X86Mem){REG(t), X86_NOREG, 1, -8}); // movq -8(%rt), %rax (hi)
        x86_mov_mr(cg_sec, 8, x86_mem(REG(addr), 8), X86_RAX); // movq %rax, 8(addr)
        free_reg(t);
        cg_def_label(format(".L.va128_d.%d", c));
        free_reg(r);
        return addr;
    }
#endif
    case ND_POST_DEC:
    case ND_PRE_INC:
    case ND_PRE_DEC: {
        bool is_pre = (node->kind == ND_PRE_INC || node->kind == ND_PRE_DEC);
        bool is_inc = (node->kind == ND_POST_INC || node->kind == ND_PRE_INC);
        VReg lhs_a = gen_addr(node->lhs);
        VReg old_addr = is_pre ? R_NONE : alloc_int128_addr();
#ifdef ARCH_ARM64
        VReg t1 = alloc_reg(), t2 = alloc_reg();
        asm_ldr_reg_off(cg_sec, t1, lhs_a, 8, 0); // ldr x{t1}, [x{lhs_a}]
        asm_ldr_reg_off(cg_sec, t2, lhs_a, 8, 8); // ldr x{t2}, [x{lhs_a}, #8]
        if (!is_pre) {
            asm_str_reg_off(cg_sec, t1, old_addr, 8, 0); // str x{t1}, [x{old_addr}]
            asm_str_reg_off(cg_sec, t2, old_addr, 8, 8); // str x{t2}, [x{old_addr}, #8]
        }
        if (is_inc) {
            arm64_adds_imm(cg_sec, 1, REG(t1), REG(t1), 1, 0); // adds x{t1}, x{t1}, #1
            asm_adc_phy(cg_sec, t2, t2, ARM64_XZR); // adc x{t2}, x{t2}, xzr
        } else {
            arm64_subs_imm(cg_sec, 1, REG(t1), REG(t1), 1, 0); // subs x{t1}, x{t1}, #1
            asm_sbc_phy(cg_sec, t2, t2, ARM64_XZR); // sbc x{t2}, x{t2}, xzr
        }
        asm_str_reg_off(cg_sec, t1, lhs_a, 8, 0); // str x{t1}, [x{lhs_a}]
        asm_str_reg_off(cg_sec, t2, lhs_a, 8, 8); // str x{t2}, [x{lhs_a}, #8]
        free_reg(t1);
        free_reg(t2);
#else
        asm_mov_mem_rax(cg_sec, lhs_a); // movq (lhs_a), %rax
        asm_mov_mem8_rdx(cg_sec, lhs_a); // movq 8(lhs_a), %rdx
        if (!is_pre) {
            asm_mov_rax_mem(cg_sec, old_addr); // movq %rax, (old_addr)
            asm_mov_rdx_mem8(cg_sec, old_addr); // movq %rdx, 8(old_addr)
        }
        if (is_inc) {
            x86_add_ri(cg_sec, 8, X86_RAX, 1); // addq $1, %rax
            secbuf_emit32le(cg_sec, 0x00D28348); // adcq $0, %rdx (REX.W 83 /2 ib)
        } else {
            x86_sub_ri(cg_sec, 8, X86_RAX, 1); // subq $1, %rax
            secbuf_emit32le(cg_sec, 0x00DA8348); // sbbq $0, %rdx (REX.W 83 /3 ib)
        }
        asm_mov_rax_mem(cg_sec, lhs_a); // movq %rax, (lhs_a)
        asm_mov_rdx_mem8(cg_sec, lhs_a); // movq %rdx, 8(lhs_a)
#endif
        if (is_pre)
            return lhs_a;
        free_reg(lhs_a);
        return old_addr;
    }
    case ND_STMT_EXPR: {
        // GNU statement expression `({ ...; last_expr; })` in int128
        // context (e.g. the kernel's typeof()/min()/max() macros applied
        // to a __int128/unsigned __int128 operand): run the body for side
        // effects and materialize the trailing expression as the result.
        VReg result = R_NONE;
        for (Node *n = node->body; n; n = n->next) {
            if (node->stmt_expr_result && n->kind == ND_EXPR_STMT && n->lhs == node->stmt_expr_result) {
                result = gen_to_int128(node->stmt_expr_result);
            } else {
                VReg r = gen(n);
                if (r != -1) free_reg(r);
            }
        }
        if (result == R_NONE)
            result = alloc_int128_addr();
        return result;
    }
    default:
        error("int128: unsupported node kind %d", node->kind);
        return R_NONE;
    }
}

#ifdef ARCH_ARM64
// Element-wise vector (__attribute__((vector_size))) arithmetic/compare/unary.
// Convention (mirrors int128): the value lives in a 16-byte stack slot and this
// returns a GP register holding the slot address. Operands are also vectors and
// gen() returns their addresses, so we movups them into xmm and emit packed ops.
// Get the address of a scalar operand for vector broadcast.  For lvalues this
// is just gen_addr(); for non-lvalues (constants, expressions) we evaluate
// with gen() and spill to a 16-byte-aligned scratch slot.
VReg gen_scalar_addr(Node *node) {
    VReg a = gen_addr(node);
    if (a >= 0) return a;
    VReg val = gen(node);
    VReg slot = alloc_int128_addr();
#ifdef ARCH_ARM64
    arm64_str_imm(cg_sec, 1, REG(val), REG(slot), 0, false);
#else // unused
    x86_mov_mr(cg_sec, 8, x86_mem(REG(slot), 0), REG(val));
#endif
    free_reg(val);
    return slot;
}
#endif /* !ARCH_ARM64 */

static VReg gen_cast_reg(VReg r, Type *from, Type *to) {
    if (to->kind == TY_BOOL && !is_complex(from)) {
        // C11 6.3.1.2: converting any scalar value to _Bool yields 0 if
        // the value compares equal to 0, 1 otherwise. This must produce
        // an actual 0/1 normalization, not a truncating bit-copy.
        if (is_flonum(from)) {
            // IEEE truthiness: -0.0 == 0.0 (false), NaN truthy. On ARM64 a
            // bare `cset r, ne` after fcmp misclassifies NaN (unordered
            // sets Z=1) as false; gen_flonum_truthiness handles both.
            gen_flonum_truthiness(r);
        } else {
#ifdef ARCH_ARM64
            asm_cmp_zero(cg_sec, r, from->size > 0 ? from->size : 8);
            asm_cset(cg_sec, r, ARM64_NE);
#else
            asm_cmp_zero(cg_sec, r, from->size > 0 ? from->size : 8);
            asm_setcc(cg_sec, X86_RAX, X86_NE); // setne %al
            asm_movzx_phys(cg_sec, r, X86_RAX, 4, 1); // movzbl %al, r
#endif
        }
        return r;
    }
    if (is_flonum(from) && is_integer(to)) {
#ifdef ARCH_ARM64
        asm_fmov_i2f(cg_sec, 0, r, 1); // fmov d0, x{r}
        {
            if (to->is_unsigned)
                asm_fcvtzu(cg_sec, r, to->size); // fcvtzu w/x{r}, d0
            else
                asm_cvttsd2si(cg_sec, r, to->size); // fcvtzs w/x{r}, d0
        }
#else
        asm_movq_r_xmm(cg_sec, X86_XMM0, r); // movq r, %xmm0
        if (to->size == 8 && to->is_unsigned) {
            int c = ++rcc_label_count;
            asm_movabs_phy(cg_sec, X86_RAX, 0x43e0000000000000ULL); // movabs $0x43e0000000000000, %rax (2^63 as double)
            x86_movq_r_xmm(cg_sec, X86_XMM1, X86_RAX); // movq %rax, %xmm1
            asm_ucomisd(cg_sec); // comisd %xmm1, %xmm0
            {
                size_t o = asm_jcc_label(cg_sec, X86_B); // jb .L.ucast.c
                asm_fixup_add(cg_sec, o, format(".L.ucast.%d", c), 1);
            }
            asm_subsd(cg_sec); // subsd %xmm1, %xmm0
            asm_cvttsd2si(cg_sec, r, 8); // cvttsd2si %xmm0, rr
            x86_movabs(cg_sec, X86_RCX, 1ULL << 63); // movabs $0x8000000000000000, %rcx
            x86_or_rr(cg_sec, 8, REG(r), X86_RCX); // orq %rcx, rr
            {
                size_t o = asm_jmp_label(cg_sec); // jmp .L.ucast_end.c
                asm_fixup_add(cg_sec, o, format(".L.ucast_end.%d", c), 0);
            }
            cg_def_label(format(".L.ucast.%d", c)); // .L.ucast.c:
            asm_cvttsd2si(cg_sec, r, 8); // cvttsd2si %xmm0, rr
            cg_def_label(format(".L.ucast_end.%d", c)); // .L.ucast_end.c:
        } else if (to->size <= 4 && to->is_unsigned) {
            // float-to-unsigned-int: cvttsd2si is signed, so handle [2^31, 2^32) range.
            int c = ++rcc_label_count;
            asm_movabs_phy(cg_sec, X86_RAX, 0x41e0000000000000ULL); // movabs $0x41e0000000000000, %rax (2^31 as double)
            x86_movq_r_xmm(cg_sec, X86_XMM1, X86_RAX); // movq %rax, %xmm1
            asm_ucomisd(cg_sec); // comisd %xmm1, %xmm0
            {
                size_t o = asm_jcc_label(cg_sec, X86_B); // jb .L.ucast32.c
                asm_fixup_add(cg_sec, o, format(".L.ucast32.%d", c), 1);
            }
            asm_subsd(cg_sec); // subsd %xmm1, %xmm0
            asm_cvttsd2si(cg_sec, r, 4); // cvttsd2si %xmm0, rr
            asm_add_imm(cg_sec, r, 4, (int)(1U << 31)); // addl $0x80000000, rr
            {
                size_t o = asm_jmp_label(cg_sec); // jmp .L.ucast32_end.c
                asm_fixup_add(cg_sec, o, format(".L.ucast32_end.%d", c), 0);
            }
            cg_def_label(format(".L.ucast32.%d", c)); // .L.ucast32.c:
            asm_cvttsd2si(cg_sec, r, 4); // cvttsd2si %xmm0, rr
            cg_def_label(format(".L.ucast32_end.%d", c)); // .L.ucast32_end.c:
        } else if (to->size <= 4 && !to->is_unsigned) {
            int c = ++rcc_label_count;
            asm_cvttsd2si(cg_sec, r, 4); // cvttsd2si xmm0, rr
            asm_cmp_imm(cg_sec, r, 4, (int32_t)0x80000000); // cmpl $0x80000000, rr
            {
                size_t o = asm_jcc_label(cg_sec, X86_NE); // jcc label
                asm_fixup_add(cg_sec, o, format(".L.u2f.end.%d", c), 1);
            }
            x86_xorpd(cg_sec, X86_XMM1, X86_XMM1); // xorpd %xmm1, %xmm1
            asm_ucomisd(cg_sec); // ucomisd %xmm1, %xmm0
            {
                size_t o = asm_jcc_label(cg_sec, X86_B); // jb .L.sat_end.c
                asm_fixup_add(cg_sec, o, format(".L.u2f.end.%d", c), 1);
            }
            asm_mov_imm(cg_sec, r, 4, 0x7fffffff); // movl $0x7fffffff, rr
            cg_def_label(format(".L.saturate.%d", c)); // (saturate value)
            cg_def_label(format(".L.u2f.end.%d", c));
        } else {
            asm_cvttsd2si(cg_sec, r, to->size); // cvttsd2si %xmm0, rr
        }
#endif
    } else if (is_integer(from) && is_flonum(to)) {
#ifdef ARCH_ARM64
        int sf = (from->size == 8) ? 1 : 0;
        if (from->is_unsigned)
            asm_ucvtf(cg_sec, 0, r, sf); // ucvtf d0, w/x{r}
        else
            asm_scvtf(cg_sec, 0, r, sf); // scvtf d0, w/x{r}
        if (to->kind == TY_FLOAT) {
            asm_fcvt(cg_sec, 0, 1, 0, 0); // fcvt s0, d0 (double→single, round to float)
            asm_fcvt(cg_sec, 1, 0, 0, 0); // fcvt d0, s0 (single→double, back to GP-friendly)
        }
        asm_fmov_f2i(cg_sec, r, 0, 1); // fmov x{r}, d0 (store as int bits)
#else
        if (from->is_unsigned && from->size == 8) {
            int c = ++rcc_label_count;
            asm_test(cg_sec, REG(r), REG(r), 8); // test r, r
            {
                size_t o = asm_jcc_label(cg_sec, X86_S); // jcc label
                asm_fixup_add(cg_sec, o, format(".L.u2f.high.%d", c), 1);
            }
            asm_cvtsi2sd(cg_sec, r, 8); // cvtsi2sd r, xmm0
            {
                size_t o = asm_jmp_label(cg_sec); // cvttsd2si %%xmm0, %s
                asm_fixup_add(cg_sec, o, format(".L.u2f.end.%d", c), 0);
            }
            cg_def_label(format(".L.u2f.high.%d", c)); // ucvtf d0, %s
            x86_mov_rr(cg_sec, 8, X86_RCX, REG(r)); // movq r, %rcx
            x86_shr_ri(cg_sec, 8, X86_RCX, 1); // shrq $1, %rcx
            x86_cvtsi2sd(cg_sec, 8, X86_XMM0, X86_RCX); // cvtsi2sd %rcx, %xmm0
            x86_addsd(cg_sec, X86_XMM0, X86_XMM0); // addsd %xmm0, %xmm0 (double it)
            cg_def_label(format(".L.u2f.end.%d", c)); // .L.u2f.end.%d:
        } else if (from->is_unsigned && from->size == 4) {
            asm_cvtsi2sd(cg_sec, r, 8); // cvtsi2sd rr, %xmm0
        } else {
            asm_cvtsi2sd(cg_sec, r, from->size); // cvtsi2sd rr, %xmm0
        }
        if (to->kind == TY_FLOAT) {
            asm_cvtsd2ss(cg_sec); // jmp .L.u2f.end.%d
            asm_cvtss2sd(cg_sec); // .L.u2f.high.%d:
        }
        asm_movq_xmm_r(cg_sec, r, X86_XMM0); // movq %xmm0, r
#endif
    } else if (is_flonum(from) && is_flonum(to)) {
#ifdef ARCH_ARM64
        if (to->kind == TY_FLOAT && from->kind != TY_FLOAT) {
            asm_fmov_i2f(cg_sec, 0, r, 1); // fmov d0, x{r}
            asm_fcvt(cg_sec, 0, 1, 0, 0); // fcvt s0, d0 (opc=0=single dest, ftype=1=double src)
            asm_fcvt(cg_sec, 1, 0, 0, 0); // fcvt d0, s0 (opc=1=d, ftype=0=s)
            asm_fmov_f2i(cg_sec, r, 0, 1); // fmov x{r}, d0
        }
#else
        if (to->kind == TY_FLOAT && from->kind != TY_FLOAT) {
            asm_movq_r_xmm(cg_sec, X86_XMM0, r); // movq r, %xmm0
            asm_cvtsd2ss(cg_sec); // cvtsd2ss %xmm0, %xmm0
            asm_cvtss2sd(cg_sec); // cvtss2sd %xmm0, %xmm0
            asm_movq_xmm_r(cg_sec, r, X86_XMM0); // movq %xmm0, r
        }
#endif
    } else if ((is_flonum(from) || is_integer(from)) && is_complex(to)) {
        int alloc = (to->size + 7) & ~7;
        fn_struct_ret_off += alloc;
        if (fn_struct_ret_off > fn_struct_ret_total) fn_struct_ret_total = fn_struct_ret_off;
        int result_off = current_fn_stack_size + fn_struct_ret_off;
        int result = alloc_reg();
#ifdef ARCH_ARM64
        asm_sub_reg_fp_imm(cg_sec, result, result_off); // sub result, x29, #result_off
#else
        asm_lea_rbp_reg(cg_sec, result, 8, result_off);
#endif
        emit_scalar_to_complex(r, from, to->base, result);
        free_reg(r); // the scalar source register is dead; the value lives at (result)
        return result;
    } else if (is_complex(from) && (is_flonum(to) || is_integer(to))) {
        // complex → float/int: load real part from address
        if (is_flonum(to)) {
            if (is_integer(from->base)) {
                // int complex → float: convert via cvtsi2ss/cvtsi2sd
#ifndef ARCH_ARM64
                if (from->base->size <= 4) {
                    x86_mov_rm(cg_sec, 4, REG(r), x86_mem(REG(r), 0)); // movl (r), r
                    asm_cvtsi2ss(cg_sec, REG(r), 4); // cvtsi2ss r32, %xmm0
                } else {
                    x86_mov_rm(cg_sec, 4, REG(r), x86_mem(REG(r), 0)); // movl (r), r
                    asm_cvtsi2sd(cg_sec, r, 4); // cvtsi2sd r32, %xmm0
                }
                if (to->size == 4)
                    asm_cvtsd2ss(cg_sec);
                asm_movq_xmm_r(cg_sec, r, X86_XMM0); // movq %xmm0, r
#else
                // ARM64: load real part, convert to float
                if (from->base->size <= 4)
                    arm64_ldr_uoff(cg_sec, 2, ARM64_X0, REG(r), 0); // ldr w0, [r]
                else
                    arm64_ldr_uoff(cg_sec, 3, ARM64_X0, REG(r), 0); // ldr x0, [r]
                if (from->is_unsigned)
                    arm64_ucvtf(cg_sec, (from->base->size <= 4 ? 0 : 1), 1, ARM64_D0, ARM64_X0); // ucvtf d0, w0/x0
                else
                    arm64_scvtf(cg_sec, (from->base->size <= 4 ? 0 : 1), 1, ARM64_D0, ARM64_X0); // scvtf d0, w0/x0
                if (to->size == 4)
                    arm64_fcvt(cg_sec, 1, 2, ARM64_S0, ARM64_D0); // fcvt s0, d0
                arm64_fmov_f2i(cg_sec, (to->size == 4 ? 0 : 1), REG(r), (to->size == 4 ? ARM64_S0 : ARM64_D0)); // fmov r, s0/d0
#endif
            } else {
#ifndef ARCH_ARM64
                if (from->base->size == 4 && to->size == 8) {
                    // float complex → double: promote via cvtss2sd
                    asm_mov_fp_rm(cg_sec, 4, X86_XMM0, x86_mem(REG(r), 0)); // movss (r), %xmm0
                    asm_cvtss2sd(cg_sec);
                    asm_movq_xmm_r(cg_sec, r, X86_XMM0); // movq %xmm0, r
                } else if (to->size == 4) {
                    asm_mov_fp_rm(cg_sec, 4, X86_XMM0, x86_mem(REG(r), 0)); // movss (r), %xmm0
                    asm_movq_xmm_r(cg_sec, r, X86_XMM0); // movq %xmm0, r
                } else {
                    asm_mov_fp_rm(cg_sec, 8, X86_XMM0, x86_mem(REG(r), 0)); // movsd (r), %xmm0
                    asm_movq_xmm_r(cg_sec, r, X86_XMM0); // movq %xmm0, r
                }
#else
                if (from->base->size == 4 && to->size == 8) {
                    arm64_ldr_fp(cg_sec, 2, 0, REG(r), 0); // ldr s0, [r]
                    arm64_fcvt(cg_sec, 1, 2, ARM64_D0, ARM64_S0); // fcvt d0, s0
                    arm64_fmov_f2i(cg_sec, 1, REG(r), ARM64_D0); // fmov r, d0
                } else if (to->size == 4) {
                    arm64_ldr_fp(cg_sec, 2, 0, REG(r), 0); // ldr s0, [r]
                    arm64_fmov_f2i(cg_sec, 0, REG(r), ARM64_S0); // fmov r, s0
                } else {
                    arm64_ldr_fp(cg_sec, 3, 0, REG(r), 0); // ldr d0, [r]
                    arm64_fmov_f2i(cg_sec, 1, REG(r), ARM64_D0); // fmov r, d0
                }
#endif
            }
        } else {
            // integer: convert from float or load directly
            if (is_flonum(from->base)) {
#ifndef ARCH_ARM64
                if (from->base->size == 4) {
                    asm_mov_fp_rm(cg_sec, 4, X86_XMM0, x86_mem(REG(r), 0)); // movss (r), %xmm0
                    asm_cvttsd2si(cg_sec, r, 4); // cvttss2si %xmm0, r
                } else {
                    asm_mov_fp_rm(cg_sec, 8, X86_XMM0, x86_mem(REG(r), 0)); // movsd (r), %xmm0
                    asm_cvttsd2si(cg_sec, r, 8); // cvttsd2si %xmm0, r
                }
#else
                if (from->base->size == 4) {
                    arm64_ldr_fp(cg_sec, 2, 0, REG(r), 0); // ldr s0, [r]
                    if (to->is_unsigned)
                        arm64_fcvtzu(cg_sec, 0, 2, REG(r), ARM64_S0); // fcvtzu w{r}, s0
                    else
                        arm64_fcvtzs(cg_sec, 0, 2, REG(r), ARM64_S0); // fcvtzs w{r}, s0
                } else {
                    arm64_ldr_fp(cg_sec, 3, 0, REG(r), 0); // ldr d0, [r]
                    if (to->is_unsigned)
                        arm64_fcvtzu(cg_sec, 1, 3, REG(r), ARM64_D0); // fcvtzu x{r}, d0
                    else
                        arm64_fcvtzs(cg_sec, 1, 3, REG(r), ARM64_D0); // fcvtzs x{r}, d0
                }
#endif
            } else {
#ifdef ARCH_ARM64
                emit_load(to, r, r, 0);
#else
                emit_load(to, r, r, 0);
#endif
            }
        }
    } else if (is_complex(from) && is_complex(to) && from->size != to->size) {
        // Complex type promotion/demotion (e.g., _Complex float → _Complex double)
        int alloc = (to->size + 7) & ~7;
        fn_struct_ret_off += alloc;
        if (fn_struct_ret_off > fn_struct_ret_total) fn_struct_ret_total = fn_struct_ret_off;
        int result_off = current_fn_stack_size + fn_struct_ret_off;
        int result = alloc_reg();
        int base_to = to->base ? to->base->size : 8;
        int base_from = from->base ? from->base->size : 4;
#ifdef ARCH_ARM64
        asm_sub_reg_fp_imm(cg_sec, result, result_off); // sub result, x29, #result_off
        if (base_from == 4 && base_to == 8) {
            if (is_flonum(from->base)) {
                // _Complex float → _Complex double
                arm64_ldr_fp(cg_sec, 2, ARM64_S0, REG(r), 0);
                arm64_fcvt(cg_sec, 1, 0, ARM64_D0, ARM64_S0);
                arm64_str_fp(cg_sec, 3, ARM64_D0, REG(result), 0);
                arm64_ldr_fp(cg_sec, 2, ARM64_S0, REG(r), 4);
                arm64_fcvt(cg_sec, 1, 0, ARM64_D0, ARM64_S0);
                arm64_str_fp(cg_sec, 3, ARM64_D0, REG(result), 8);
            } else {
                // _Complex int → _Complex double
                arm64_ldrsw_uoff(cg_sec, ARM64_X0, REG(r), 0);
                arm64_ldrsw_uoff(cg_sec, ARM64_X1, REG(r), 1);
                arm64_scvtf(cg_sec, 0, 1, ARM64_D0, ARM64_X0);
                arm64_scvtf(cg_sec, 0, 1, ARM64_D1, ARM64_X1);
                arm64_str_fp(cg_sec, 3, ARM64_D0, REG(result), 0);
                arm64_str_fp(cg_sec, 3, ARM64_D1, REG(result), 8);
            }
        }
#else
        asm_lea_rbp_reg(cg_sec, result, 8, result_off);
        if (base_from == 4 && base_to == 8) {
            // float complex → double complex
            asm_mov_fp_rm(cg_sec, 4, X86_XMM0, x86_mem(REG(r), 0)); // movss (r), %xmm0
            asm_cvtss2sd(cg_sec); // cvtss2sd %xmm0, %xmm0
            x86_movsd_mr(cg_sec, x86_mem(REG(result), 0), X86_XMM0); // movsd %xmm0, (result)
            asm_mov_fp_rm(cg_sec, 4, X86_XMM0, x86_mem(REG(r), 4)); // movss 4(r), %xmm0
            asm_cvtss2sd(cg_sec);
            x86_movsd_mr(cg_sec, x86_mem(REG(result), 8), X86_XMM0); // movsd %xmm0, 8(result)
        }
#endif
        free_reg(r);
        return result;
    } else if (to->size == 1) {
#ifdef ARCH_ARM64
        if (to->is_unsigned)
            asm_and_imm(cg_sec, r, 4, 0xff); // and w{r}, w{r}, #0xff
        else
            asm_movsx(cg_sec, r, r, 4, 1); // sxtb w{r}, w{r}
#else
        if (to->is_unsigned)
            asm_movzx(cg_sec, r, r, 4, 1); // movzbl r, r
        else
            asm_movsx(cg_sec, r, r, 4, 1); // movsbl r, r
#endif
    } else if (to->size == 2) {
#ifdef ARCH_ARM64
        if (to->is_unsigned)
            asm_and_imm(cg_sec, r, 4, 0xffff); // and w{r}, w{r}, #0xffff
        else
            asm_movsx(cg_sec, r, r, 4, 2); // sxth w{r}, w{r}
#else
        if (to->is_unsigned)
            asm_movzx(cg_sec, r, r, 4, 2); // movzx4->r rr, rr
        else
            asm_movsx(cg_sec, r, r, 4, 2); // cvtsd2ss %%xmm0, %%xmm0
#endif
    } else if (to->size == 4 && from->size == 8) {
        asm_mov_reg_reg(cg_sec, r, r, 4); // cvtss2sd %%xmm0, %%xmm0
    } else if (to->size == 8 && from->size < 8) {
        if (from->kind == TY_ARRAY || from->kind == TY_VLA) {
            // Array/VLA decayed to pointer; already 8-byte address in reg
        } else if (from->is_unsigned)
            zero_extend_to(r, from->size, 8);
        else
            sign_extend_to(r, from->size, 8);
    }
    return r;
}

// ===== _BitInt(N) with N > 64 (wide multi-limb values) =====
// Convention: a TY_BITINT value with size > 8 lives in an `size`-byte stack
// slot, little-endian, padding bits above N sign/zero-extended (see
// bitint_type() in type.c). gen_bitint(node) returns a GP register holding
// the slot address, mirroring gen_int128's convention. All arithmetic is
// done by the per-TU runtime helpers in src/bitint_rt.c (self-hosted: the
// driver parses that source into every TU that uses a wide _BitInt, see
// main.c's bitint-runtime injection). Helpers take (bits, ...) with the
// standard calling convention; the code below stages operands into fresh
// slots first because the in-place helpers clobber their rp (and div also
// clobbers lp).

// Move `bits` (the _BitInt width) into the first integer arg register.
static void emit_bitint_bits_arg(int bits) {
#ifdef ARCH_ARM64
    asm_mov_imm_phy(cg_sec, ARM64_X0, 4, bits); // mov w0, #bits
#elif defined(_WIN32)
    x86_mov_ri(cg_sec, 4, X86_RCX, bits); // movl $bits, %ecx
#else
    x86_mov_ri(cg_sec, 4, X86_RDI, bits); // movl $bits, %edi
#endif
}

// Move VReg `r` (holding an address) into integer arg register N (0-based).
static void emit_bitint_addr_arg(VReg r, int n) {
#ifdef ARCH_ARM64
    static const Arm64Reg a64[] = {ARM64_X1, ARM64_X2, ARM64_X3, ARM64_X4};
    asm_mov_phy_reg(cg_sec, a64[n], r, 1); // mov xN, x{r}
#elif defined(_WIN32)
    static const X86Reg aw[] = {X86_RDX, X86_R8, X86_R9};
    x86_mov_rr(cg_sec, 8, aw[n], cg_x86_reg[r]); // movq %r, %arg
#else
    static const X86Reg al[] = {X86_RSI, X86_RDX, X86_RCX, X86_R8};
    x86_mov_rr(cg_sec, 8, al[n], cg_x86_reg[r]); // movq %r, %arg
#endif
}

// Move an immediate into integer arg register N (0-based).
static void emit_bitint_imm_arg(int imm, int n) {
#ifdef ARCH_ARM64
    static const Arm64Reg a64[] = {ARM64_X0, ARM64_X1, ARM64_X2, ARM64_X3, ARM64_X4};
    asm_mov_imm_phy(cg_sec, a64[n], 4, imm); // mov wN, #imm
#elif defined(_WIN32)
    static const X86Reg aw[] = {X86_RCX, X86_RDX, X86_R8, X86_R9};
    x86_mov_ri(cg_sec, 4, aw[n], imm); // movl $imm, %arg
#else
    static const X86Reg al[] = {X86_RDI, X86_RSI, X86_RDX, X86_RCX, X86_R8};
    x86_mov_ri(cg_sec, 4, al[n], imm); // movl $imm, %arg
#endif
}

// lea the address of frame slot `slot` (a positive offset below rbp) into
// integer arg register N. Used for helper call arguments whose address is a
// compile-time frame offset: avoids holding the address in a caller-saved
// VReg across the call (the helpers clobber r10/r11, rcc's own scratch).
static void emit_bitint_slot_arg(int slot, int n) {
#ifdef ARCH_ARM64
    static const Arm64Reg a64[] = {ARM64_X1, ARM64_X2, ARM64_X3, ARM64_X4};
    if (slot <= 4095) {
        arm64_sub_imm(cg_sec, 1, a64[n], ARM64_X29, slot, 0); // sub xN, x29, #slot
    } else {
        asm_mov_imm_phy(cg_sec, ARM64_X16, 8, slot);
        arm64_sub_reg(cg_sec, 1, a64[n], ARM64_X29, ARM64_X16, ARM64_LSL, 0); // sub xN, x29, x16
    }
#elif defined(_WIN32)
    static const X86Reg aw[] = {X86_RDX, X86_R8, X86_R9};
    asm_lea_rbp(cg_sec, aw[n], 8, slot);
#else
    static const X86Reg al[] = {X86_RSI, X86_RDX, X86_RCX, X86_R8};
    asm_lea_rbp(cg_sec, al[n], 8, slot);
#endif
}

// Move a runtime shift amount (VReg holding an int) into arg register N.
static void emit_bitint_amount_arg(VReg r, int n) {
#ifdef ARCH_ARM64
    static const Arm64Reg a64[] = {ARM64_X0, ARM64_X1, ARM64_X2, ARM64_X3, ARM64_X4};
    asm_mov_phy_reg(cg_sec, a64[n], r, 1); // mov xN, x{r} (amount is an int; low bits)
#elif defined(_WIN32)
    static const X86Reg aw[] = {X86_RCX, X86_RDX, X86_R8, X86_R9};
    x86_mov_rr(cg_sec, 8, aw[n], cg_x86_reg[r]); // movq %r, %arg
#else
    static const X86Reg al[] = {X86_RDI, X86_RSI, X86_RDX, X86_RCX, X86_R8};
    x86_mov_rr(cg_sec, 8, al[n], cg_x86_reg[r]); // movq %r, %arg
#endif
}

// Copy `size` bytes from src slot to dst slot (both VReg addresses).
static void emit_bitint_copy_bytes(VReg dst, VReg src, int size) {
#ifdef ARCH_ARM64
    VReg tmp = alloc_reg();
    for (int off = 0; off + 8 <= size; off += 8) {
        asm_ldr_reg_off(cg_sec, tmp, src, 8, (uint32_t)off); // ldr x{tmp}, [src, #off]
        asm_str_reg_off(cg_sec, tmp, dst, 8, (uint32_t)off); // str x{tmp}, [dst, #off]
    }
    free_reg(tmp);
#else
    for (int off = 0; off + 8 <= size; off += 8) {
        x86_mov_rm(cg_sec, 8, X86_RAX, x86_mem(REG(src), off)); // movq off(src), %rax
        x86_mov_mr(cg_sec, 8, x86_mem(REG(dst), off), X86_RAX); // movq %rax, off(dst)
    }
#endif
}

// Copy a compile-time constant (int64) into a bitint slot, sign/zero-
// extended to the slot's width per `is_unsigned`.
static void emit_bitint_const_fill(VReg dst, long long val, int size, bool is_unsigned) {
#ifdef ARCH_ARM64
    asm_mov_imm_phy(cg_sec, ARM64_X16, 8, val); // mov x16, #val
    asm_str_reg_off_phy(cg_sec, ARM64_X16, dst, 8, 0); // str x16, [dst]
    long long fill = is_unsigned ? 0 : (val < 0 ? -1 : 0);
    for (int off = 8; off + 8 <= size; off += 8) {
        asm_mov_imm_phy(cg_sec, ARM64_X16, 8, fill); // mov x16, #fill
        asm_str_reg_off_phy(cg_sec, ARM64_X16, dst, 8, (uint32_t)off); // str x16, [dst, #off]
    }
#else
    if (val == (int32_t)val) {
        x86_mov_mi(cg_sec, 8, x86_mem(REG(dst), 0), val); // movq $val, (dst)
    } else {
        asm_movabs_phy(cg_sec, X86_RAX, val); // movabsq $val, %rax
        asm_mov_rax_mem(cg_sec, dst); // movq %rax, (dst)
    }
    long long fill = is_unsigned ? 0 : (val < 0 ? -1 : 0);
    for (int off = 8; off + 8 <= size; off += 8) {
        if (fill == (int32_t)fill)
            x86_mov_mi(cg_sec, 8, x86_mem(REG(dst), off), fill);
        else {
            asm_movabs_phy(cg_sec, X86_RAX, fill); // movabsq $fill, %rax
            x86_mov_mr(cg_sec, 8, x86_mem(REG(dst), off), X86_RAX); // movq %rax, off(dst)
        }
    }
#endif
}

// Widen a 64-bit GP value to a fresh bitint slot. The extension above bit
// 63 follows the SOURCE value's signedness (`src_is_unsigned`), never the
// target bitint type's: `(B)(unsigned long long)0xF0...` must zero-extend
// even when B is signed (a cast preserves the source's value; the source
// value is already 64-bit, so its signedness decides the fill).
static VReg widen_to_bitint(VReg val, Type *ty, bool src_is_unsigned) {
    int slot = alloc_wide_slot(ty->size);
#ifdef ARCH_ARM64
    VReg addr = alloc_reg();
    asm_sub_reg_fp_imm(cg_sec, addr, slot); // sub addr, x29, #slot
    asm_str_reg_off(cg_sec, val, addr, 8, 0); // str val, [addr]
    free_reg(val);
    free_reg(addr);
    emit_bitint_bits_arg(64); // w0 = 64 (current width)
    emit_bitint_imm_arg(src_is_unsigned ? 1 : 0, 1); // w1 = is_unsigned
    emit_bitint_slot_arg(slot, 1); // x2 = addr
    emit_bitint_imm_arg(ty->bitint_width, 3); // w3 = N
    emit_bitint_call((char *)"rcc_bitint_sign_ext");
    VReg r = alloc_reg();
    asm_sub_reg_fp_imm(cg_sec, r, slot); // sub r, x29, #slot
    return r;
#else
    VReg addr = alloc_reg();
    asm_lea_rbp_reg(cg_sec, addr, 8, slot); // lea slot, addr
    asm_mov_reg_to_retval(cg_sec, val, 8); // movq val, %rax
    asm_mov_rax_mem(cg_sec, addr); // movq %rax, (addr)
    free_reg(val);
    free_reg(addr);
    emit_bitint_bits_arg(64); // w/edi = 64 (current width)
    emit_bitint_imm_arg(src_is_unsigned ? 1 : 0, 1); // is_unsigned
    emit_bitint_slot_arg(slot, 1); // lp = addr
    emit_bitint_imm_arg(ty->bitint_width, 3); // bits2 = N
    emit_bitint_call((char *)"rcc_bitint_sign_ext");
    VReg r = alloc_reg();
    asm_lea_rbp_reg(cg_sec, r, 8, slot); // lea slot, r
    return r;
#endif
}

// Ensure operand is available as a bitint slot address.
static VReg gen_to_bitint(Node *operand) {
    if (operand->ty && is_wide_bitint(operand->ty))
        return gen_bitint(operand);
    int val = gen(operand);
    Type *ty = operand->ty ? operand->ty : ty_llong;
    // Extend to 64-bit first if narrower
    if (ty->size < 8) {
        if (ty->is_unsigned)
            zero_extend_to(val, ty->size, 8);
        else
            sign_extend_to(val, ty->size, 8);
    }
    Type *bty = bitint_type(ty->size * 8, ty->is_unsigned);
    return widen_to_bitint(val, bty, ty->is_unsigned);
}

static VReg gen_bitint(Node *node) {
    if (!node) return R_NONE;
    if (opt_g) emit_loc(node);

    switch (node->kind) {

    case ND_NUM: {
        int addr = alloc_wide_addr(node->ty->size);
        emit_bitint_const_fill(addr, node->val, node->ty->size, node->ty->is_unsigned);
        return addr;
    }

    case ND_LVAR:
    case ND_MEMBER:
        return gen_addr(node);

    case ND_DEREF:
        // *ptr where ptr → bitint value: the pointer value IS the address
        return gen(node->lhs);

    case ND_ADDR:
        return gen_addr(node->lhs);

    case ND_COMMA:
    case ND_CHAIN: {
        int r = gen(node->lhs);
        if (r >= 0) free_reg(r);
        return gen_bitint(node->rhs);
    }

    case ND_ASSIGN: {
        // Copy rhs slot -> lhs slot (size bytes)
        VReg rhs_a = gen_to_bitint(node->rhs);
        VReg lhs_a = gen_addr(node->lhs);
        emit_bitint_copy_bytes(lhs_a, rhs_a, node->ty->size);
        free_reg(rhs_a);
        return lhs_a;
    }

    case ND_CAST: {
        Type *from = node->lhs->ty;
        Type *to = node->ty;
        // bitint -> bitint: copy + sign/zero extend to target width
        if (from && from->kind == TY_BITINT && to->kind == TY_BITINT) {
            VReg src = gen_bitint(node->lhs);
            int slot = alloc_wide_slot(to->size);
            VReg dst = alloc_reg();
#ifdef ARCH_ARM64
            asm_sub_reg_fp_imm(cg_sec, dst, slot);
#else
            asm_lea_rbp_reg(cg_sec, dst, 8, slot); // lea slot, dst
#endif
            emit_bitint_copy_bytes(dst, src, from->size);
            free_reg(src);
            if (to->bitint_width != from->bitint_width ||
                to->is_unsigned != from->is_unsigned) {
                emit_bitint_bits_arg(from->bitint_width);
                emit_bitint_imm_arg(to->is_unsigned ? 1 : 0, 1);
                emit_bitint_addr_arg(dst, 1);
                emit_bitint_imm_arg(to->bitint_width, 3);
                free_reg(dst);
                emit_bitint_call((char *)"rcc_bitint_sign_ext");
            } else {
                free_reg(dst);
            }
            VReg r = alloc_reg();
#ifdef ARCH_ARM64
            asm_sub_reg_fp_imm(cg_sec, r, slot);
#else
            asm_lea_rbp_reg(cg_sec, r, 8, slot); // re-lea result
#endif
            return r;
        }
        // int -> bitint: widen. The fill above bit 63 follows the source
        // type's signedness (C: casting a value preserves it; a signed
        // source sign-extends, an unsigned source zero-extends), then the
        // result's own type is the target bitint (the slot keeps the
        // fill; the target's signedness governs later reads).
        if (to->kind == TY_BITINT) {
            VReg src = gen(node->lhs);
            Type *f = from ? from : ty_llong;
            bool src_u = f->is_unsigned;
            if (f->size < 8) {
                if (f->is_unsigned)
                    zero_extend_to(src, f->size, 8);
                else
                    sign_extend_to(src, f->size, 8);
            }
            return widen_to_bitint(src, to, src_u);
        }
        // bitint -> smaller int: load low bytes, truncate
        if (from && from->kind == TY_BITINT && is_integer(to)) {
            VReg addr = gen_bitint(node->lhs);
            VReg r = alloc_reg();
#ifdef ARCH_ARM64
            asm_ldr_reg_off(cg_sec, r, addr, 8, 0); // ldr x{r}, [addr]
            free_reg(addr);
            if (to->size < 8) {
                if (to->is_unsigned) {
                    unsigned long long m = (1ULL << (to->size * 8)) - 1;
                    if (m <= 0xffff) {
                        asm_and_imm(cg_sec, r, 4, (int32_t)m); // and w{r}, w{r}, #m
                    } else {
                        emit_mov_imm64(ARM64_X16, m);
                        asm_and_reg_phy(cg_sec, r, ARM64_X16, 4); // and w{r}, w{r}, w16
                    }
                } else
                    asm_movsx(cg_sec, r, r, 4, to->size);
            }
#else
            x86_mov_rm(cg_sec, 8, REG(r), x86_mem(REG(addr), 0)); // movq (addr), r
            free_reg(addr);
            if (to->size < 8) {
                if (to->is_unsigned)
                    asm_movzx(cg_sec, r, r, 4, to->size);
                else
                    asm_movsx(cg_sec, r, r, 4, to->size);
            }
#endif
            return r;
        }
        // bitint -> bool: to_bool helper
        if (from && from->kind == TY_BITINT && to->kind == TY_BOOL) {
            VReg addr = gen_bitint(node->lhs);
            emit_bitint_bits_arg(from->bitint_width);
            emit_bitint_addr_arg(addr, 0);
            free_reg(addr);
            emit_bitint_call((char *)"rcc_bitint_to_bool");
            VReg r = alloc_reg();
            asm_mov_retval(cg_sec, r, 8); // movq %rax, r
            return r;
        }
        // float -> bitint (convert to int64 first, then extend)
        if (to->kind == TY_BITINT && from && is_flonum(from)) {
            VReg r = gen(node->lhs);
#ifdef ARCH_ARM64
            asm_fmov_d0_x(cg_sec, r); // fmov d0, x{r}
            free_reg(r);
            if (to->is_unsigned)
                asm_fcvtzu_x16(cg_sec); // fcvtzu x16, d0
            else
                asm_fcvtzs_x16(cg_sec); // fcvtzs x16, d0
            asm_mov_phy_reg(cg_sec, ARM64_X0, ARM64_X16, 0); // mov x0, x16
            VReg rv = alloc_reg();
            asm_mov_retval(cg_sec, rv, 8); // mov x{rv}, x0
            return widen_to_bitint(rv, to, to->is_unsigned);
#else
            if (to->is_unsigned) {
                int c = ++rcc_label_count;
                asm_movabs_phy(cg_sec, X86_RAX, 0x43e0000000000000LL); // 2^63 as double
                x86_movq_r_xmm(cg_sec, X86_XMM1, X86_RAX);
                x86_comisd(cg_sec, X86_XMM0, X86_XMM1);
                emit_jcc_fixup(cg_sec, X86_B, format(".L.f2bu.%d", c)); // jb
                x86_subsd(cg_sec, X86_XMM0, X86_XMM1);
                x86_cvttsd2si(cg_sec, 8, X86_RAX, X86_XMM0);
                asm_movabs_phy(cg_sec, X86_RDX, 0x8000000000000000LL);
                x86_xor_rr(cg_sec, 8, X86_RAX, X86_RDX);
                emit_jmp_fixup(cg_sec, format(".L.f2bu_d.%d", c));
                cg_def_label(format(".L.f2bu.%d", c));
                x86_cvttsd2si(cg_sec, 8, X86_RAX, X86_XMM0);
                cg_def_label(format(".L.f2bu_d.%d", c));
            } else {
                x86_cvttsd2si(cg_sec, 8, X86_RAX, X86_XMM0); // cvttsd2si %xmm0, %rax
            }
            free_reg(r);
            VReg rv = alloc_reg();
            asm_mov_retval(cg_sec, rv, 8); // movq %rax, rv
            return widen_to_bitint(rv, to, to->is_unsigned);
#endif
        }
        // bitint -> float: convert low 64 bits
        if (from && from->kind == TY_BITINT && to && is_flonum(to)) {
            VReg addr = gen_bitint(node->lhs);
#ifdef ARCH_ARM64
            asm_ldr_reg_off(cg_sec, ARM64_X0, addr, 8, 0); // ldr x0, [addr]
            free_reg(addr);
            if (from->is_unsigned)
                arm64_ucvtf(cg_sec, 1, 1, ARM64_D0, ARM64_X0);
            else
                arm64_scvtf(cg_sec, 1, 1, ARM64_D0, ARM64_X0);
            VReg r = alloc_reg();
            asm_fmov_f2i(cg_sec, r, 0, 1); // fmov x{r}, d0
            return r;
#else
            VReg lo = alloc_reg();
            x86_mov_rm(cg_sec, 8, REG(lo), x86_mem(REG(addr), 0)); // movq (addr), lo
            free_reg(addr);
            // Low 64 bits as signed; matches rcc's int128->float behavior.
            asm_cvtsi2sd(cg_sec, lo, 8); // cvtsi2sd lo, %xmm0
            free_reg(lo);
            VReg r = alloc_reg();
            asm_movq_xmm_r(cg_sec, r, X86_XMM0); // movq %xmm0, r
            return r;
#endif
        }
        // bitint -> complex / other: not supported yet
        error_tok(node->tok, "_BitInt cast to this type not supported");
        return R_NONE;
    }

    case ND_NEG:
    case ND_BITNOT: {
        bool is_neg = (node->kind == ND_NEG);
        VReg src = gen_to_bitint(node->lhs);
        int slot = alloc_wide_slot(node->ty->size);
        VReg dst = alloc_reg();
#ifdef ARCH_ARM64
        asm_sub_reg_fp_imm(cg_sec, dst, slot);
#else
        asm_lea_rbp_reg(cg_sec, dst, 8, slot); // lea slot, dst
#endif
        emit_bitint_copy_bytes(dst, src, node->ty->size);
        free_reg(src);
        free_reg(dst);
        emit_bitint_bits_arg(node->ty->bitint_width);
        emit_bitint_slot_arg(slot, 0); // lp = addr (physical lea, no VReg across call)
        emit_bitint_call((char *)(is_neg ? "rcc_bitint_neg" : "rcc_bitint_bitnot"));
        VReg r = alloc_reg();
#ifdef ARCH_ARM64
        asm_sub_reg_fp_imm(cg_sec, r, slot);
#else
        asm_lea_rbp_reg(cg_sec, r, 8, slot); // re-lea result after call
#endif
        return r;
    }

    case ND_ADD:
    case ND_SUB:
    case ND_MUL:
    case ND_BITAND:
    case ND_BITOR:
    case ND_BITXOR: {
        // in-place helpers: result in rp; stage rhs into dst first
        VReg lhs = gen_to_bitint(node->lhs);
        VReg rhs = gen_to_bitint(node->rhs);
        int slot = alloc_wide_slot(node->ty->size);
        VReg dst = alloc_reg();
#ifdef ARCH_ARM64
        asm_sub_reg_fp_imm(cg_sec, dst, slot);
#else
        asm_lea_rbp_reg(cg_sec, dst, 8, slot); // lea slot, dst
#endif
        emit_bitint_copy_bytes(dst, rhs, node->ty->size);
        free_reg(rhs);
        const char *fn = node->kind == ND_ADD ? "rcc_bitint_add"
            : node->kind == ND_SUB            ? "rcc_bitint_sub"
            : node->kind == ND_MUL            ? "rcc_bitint_mul"
            : node->kind == ND_BITAND         ? "rcc_bitint_bitand"
            : node->kind == ND_BITOR          ? "rcc_bitint_bitor"
                                              : "rcc_bitint_bitxor";
        free_reg(dst);
        emit_bitint_bits_arg(node->ty->bitint_width);
        emit_bitint_addr_arg(lhs, 0); // lp (VReg moved before call; lhs freed after)
        free_reg(lhs);
        emit_bitint_slot_arg(slot, 1); // rp = dst (physical lea)
        emit_bitint_call((char *)fn);
        VReg r = alloc_reg();
#ifdef ARCH_ARM64
        asm_sub_reg_fp_imm(cg_sec, r, slot);
#else
        asm_lea_rbp_reg(cg_sec, r, 8, slot); // re-lea result after call
#endif
        return r;
    }

    case ND_DIV:
    case ND_MOD: {
        // div(bits, flags, lp, rp): flags bit0=unsigned, bit1=div; lp is the
        // dividend (clobbered), rp the divisor; result lands in rp. Both
        // operands must be staged copies.
        VReg lhs = gen_to_bitint(node->lhs);
        VReg rhs = gen_to_bitint(node->rhs);
        int s1 = alloc_wide_slot(node->ty->size);
        int s2 = alloc_wide_slot(node->ty->size);
        VReg t1 = alloc_reg();
#ifdef ARCH_ARM64
        asm_sub_reg_fp_imm(cg_sec, t1, s1);
#else
        asm_lea_rbp_reg(cg_sec, t1, 8, s1); // lea s1, t1
#endif
        emit_bitint_copy_bytes(t1, lhs, node->ty->size);
        free_reg(lhs);
        free_reg(t1);
        VReg t2 = alloc_reg();
#ifdef ARCH_ARM64
        asm_sub_reg_fp_imm(cg_sec, t2, s2);
#else
        asm_lea_rbp_reg(cg_sec, t2, 8, s2); // lea s2, t2
#endif
        emit_bitint_copy_bytes(t2, rhs, node->ty->size);
        free_reg(rhs);
        free_reg(t2);
        int flags = (node->ty->is_unsigned ? 1 : 0) | (node->kind == ND_DIV ? 2 : 0);
        emit_bitint_bits_arg(node->ty->bitint_width);
        emit_bitint_imm_arg(flags, 1);
        emit_bitint_slot_arg(s1, 1); // lp = dividend slot
        emit_bitint_slot_arg(s2, 2); // rp = divisor slot
        emit_bitint_call((char *)"rcc_bitint_div");
        VReg r = alloc_reg();
#ifdef ARCH_ARM64
        asm_sub_reg_fp_imm(cg_sec, r, s2);
#else
        asm_lea_rbp_reg(cg_sec, r, 8, s2); // re-lea result after call
#endif
        return r;
    }

    case ND_SHL:
    case ND_SHR: {
        bool is_shl = (node->kind == ND_SHL);
        VReg lhs = gen_to_bitint(node->lhs);
        int slot = alloc_wide_slot(node->ty->size);
        VReg dst = alloc_reg();
#ifdef ARCH_ARM64
        asm_sub_reg_fp_imm(cg_sec, dst, slot);
#else
        asm_lea_rbp_reg(cg_sec, dst, 8, slot); // lea slot, dst
#endif
        emit_bitint_copy_bytes(dst, lhs, node->ty->size);
        free_reg(lhs);
        free_reg(dst);
        // amount: constant or runtime
        if (node->rhs->kind == ND_NUM) {
            long amt = node->rhs->val;
            emit_bitint_bits_arg(node->ty->bitint_width);
            if (is_shl) {
                emit_bitint_slot_arg(slot, 0); // sp = dst
                emit_bitint_slot_arg(slot, 1); // dp = dst
                emit_bitint_imm_arg((int)amt, 3); // amount
            } else {
                int flags = (int)((amt << 1) | (node->ty->is_unsigned ? 1 : 0));
                emit_bitint_imm_arg(flags, 1);
                emit_bitint_slot_arg(slot, 1); // sp = dst
                emit_bitint_slot_arg(slot, 2); // dp = dst
            }
            emit_bitint_call((char *)(is_shl ? "rcc_bitint_shl" : "rcc_bitint_shr"));
        } else {
            VReg amt = gen(node->rhs);
            if (is_shl) {
                emit_bitint_bits_arg(node->ty->bitint_width);
                emit_bitint_slot_arg(slot, 0); // sp = dst
                emit_bitint_slot_arg(slot, 1); // dp = dst
                emit_bitint_amount_arg(amt, 3);
            } else {
                // flags = (amount << 1) | is_unsigned
                VReg flags = alloc_reg();
#ifdef ARCH_ARM64
                asm_mov_imm(cg_sec, flags, 4, node->ty->is_unsigned ? 1 : 0); // mov flags, #u
                arm64_add_reg(cg_sec, 0, REG(flags), REG(flags), REG(amt), ARM64_LSL, 1); // add flags, flags, amt, lsl #1
#else
                asm_mov_reg_reg(cg_sec, flags, amt, 4); // movl amt, flags
                x86_shl_ri(cg_sec, 4, REG(flags), 1); // shll $1, flags
                x86_or_ri(cg_sec, 4, REG(flags), node->ty->is_unsigned ? 1 : 0); // orl $u, flags
#endif
                free_reg(amt);
                emit_bitint_bits_arg(node->ty->bitint_width);
                emit_bitint_amount_arg(flags, 1);
                emit_bitint_slot_arg(slot, 1); // sp = dst
                emit_bitint_slot_arg(slot, 2); // dp = dst
                free_reg(flags);
            }
            emit_bitint_call((char *)(is_shl ? "rcc_bitint_shl" : "rcc_bitint_shr"));
        }
        VReg r = alloc_reg();
#ifdef ARCH_ARM64
        asm_sub_reg_fp_imm(cg_sec, r, slot);
#else
        asm_lea_rbp_reg(cg_sec, r, 8, slot); // re-lea result after call
#endif
        return r;
    }

    case ND_EQ:
    case ND_NE:
    case ND_LT:
    case ND_LE: {
        // cmp helper: 0=eq, 1=lt, 2=gt. Stage both operands into fresh
        // slots first: gen(rhs) may emit helper calls that clobber lhs's
        // VReg, and the cmp call itself clobbers caller-saved registers.
        VReg lhs = gen_to_bitint(node->lhs);
        int s1 = alloc_wide_slot(node->lhs->ty->size);
        VReg t1 = alloc_reg();
#ifdef ARCH_ARM64
        asm_sub_reg_fp_imm(cg_sec, t1, s1);
#else
        asm_lea_rbp_reg(cg_sec, t1, 8, s1);
#endif
        emit_bitint_copy_bytes(t1, lhs, node->lhs->ty->size);
        free_reg(lhs);
        free_reg(t1);
        VReg rhs = gen_to_bitint(node->rhs);
        int s2 = alloc_wide_slot(node->rhs->ty->size);
        VReg t2 = alloc_reg();
#ifdef ARCH_ARM64
        asm_sub_reg_fp_imm(cg_sec, t2, s2);
#else
        asm_lea_rbp_reg(cg_sec, t2, 8, s2);
#endif
        emit_bitint_copy_bytes(t2, rhs, node->rhs->ty->size);
        free_reg(rhs);
        free_reg(t2);
        emit_bitint_bits_arg(node->lhs->ty->bitint_width);
        emit_bitint_imm_arg(node->lhs->ty->is_unsigned ? 1 : 0, 1);
        emit_bitint_slot_arg(s1, 1); // lp
        emit_bitint_slot_arg(s2, 2); // rp
        emit_bitint_call((char *)"rcc_bitint_cmp");
        VReg r = alloc_reg();
        asm_mov_retval(cg_sec, r, 4); // movl %eax, r (cmp result: 0=eq, 1=lt, 2=gt)
        // map 0/1/2 to the comparison result
        int val = 0;
        bool is_eq_kind = false;
        switch (node->kind) {
        case ND_EQ:
            val = 0;
            is_eq_kind = true;
            break;
        case ND_NE:
            val = 0;
            is_eq_kind = false;
            break;
        case ND_LT:
            val = 1;
            is_eq_kind = true;
            break;
        case ND_LE:
            val = 2;
            is_eq_kind = false;
            break;
        default: break;
        }
        // result = (r == val) for EQ/LT, (r != val) for NE/LE. The cmp
        // helper returns 0 for equal, so EQ -> r==0, NE -> r!=0,
        // LT -> r==1, LE -> r!=2 (i.e. r==0 || r==1, the <= case).
#ifdef ARCH_ARM64
        asm_cmp_imm(cg_sec, r, 4, val); // cmp w{r}, #val
        asm_cset(cg_sec, r, is_eq_kind ? ARM64_EQ : ARM64_NE);
#else
        asm_cmp_imm(cg_sec, r, 4, val); // cmpl $val, r
        asm_setcc(cg_sec, X86_RAX, is_eq_kind ? X86_E : X86_NE);
        asm_movzx_phys(cg_sec, r, X86_RAX, 4, 1); // movzbl %al, r
#endif
        return r;
    }

    default:
        // Anything else with a wide-bitint type (VA_ARG, STMT_EXPR, ...):
        // not supported yet; clear error rather than silently truncating.
        error_tok(node->tok, "_BitInt(%d): operation not supported",
                  node->ty ? node->ty->bitint_width : 0);
        return R_NONE;
    }
}

// Generate code for a given node.
VReg gen(Node *node) {
    if (!node) return R_NONE;
    if (node->tok && node->tok->lineno > 0) cg_dbg_line = node->tok->lineno;

    if (opt_g)
        emit_loc(node);

    // Dispatch int128 nodes to gen_int128()
    if (node->ty && node->ty->kind == TY_INT128)
        return gen_int128(node);
    // Dispatch wide _BitInt (N > 64) nodes to gen_bitint()
    if (node->ty && is_wide_bitint(node->ty))
        return gen_bitint(node);
    // Dispatch _Decimal32/64/128 nodes to gen_decimal() (BID runtime calls)
    if (node->ty && is_decimal(node->ty))
        return gen_decimal(node);
    // Decimal comparisons: the ND_EQ/NE/LT/LE node's own type is int
    // (comparison result), so dispatch on the operand types.
    if ((node->kind == ND_EQ || node->kind == ND_NE || node->kind == ND_LT || node->kind == ND_LE) &&
        ((node->lhs && node->lhs->ty && is_decimal(node->lhs->ty)) ||
         (node->rhs && node->rhs->ty && is_decimal(node->rhs->ty))))
        return gen_decimal(node);

    // Element-wise vector ops (__attribute__((vector_size))) → packed SSE/NEON.
    // Only arithmetic/compare/unary route here; lvalue/assign/call/init on a
    // vector already work via the TY_STRUCT aggregate paths below.
    if (node->ty && node->ty->is_vector) {
        switch (node->kind) {
        case ND_ADD:
        case ND_SUB:
        case ND_MUL:
        case ND_DIV:
        case ND_MOD:
        case ND_BITAND:
        case ND_BITOR:
        case ND_BITXOR:
        case ND_EQ:
        case ND_NE:
        case ND_LT:
        case ND_LE:
        case ND_NEG:
        case ND_BITNOT:
#ifndef ARCH_ARM64
            if (node->ty->size == 64)
                return gen_vector64_x86(node);
            if (node->ty->size == 32)
                return gen_vector32_x86(node);
#endif
            return gen_vector(node);
        default:
            break;
        }
    }
    // __builtin_ia32_* SIMD intrinsics (x86) / clean error (ARM64).
    if (node->kind == ND_FUNCALL && node->funcname &&
        !strncmp(node->funcname, "__builtin_ia32_", 15)) {
#ifdef ARCH_ARM64
        // ARM64 keeps the original packed-SSE/NEON generators for the few
        // builtins they covered; everything else is a clean error.
        if (!strcmp(node->funcname, "__builtin_ia32_sqrtps") ||
            !strcmp(node->funcname, "__builtin_ia32_sqrtss") ||
            !strcmp(node->funcname, "__builtin_ia32_rsqrtps") ||
            !strcmp(node->funcname, "__builtin_ia32_sqrtpd") ||
            !strcmp(node->funcname, "__builtin_ia32_sqrtsd"))
            return gen_vector_unary_builtin(node);
        if (!strcmp(node->funcname, "__builtin_ia32_pshufb128"))
            return gen_vector_binary_builtin(node);
        error_tok(node->tok, "__builtin_ia32_%s: not implemented on this target",
                  node->funcname + 15);
        return R_NONE;
#else
        return gen_ia32_builtin(node);
#endif
    }
    // Cast from int128 to a smaller type: extract value from 16-byte slot
    if (node->kind == ND_CAST && node->lhs && node->lhs->ty &&
        node->lhs->ty->kind == TY_INT128 && node->ty && node->ty->kind != TY_INT128) {
        VReg addr = gen_int128(node->lhs);
        VReg r = alloc_reg();
#ifdef ARCH_ARM64
        if (is_flonum(node->ty)) {
            // int128 → float: load lo word, convert
            asm_ldr_reg_off(cg_sec, r, addr, 8, 0);
            free_reg(addr);
            asm_fmov_d0_x(cg_sec, r); // fmov d0, x{r}
            arm64_scvtf(cg_sec, 1, 1, ARM64_D0, ARM64_D0); // scvtf d0, d0
            asm_fmov_x_d0(cg_sec, r); // fmov x{r}, d0
        } else {
            asm_ldr_reg_off(cg_sec, r, addr, 8, 0);
            free_reg(addr);
            if (node->ty->size < 8) {
                if (!node->ty->is_unsigned) sign_extend_to(r, node->ty->size, 8);
                else
                    zero_extend_to(r, node->ty->size, 8);
            }
        }
#else
        asm_ldr_reg_off(cg_sec, r, addr, 8, 0); // movq (%s), r
        free_reg(addr);
        if (node->ty->size < 8) {
            if (!node->ty->is_unsigned) sign_extend_to(r, node->ty->size, 8);
            else
                zero_extend_to(r, node->ty->size, 8);
        }
#endif
        return r;
    }
    // Cast from wide _BitInt to a smaller type: gen_bitint handles it
    if (node->kind == ND_CAST && node->lhs && node->lhs->ty && is_wide_bitint(node->lhs->ty))
        return gen_bitint(node);
    // Comparisons where the operand is int128 (result is int)
    if (node->lhs && node->lhs->ty && node->lhs->ty->kind == TY_INT128) {
        switch (node->kind) {
        case ND_EQ:
        case ND_NE:
        case ND_LT:
        case ND_LE:
            return gen_int128(node);
        default: break;
        }
    }
    // Comparisons where an operand is a wide _BitInt (result is int)
    if (node->lhs && node->lhs->ty && is_wide_bitint(node->lhs->ty)) {
        switch (node->kind) {
        case ND_EQ:
        case ND_NE:
        case ND_LT:
        case ND_LE:
            return gen_bitint(node);
        default: break;
        }
    }
    if (node->rhs && node->rhs->ty && is_wide_bitint(node->rhs->ty) && node->lhs && node->lhs->ty &&
        !is_wide_bitint(node->lhs->ty)) {
        switch (node->kind) {
        case ND_EQ:
        case ND_NE:
        case ND_LT:
        case ND_LE:
            return gen_bitint(node);
        default: break;
        }
    }


    // codeql[cpp/long-switch]: central AST-node-kind dispatch; splitting cases into helpers is a large, purely-cosmetic refactor of core compiler internals, not attempted here.
    switch (node->kind) {
    case ND_NUM: {
        VReg r = alloc_reg();
#ifdef ARCH_ARM64
        uint64_t v = (uint64_t)(long long)node->val;
        // Use x register for 64-bit immediates, w for 32-bit (stops at lsl #16)
        if (op_size(node->ty) == 4) {
            asm_mov_imm(cg_sec, r, 4, v & 0xffff); // mov $v & 0xffff, rr
            v >>= 16;
            if (v) {
                asm_movk(cg_sec, REG(r), 0, (uint16_t)(v), 16); // movk w{r}, #v, lsl #16
            }
        } else {
            asm_mov_imm(cg_sec, r, 8, v & 0xffff); // mov $v & 0xffff, rr
            v >>= 16;
            int shift = 16;
            while (v) {
                asm_movk(cg_sec, REG(r), 1, (uint16_t)(v & 0xffff), shift); // movk x{r}, #v, lsl #shift
                v >>= 16;
                shift += 16;
            }
        }
#else
        if (node->val == (int32_t)node->val) {
            asm_mov_imm(cg_sec, r, op_size(node->ty), (long long)node->val); // mov $(long long)node->val, rr
        } else {
            asm_mov_imm(cg_sec, r, 8, (long long)node->val); // mov $(long long)node->val, rr
        }
#endif
        return r;
    }
    case ND_FNUM: {
        VReg r = alloc_reg();
#ifdef ARCH_ARM64
        int fsize = (node->ty && node->ty->kind == TY_FLOAT) ? 4 : 8;
#else
        int fsize = 8; // Always store as double; movsd loads 8 bytes
#endif
        int id = add_float_literal(node->fval, fsize);
#ifdef ARCH_ARM64
        emit_adrp_add(r, format(".LF%d", id));
        if (fsize == 4) {
            asm_ldr_fp(cg_sec, 0, r, 4); // ldr s0, [x{r}]
            asm_fcvt(cg_sec, 1, 0, 0, 0); // fcvt d0, s0 (promote to double for GP storage)
        } else {
            asm_ldr_fp(cg_sec, 0, r, 8); // ldr d0, [x{r}]
        }
        asm_fmov_f2i(cg_sec, r, 0, 1); // fmov x{r}, d0 (always 64-bit)
#else
        asm_movsd_rip_xmm(cg_sec, format(".LF%d", id)); // movsd .LF%d(%rip), %%xmm0
        asm_movq_xmm_r(cg_sec, r, X86_XMM0); // movq %%xmm0, %s
#endif
        return r;
    }
    case ND_LVAR: {
        VReg r = alloc_reg();
        if (opt_W) reg_owner[r] = node->var->name;
#ifndef ARCH_ARM64
        __attribute__((unused)) char *label = var_label(node->var);
#endif
        if (node->var->ty->kind == TY_VLA || ((node->var->ty->kind == TY_STRUCT || node->var->ty->kind == TY_UNION) && node->var->ty->vla_len_expr)) {
            if (node->var->is_local) {
                if (node->chain_depth > 0) {
                    // VLA locals are stored as fat pointers: the value
                    // needed here is the pointer itself, read from
                    // ancestor_frame-(offset-8) - see gen_addr's ND_LVAR
                    // TY_VLA branch for the matching address-of case.
                    VReg base = alloc_reg();
                    gen_chain_walk_into(base, node->chain_depth);
#ifdef ARCH_ARM64
                    arm64_load_from_reg_minus(REG(base), node->var->offset - 8, REG(r));
#else
                    x86_mov_rm(cg_sec, 8, REG(r), x86_mem(REG(base), -(node->var->offset - 8))); // mov -(offset-8)(%rN), r
#endif
                    free_reg(base);
                } else {
#ifdef ARCH_ARM64
                    arm64_load_from_fp_minus(node->var->offset - 8, REG(r));
#else
                    asm_mov_rbp_reg(cg_sec, r, 8, node->var->offset - 8); // mov [rbp-8], rr
#endif
                }
            } else {
#ifdef ARCH_ARM64
                if (node->var->is_tls)
                    emit_tls_addr(r, node->var);
                else if (var_needs_got(node->var))
                    emit_adrp_got(r, asm_sym_name(var_sym_label(node->var)));
                else
                    emit_adrp_add(r, asm_sym_name(var_sym_label(node->var)));
                asm_ldr_reg_off(cg_sec, r, r, 8, 0); // ldr x{r}, [x{r}]
#else
#ifdef _WIN32
                if (var_needs_refptr(node->var)) {
                    asm_mov_refptr_rip_reg(cg_sec, r, var_sym_label(node->var));
                    asm_mov_mem_reg(cg_sec, r, r, 8); // movq (%r), rr
                } else
#endif
                    if (var_needs_got(node->var)) {
                    asm_mov_got_rip_reg(cg_sec, r, var_sym_label(node->var)); // mov sym@GOTPCREL(%rip), r
                    asm_mov_mem_reg(cg_sec, r, r, 8); // movq (%r), rr
                } else {
                    asm_lea_rip_reg(cg_sec, r, var_sym_label(node->var)); // lea rip, rr
                    asm_mov_mem_reg(cg_sec, r, r, 8); // movq (%r), rr
                }
#endif
            }
        } else if (node->var->ty->kind == TY_ARRAY || node->var->ty->kind == TY_COMPLEX) {
            if (node->var->is_local)
#ifdef ARCH_ARM64
                if (node->var->offset <= 4095)
                    asm_sub_reg_fp_imm(cg_sec, r, node->var->offset); // sub r, x29, #node->var->offset
                else {
                    int v = node->var->offset;
                    asm_mov_imm(cg_sec, r, 8, v & 0xffff); // mov r, #v
                    v >>= 16;
                    int s = 16;
                    while (v) {
                        asm_movk(cg_sec, REG(r), 1, (uint16_t)(v & 0xffff), s); // movk r, #v, lsl #s
                        v >>= 16;
                        s += 16;
                    }
                    asm_sub_reg_fp_reg(cg_sec, r, r, 8); // sub r, x29, r
                }
#else
                asm_lea_rbp_reg(cg_sec, r, 8, node->var->offset); // lea [rbp-8], rr
#endif
            else {
#ifdef ARCH_ARM64
                // Array/complex decays to its ADDRESS, not a loaded value.
                // emit_adrp_add already yields the address (via GOT on Darwin,
                // adrp+add on ELF); emit_adrp_got would add a spurious deref
                // (returning the first bytes of the array as a pointer).
                if (node->var->is_tls)
                    emit_tls_addr(r, node->var);
                else
                    emit_adrp_add(r, asm_sym_name(var_sym_label(node->var)));
#else
                if (node->var->is_tls) {
#ifdef _WIN32
                    emit_emutls_addr(r, var_label(node->var));
#else
                    VReg base = alloc_reg();
                    asm_mov_fs0_reg(cg_sec, base);
                    asm_lea_tpoff_base_reg(cg_sec, r, base, var_sym_label(node->var));
                    free_reg(base);
#endif
                } else
#ifdef _WIN32
                    if (var_needs_refptr(node->var))
                    asm_mov_refptr_rip_reg(cg_sec, r, var_sym_label(node->var));
                else
#endif
                    if (var_needs_got(node->var))
                    asm_mov_got_rip_reg(cg_sec, r, var_sym_label(node->var));
                else
                    asm_lea_rip_reg(cg_sec, r, var_sym_label(node->var));
#endif
            }
        } else if (!node->var->is_local && node->var->is_function) {
            if (node->var->is_nested_fn) {
                // GNU nested function used as a *value* (not a direct
                // call) - e.g. passed as a function pointer, stored,
                // returned. Needs a runtime trampoline: allocate a fresh
                // TRAMPOLINE_SIZE scratch slot in THIS (enclosing)
                // function's own frame (see fn_trampoline_off/total in
                // rcc.h/above - same two-pass-stable scheme as
                // fn_struct_ret_off/total) and write the stub bytes into
                // it (fixed template bytes as compile-time immediates,
                // plus the two runtime-computed fields: static-chain
                // value and the nested function's real code address).
                // For the resulting stack-resident code to be
                // executable, cg_obj->uses_trampoline (checked below)
                // marks .note.GNU-stack executable, which the linker
                // turns into an RWE PT_GNU_STACK header - the kernel then
                // maps the *entire* stack executable at load time. This
                // is deliberately NOT done via libgcc's
                // __enable_execute_stack (a runtime mprotect on just this
                // page): that call is unreliable across real
                // environments (verified: it's a silent no-op under this
                // project's own sandbox, confirmed independently of rcc
                // by an __enable_execute_stack call from plain gcc-built
                // code) - the load-time GNU_STACK marking, which is what
                // GCC itself relies on for its own nested functions, has
                // no such runtime dependency.
                cg_obj->uses_trampoline = true;
                fn_trampoline_off += TRAMPOLINE_SIZE;
                if (fn_trampoline_off > fn_trampoline_total)
                    fn_trampoline_total = fn_trampoline_off;
                int off = current_fn_stack_size + fn_struct_ret_total + fn_trampoline_off;
#ifdef ARCH_ARM64
                // Template (verified via `as`/objdump against this exact
                // instruction sequence - see the trampoline design note):
                //   ldr x17, #16   ; 58000091  (loads from offset+16 = func addr)
                //   ldr x18, #16   ; 580000b2  (loads from offset+20 rel. to itself = offset+24 = chain)
                //   br  x17        ; d61f0220
                //   nop            ; d503201f  (padding, keeps data 8-byte aligned)
                //   <8 bytes: nested function's code address>
                //   <8 bytes: static-chain value>
                VReg base = alloc_reg();
                asm_sub_reg_fp_imm(cg_sec, base, off); // base = x29 - off
                VReg tmp = alloc_reg();
                emit_mov_imm64(REG(tmp), 0x58000091u);
                arm64_str_uoff(cg_sec, 2, REG(tmp), REG(base), 0);
                emit_mov_imm64(REG(tmp), 0x580000b2u);
                arm64_str_uoff(cg_sec, 2, REG(tmp), REG(base), 1);
                emit_mov_imm64(REG(tmp), 0xd61f0220u);
                arm64_str_uoff(cg_sec, 2, REG(tmp), REG(base), 2);
                emit_mov_imm64(REG(tmp), 0xd503201fu);
                arm64_str_uoff(cg_sec, 2, REG(tmp), REG(base), 3);
                emit_adrp_add(tmp, asm_sym_name(var_sym_label(node->var)));
                arm64_str_uoff(cg_sec, 3, REG(tmp), REG(base), 2); // func addr @ +16
                if (node->chain_depth == 0)
                    arm64_orr_reg(cg_sec, 1, REG(tmp), ARM64_XZR, ARM64_X29, ARM64_LSL, 0);
                else
                    gen_chain_walk_into(tmp, node->chain_depth);
                arm64_str_uoff(cg_sec, 3, REG(tmp), REG(base), 3); // chain value @ +24
                free_reg(tmp);
                free_reg(base);
                asm_sub_reg_fp_imm(cg_sec, r, off); // r = trampoline address
#else
                // x86-64 SysV/Win64 trampoline (classic GCC layout, 22 of
                // TRAMPOLINE_SIZE's 32 bytes used):
                //   49 ba <8-byte chain value>   ; mov r10, imm64
                //   48 b8 <8-byte func address>  ; mov rax, imm64
                //   ff e0                        ; jmp rax
                VReg chain_reg = alloc_reg();
                if (node->chain_depth == 0)
                    x86_mov_rr(cg_sec, 8, REG(chain_reg), X86_RBP);
                else
                    gen_chain_walk_into(chain_reg, node->chain_depth);
                VReg func_reg = alloc_reg();
                asm_lea_rip_reg(cg_sec, func_reg, var_sym_label(node->var));
                x86_mov_mi(cg_sec, 2, x86_mem(X86_RBP, -off), 0xba49);
                x86_mov_mr(cg_sec, 8, x86_mem(X86_RBP, -off + 2), REG(chain_reg));
                x86_mov_mi(cg_sec, 2, x86_mem(X86_RBP, -off + 10), 0xb848);
                x86_mov_mr(cg_sec, 8, x86_mem(X86_RBP, -off + 12), REG(func_reg));
                x86_mov_mi(cg_sec, 2, x86_mem(X86_RBP, -off + 20), 0xe0ff);
                free_reg(chain_reg);
                free_reg(func_reg);
                asm_lea_rbp_reg(cg_sec, r, 8, off); // r = trampoline address
#endif
                return r;
            }
            if (node->var->is_weak)
                cg_weak_declare(asm_sym_name(var_sym_label(node->var)));
#ifdef ARCH_ARM64
            if (node->var->is_weak)
                emit_adrp_got(r, asm_sym_name(var_sym_label(node->var)));
            else
                emit_adrp_add(r, asm_sym_name(var_sym_label(node->var)));
#else
            if (node->var->is_weak || var_needs_got(node->var))
                asm_mov_got_rip_reg(cg_sec, r, var_sym_label(node->var)); // mov sym@GOTPCREL(%rip), r
            else
                asm_lea_rip_reg(cg_sec, r, var_sym_label(node->var)); // lea rip, rr
#endif
        } else if (is_flonum(node->var->ty)) {
            {
                if (node->var->is_local) {
                    if (node->var->ty->size == 4) {
#ifdef ARCH_ARM64
                        if (node->var->offset <= 4095) {
                            asm_sub_fp_imm(cg_sec, ARM64_X17, node->var->offset);
                            arm64_ldr_fp(cg_sec, 2, ARM64_S0, ARM64_X17, 0); // ldr s0, [x17]
                        } else {
                            emit_mov_imm64(ARM64_X17, (uint64_t)node->var->offset);
                            asm_sub_fp_reg(cg_sec, ARM64_X17, ARM64_X17);
                            arm64_ldr_fp(cg_sec, 2, ARM64_S0, ARM64_X17, 0); // ldr s0, [x17]
                        }
                        asm_fcvt(cg_sec, 1, 0, 0, 0); // fcvt d0, s0
#else
                        asm_movss_rm_rbp(cg_sec, 0, node->var->offset); // movss -off(%rbp), %xmm0
                        asm_cvtss2sd(cg_sec); // cvtss2sd %xmm0, %xmm0
#endif
                    } else {
#ifdef ARCH_ARM64
                        if (node->var->offset <= 4095) {
                            asm_sub_fp_imm(cg_sec, ARM64_X17, node->var->offset);
                            arm64_ldr_fp(cg_sec, 3, ARM64_D0, ARM64_X17, 0); // ldr d0, [x17]
                        } else {
                            emit_mov_imm64(ARM64_X17, (uint64_t)node->var->offset);
                            asm_sub_fp_reg(cg_sec, ARM64_X17, ARM64_X17);
                            arm64_ldr_fp(cg_sec, 3, ARM64_D0, ARM64_X17, 0); // ldr d0, [x17]
                        }
#else
                        asm_movsd_rm_rbp(cg_sec, 0, node->var->offset); // movsd -off(%rbp), %xmm0
#endif
                    }
                } else {
                    if (node->var->ty->size == 4) {
#ifdef ARCH_ARM64
                        if (node->var->is_tls)
                            emit_tls_addr(r, node->var);
                        else if (var_needs_got(node->var))
                            emit_adrp_got(r, asm_sym_name(var_sym_label(node->var)));
                        else
                            emit_adrp_add(r, asm_sym_name(var_sym_label(node->var)));
                        asm_ldr_fp(cg_sec, 0, r, 4); // ldr s0, [x{r}]
                        asm_fcvt(cg_sec, 1, 0, 0, 0); // fcvt d0, s0
#else
#ifdef _WIN32
                        if (var_needs_refptr(node->var)) {
                            asm_mov_refptr_rip_reg(cg_sec, r, var_sym_label(node->var));
                            asm_ldr_fp(cg_sec, 0, r, 4); // movss (r), %xmm0
                        } else
#endif
                            if (var_needs_got(node->var)) {
                            asm_mov_got_rip_reg(cg_sec, r, var_sym_label(node->var)); // mov sym@GOTPCREL(%rip), r
                            asm_ldr_fp(cg_sec, 0, r, 4); // movss (r), %xmm0
                        } else {
                            asm_lea_rip_reg(cg_sec, r, var_sym_label(node->var)); // leaq label(%rip), r
                            asm_ldr_fp(cg_sec, 0, r, 4); // movss (r), %xmm0
                        }
                        asm_cvtss2sd(cg_sec); // cvtss2sd %xmm0, %xmm0
#endif
                    } else {
#ifdef ARCH_ARM64
                        if (node->var->is_tls)
                            emit_tls_addr(r, node->var);
                        else if (var_needs_got(node->var))
                            emit_adrp_got(r, asm_sym_name(var_sym_label(node->var)));
                        else
                            emit_adrp_add(r, asm_sym_name(var_sym_label(node->var)));
                        asm_ldr_fp(cg_sec, 0, r, 8); // ldr d0, [x{r}]
#else
#ifdef _WIN32
                        if (var_needs_refptr(node->var)) {
                            asm_mov_refptr_rip_reg(cg_sec, r, var_sym_label(node->var));
                            asm_ldr_fp(cg_sec, 0, r, 8); // movsd (r), %xmm0
                        } else
#endif
                            if (var_needs_got(node->var)) {
                            asm_mov_got_rip_reg(cg_sec, r, var_sym_label(node->var)); // mov sym@GOTPCREL(%rip), r
                            asm_ldr_fp(cg_sec, 0, r, 8); // movsd (r), %xmm0
                        } else {
                            asm_lea_rip_reg(cg_sec, r, var_sym_label(node->var)); // leaq label(%rip), r
                            asm_ldr_fp(cg_sec, 0, r, 8); // movsd (r), %xmm0
                        }
#endif
                    }
                }
#ifdef ARCH_ARM64
                asm_fmov_f2i(cg_sec, r, 0, 1); // fmov x{r}, d0
#else
                asm_movq_xmm_r(cg_sec, r, X86_XMM0); // movq r, xmm0
#endif
            }
        } else {
            if (node->var->is_local)
#ifdef ARCH_ARM64
                if (node->chain_depth > 0) {
                    VReg base = alloc_reg();
                    gen_chain_walk_into(base, node->chain_depth);
                    emit_load(node->ty, r, base, -node->var->offset);
                    free_reg(base);
                } else
                    emit_load(node->ty, r, ARM64_BASE_FP, -node->var->offset);
#else
                if (node->chain_depth > 0) {
                    VReg base = alloc_reg();
                    gen_chain_walk_into(base, node->chain_depth);
                    emit_load(node->ty, r, base, -node->var->offset);
                    free_reg(base);
                } else
                    emit_load(node->ty, r, X86_BASE_RBP, node->var->offset);
#endif
            else if (node->var->is_global_reg && gen_global_reg_read(r, node->var)) {
                // Handled: r now holds the pinned hardware register's value.
            } else {
#ifdef ARCH_ARM64
                // Global variable: load address via ADRP+ADD, then deref
                int ta = alloc_reg();
                if (node->var->is_tls)
                    emit_tls_addr(ta, node->var);
                else if (var_needs_got(node->var))
                    emit_adrp_got(ta, asm_sym_name(var_sym_label(node->var)));
                else
                    emit_adrp_add(ta, asm_sym_name(var_sym_label(node->var)));
                emit_load(node->ty, r, ta, 0);
                free_reg(ta);
#else
                if (node->var->is_tls) {
#ifdef _WIN32
                    emit_emutls_addr(r, var_label(node->var));
#else
                    VReg base = alloc_reg();
                    asm_mov_fs0_reg(cg_sec, base);
                    asm_lea_tpoff_base_reg(cg_sec, r, base, var_sym_label(node->var));
                    free_reg(base);
#endif
                    emit_load(node->ty, r, r, 0);
                } else
#ifdef _WIN32
                    if (var_needs_refptr(node->var)) {
                    asm_mov_refptr_rip_reg(cg_sec, r, var_sym_label(node->var));
                    emit_load(node->ty, r, r, 0);
                } else
#endif
                    if (var_needs_got(node->var)) {
                    asm_mov_got_rip_reg(cg_sec, r, var_sym_label(node->var)); // mov sym@GOTPCREL(%rip), r
                    emit_load(node->ty, r, r, 0);
                } else {
                    asm_lea_rip_reg(cg_sec, r, var_sym_label(node->var)); // lea rip, rr
                    emit_load(node->ty, r, r, 0);
                }
#endif
            }
#ifdef ARCH_ARM64
            // emit_load already handles sign/zero-extension for narrow types
#endif
        }
        return r;
    }
    case ND_ASSIGN: {
        if ((node->lhs->ty->kind == TY_ARRAY || node->lhs->ty->kind == TY_STRUCT || node->lhs->ty->kind == TY_UNION || node->lhs->ty->kind == TY_COMPLEX) && !(node->lhs->ty->is_vector && (node->lhs->ty->size == 16 || node->lhs->ty->size == 8))) {
            int c = ++rcc_label_count;
            bool lhs_vla_struct = (node->lhs->ty->kind == TY_STRUCT || node->lhs->ty->kind == TY_UNION) && node->lhs->ty->vla_len_expr;
            VReg dst = lhs_vla_struct ? gen(node->lhs) : gen_addr(node->lhs);
            VReg src;
            if (node->rhs->kind == ND_FUNCALL && node->rhs->ty &&
                (node->rhs->ty->kind == TY_STRUCT || node->rhs->ty->kind == TY_UNION || node->rhs->ty->kind == TY_COMPLEX)) {
#ifndef ARCH_ARM64
                int sav = -1;
                if (dst == 0 || dst == 1) {
                    sav = dst;
                    asm_mov_reg_rbp(cg_sec, dst, 8, spill_offset(dst)); // movq dst, -spill_offset(dst)(%rbp)
                }
#endif
                gen_funcall(node->rhs, dst);
#ifndef ARCH_ARM64
                if (sav >= 0)
                    asm_mov_rbp_reg(cg_sec, sav, 8, spill_offset(sav)); // movq -spill_offset(sav)(%rbp), sav
#endif
                // If the call's complex return type and the destination's
                // complex type have differently-sized real-floating bases
                // (e.g. _Complex float -> _Complex double), convert the
                // components in place before anything else uses dst.
                if (node->ty->kind == TY_COMPLEX && node->rhs->ty->kind == TY_COMPLEX &&
                    is_flonum(node->ty->base) && is_flonum(node->rhs->ty->base) &&
                    node->ty->base->kind != TY_LDOUBLE && node->rhs->ty->base->kind != TY_LDOUBLE &&
                    node->ty->base->size != node->rhs->ty->base->size) {
                    emit_complex_convert_float(dst, dst, node->rhs->ty, node->ty);
                }
                // Mixed int↔float complex bases (e.g. _Complex int → _Complex double)
                if (node->ty->kind == TY_COMPLEX && node->rhs->ty->kind == TY_COMPLEX &&
                    ((is_integer(node->ty->base) && is_flonum(node->rhs->ty->base)) ||
                     (is_flonum(node->ty->base) && is_integer(node->rhs->ty->base)))) {
                    emit_complex_convert_mixed(dst, dst, node->rhs->ty, node->ty);
                }
                // For complex return values used as a condition (e.g. if (c = f())),
                // test whether the value is non-zero.
                if (node->ty->kind == TY_COMPLEX) {
                    int base_sz = node->ty->base->size;
                    int rt = alloc_reg();
#ifdef ARCH_ARM64
                    asm_ldr_reg_off(cg_sec, rt, dst, 8, 0);
                    if (node->ty->size > base_sz) {
                        int rt2 = alloc_reg();
                        asm_ldr_reg_off(cg_sec, rt2, dst, 8, base_sz); // ldr rt2, [dst, #base_sz]
                        arm64_orr_reg(cg_sec, 1, REG(rt), REG(rt), REG(rt2), ARM64_LSL, 0); // orr rt, rt, rt2
                        free_reg(rt2);
                    }
                    asm_cmp_zero(cg_sec, rt, 8);
                    asm_cset(cg_sec, rt, ARM64_NE);
#else
                    // The whole complex value fits in 8 bytes (_Complex float)
                    // or two 8-byte halves (_Complex double): load (and OR
                    // together) the raw bits and test against zero.
                    x86_mov_rm(cg_sec, 8, REG(rt), x86_mem(REG(dst), 0)); // movq (dst), rt
                    if (node->ty->size > 8)
                        x86_or_rm(cg_sec, 8, REG(rt), x86_mem(REG(dst), base_sz)); // orq base_sz(dst), rt
                    asm_cmp_zero(cg_sec, rt, 8); // cmp $0, rt
                    asm_setcc(cg_sec, X86_RAX, X86_NE); // setne %al
                    asm_movzx_phys(cg_sec, rt, X86_RAX, 4, 1); // movzbl %al, rt
#endif
                    free_reg(dst);
                    return rt;
                }
                return dst;
            }
            // String literal → char array: limit copy to actual string size, zero-fill rest
            if (node->rhs->kind == ND_STR && node->lhs->ty->kind == TY_ARRAY) {
                src = gen(node->rhs); // loads address of string
                int str_len = 0;
                for (StrLit *s = all_strs; s; s = s->next) {
                    if (s->id != node->rhs->str_id) continue;
                    if (s->prefix != 0 && s->prefix != '8') {
                        // Wide string: count Unicode chars and multiply by elem_size
                        int count = 0;
                        char *p = s->str;
                        while (*p) {
                            char *next;
                            decode_utf8(&next, p);
                            if (next == p) {
                                p++;
                                continue;
                            }
                            p = next;
                            count++;
                        }
                        str_len = (count + 1) * s->elem_size;
                    } else {
                        str_len = s->len + s->elem_size;
                    }
                    break;
                }
                int lhs_size = node->lhs->ty->size;
                int copy_len = str_len < lhs_size ? str_len : lhs_size;
                if (copy_len > 0)
#ifdef ARCH_ARM64
                {
                    emit_mov_imm64(ARM64_X9, copy_len); // mov x9, #copy_len
                    cg_def_label(format(".L.strcpy.%d", c)); // .L.copy.%d:
                    arm64_subs_imm(cg_sec, 1, ARM64_XZR, ARM64_X9, 0, 0); // cmp x9, #0
                    {
                        size_t o = asm_jcc_label(cg_sec, ARM64_EQ); // b.eq .L.strcpy_end.c
                        asm_fixup_add(cg_sec, o, format(".L.strcpy_end.%d", c), 1);
                    }
                    arm64_sub_imm(cg_sec, 1, ARM64_X9, ARM64_X9, 1, 0); // sub x9, x9, #1
                    // ldrb w16, [x{src}, x9]
                    asm_ldrb_w16_x9(cg_sec, src); // ldrb w16, [x{src}, x9]
                    // strb w16, [x{dst}, x9]
                    asm_strb_w16_x9(cg_sec, dst); // strb w16, [x{dst}, x9]
                    {
                        size_t o = asm_jmp_label(cg_sec); // b .L.strcpy.c
                        asm_fixup_add(cg_sec, o, format(".L.strcpy.%d", c), 0);
                    }
                    cg_def_label(format(".L.strcpy_end.%d", c));
                }
                if (copy_len < lhs_size) {
                    // Zero dst[copy_len .. lhs_size-1]; count x12 from lhs_size down to copy_len
                    int c2 = ++rcc_label_count;
                    emit_mov_imm64(ARM64_X9, lhs_size); // mov x9, #lhs_size
                    cg_def_label(format(".L.strzero.%d", c2)); // .L.copy_end.%d:
                    if (copy_len >= 0 && copy_len <= 4095)
                        arm64_subs_imm(cg_sec, 1, ARM64_XZR, ARM64_X9, copy_len, 0); // cmp x9, #copy_len
                    else {
                        emit_mov_imm64(ARM64_X16, (uint64_t)copy_len); // mov x16, #copy_len
                        /* TODO
                        if (copy_len >> 16)
                          arm64_movk(cg_sec, ARM64_X16, 1, (copy_len >> 16) & 0xffff, 16); // movk x16, #(copy_len>>16), lsl #16
                        */
                        arm64_subs_reg(cg_sec, 1, ARM64_XZR, ARM64_X9, ARM64_X16, ARM64_LSL, 0); // cmp x9, x16
                    }
                    {
                        size_t o = asm_jcc_label(cg_sec, ARM64_EQ); // b.eq .L.strzero_end.c2
                        asm_fixup_add(cg_sec, o, format(".L.strzero_end.%d", c2), 1);
                    }
                    arm64_sub_imm(cg_sec, 1, ARM64_X9, ARM64_X9, 1, 0); // sub x9, x9, #1
                    asm_strb_wzr_x9(cg_sec, dst); // strb wzr, [x{dst}, x9]
                    {
                        size_t o = asm_jmp_label(cg_sec); // b .L.strzero.c2
                        asm_fixup_add(cg_sec, o, format(".L.strzero.%d", c2), 0);
                    }
                    cg_def_label(format(".L.strzero_end.%d", c2));
                }
#else
                {
                    asm_mov_imm_phy(cg_sec, X86_RCX, 8, copy_len); // movq $copy_len, %rcx
                    cg_def_label(format(".L.strcpy.%d", c));
                    x86_cmp_ri(cg_sec, 8, X86_RCX, 0); // cmpq $0, %rcx
                    {
                        size_t o = asm_jcc_label(cg_sec, X86_E); // je .L.strcpy_end.c
                        asm_fixup_add(cg_sec, o, format(".L.strcpy_end.%d", c), 1);
                    }
                    x86_mov_rm(cg_sec, 1, X86_RAX, x86_mem_idx(REG(src), X86_RCX, 1, -1)); // movb -1(src, %rcx), %al
                    x86_mov_mr(cg_sec, 1, x86_mem_idx(REG(dst), X86_RCX, 1, -1), X86_RAX); // movb %al, -1(dst, %rcx)
                    x86_sub_ri(cg_sec, 8, X86_RCX, 1); // subq $1, %rcx
                    {
                        size_t o = asm_jmp_label(cg_sec); // jmp .L.strcpy.c
                        asm_fixup_add(cg_sec, o, format(".L.strcpy.%d", c), 0);
                    }
                    cg_def_label(format(".L.strcpy_end.%d", c));
                }
                if (copy_len < lhs_size) {
                    VReg cnt2 = alloc_reg();
                    asm_mov_imm(cg_sec, cnt2, 8, lhs_size - copy_len); // movq $count, cnt2
                    int c2 = ++rcc_label_count;
                    cg_def_label(format(".L.strzero.%d", c2));
                    x86_cmp_ri(cg_sec, 8, REG(cnt2), 0); // cmpq $0, cnt2
                    {
                        size_t o = asm_jcc_label(cg_sec, X86_E); // je .L.strzero_end.c2
                        asm_fixup_add(cg_sec, o, format(".L.strzero_end.%d", c2), 1);
                    }
                    // movb $0, copy_len-1(dst, cnt2)
                    x86_mov_mi(cg_sec, 1, x86_mem_idx(REG(dst), REG(cnt2), 1, copy_len - 1), 0);
                    x86_sub_ri(cg_sec, 8, REG(cnt2), 1); // subq $1, cnt2
                    {
                        size_t o = asm_jmp_label(cg_sec); // jmp .L.strzero.c2
                        asm_fixup_add(cg_sec, o, format(".L.strzero.%d", c2), 0);
                    }
                    cg_def_label(format(".L.strzero_end.%d", c2));
                    free_reg(cnt2);
                }
#endif
                free_reg(src);
                return dst;
            }

            if (node->rhs->ty && (node->rhs->ty->kind == TY_STRUCT || node->rhs->ty->kind == TY_UNION || node->rhs->ty->kind == TY_ARRAY))
                // For chain assignments (d = e = a[0] = c), use gen() to trigger
                // inner assignment evaluation, not gen_addr() which skips it.
                src = (node->rhs->kind == ND_ASSIGN) ? gen(node->rhs) : gen_addr(node->rhs);
            else if (node->rhs->ty && is_complex(node->rhs->ty) && node->rhs->kind == ND_LVAR && !node->rhs->var->is_local)
                src = gen_addr(node->rhs);
            else if (node->rhs->ty && is_complex(node->rhs->ty) && node->rhs->kind == ND_DEREF)
                src = gen_addr(node->rhs); // need source address, not loaded value
            else
                src = gen(node->rhs);
            // Complex-to-complex assignment with differing real-floating base
            // types (e.g. _Complex float = _Complex double or vice versa):
            // convert each component instead of doing a raw byte copy.
            if (node->lhs->ty->kind == TY_COMPLEX && node->rhs->ty && node->rhs->ty->kind == TY_COMPLEX &&
                is_flonum(node->lhs->ty->base) && is_flonum(node->rhs->ty->base) &&
                node->lhs->ty->base->kind != TY_LDOUBLE && node->rhs->ty->base->kind != TY_LDOUBLE &&
                node->lhs->ty->base->size != node->rhs->ty->base->size) {
                emit_complex_convert_float(src, dst, node->rhs->ty, node->lhs->ty);
                free_reg(src);
                return dst;
            }

            // Complex-to-complex assignment with differing integer base
            // types (e.g. _Complex int = _Complex char): convert components.
            if (node->lhs->ty->kind == TY_COMPLEX && node->rhs->ty && node->rhs->ty->kind == TY_COMPLEX &&
                !is_flonum(node->lhs->ty->base) && !is_flonum(node->rhs->ty->base) &&
                node->lhs->ty->base->size != node->rhs->ty->base->size) {
                emit_complex_convert_int(src, dst, node->rhs->ty, node->lhs->ty);
                free_reg(src);
                return dst;
            }

            // Complex-to-complex assignment with mixed int/float base types
            // (e.g. _Complex double = _Complex int): convert components.
            if (node->lhs->ty->kind == TY_COMPLEX && node->rhs->ty && node->rhs->ty->kind == TY_COMPLEX &&
                is_flonum(node->lhs->ty->base) != is_flonum(node->rhs->ty->base)) {
                if (is_flonum(node->lhs->ty->base)) {
                    emit_complex_convert_mixed(src, dst, node->rhs->ty, node->lhs->ty);
                } else {
                    emit_complex_convert_mixed(src, dst, node->rhs->ty, node->lhs->ty);
                }
                free_reg(src);
                return dst;
            }

            // If RHS is a scalar (e.g. pointer/function) and LHS is a struct/union,
            // store the value directly instead of copying bytes from the address in
            // the register. For aggregates larger than a register, zero-fill the
            // destination first and store the value into the leading bytes (the
            // scalar only initializes the first sub-object, e.g. `s = 0` or the
            // `{ 0 }` compound-literal member init): the old byte-copy loop read
            // from the scalar *value* register as if it were a source address.
            if (node->rhs->ty && !(node->rhs->ty->kind == TY_STRUCT || node->rhs->ty->kind == TY_UNION || node->rhs->ty->kind == TY_ARRAY || node->rhs->ty->kind == TY_COMPLEX)) {
                if (node->lhs->ty->size > 8) {
#ifdef ARCH_ARM64
                    emit_mov_imm64(ARM64_X9, (uint64_t)node->lhs->ty->size); // mov x9, #size
                    cg_def_label(format(".L.zero.%d", c));
                    arm64_subs_imm(cg_sec, 1, ARM64_XZR, ARM64_X9, 0, 0); // cmp x9, #0
                    {
                        size_t zj1 = asm_jcc_label(cg_sec, ARM64_EQ);
                        asm_fixup_add(cg_sec, zj1, format(".L.zero_end.%d", c), 1);
                    }
                    arm64_sub_imm(cg_sec, 1, ARM64_X9, ARM64_X9, 1, 0); // sub x9, x9, #1
                    asm_strb_wzr_x9(cg_sec, dst); // strb wzr, [x{dst}, x9]
                    {
                        size_t zj2 = asm_jmp_label(cg_sec);
                        asm_fixup_add(cg_sec, zj2, format(".L.zero.%d", c), 0);
                    }
                    cg_def_label(format(".L.zero_end.%d", c));
                    asm_mov_phy_reg(cg_sec, ARM64_X16, src, 1); // mov x16, x{src}
                    arm64_str_uoff(cg_sec, 3, ARM64_X16, REG(dst), 0); // str x16, [x{dst}]
#else
                    VReg cnt = alloc_reg();
                    asm_mov_imm(cg_sec, cnt, 8, node->lhs->ty->size); // movq $size, cnt
                    cg_def_label(format(".L.zero.%d", c));
                    x86_cmp_ri(cg_sec, 8, REG(cnt), 0); // cmpq $0, cnt
                    {
                        size_t zj1 = asm_jcc_label(cg_sec, X86_E);
                        asm_fixup_add(cg_sec, zj1, format(".L.zero_end.%d", c), 1);
                    }
                    x86_mov_mi(cg_sec, 1, x86_mem_idx(REG(dst), REG(cnt), 1, -1), 0); // movb $0, -1(dst,cnt)
                    x86_sub_ri(cg_sec, 8, REG(cnt), 1); // subq $1, cnt
                    {
                        size_t zj2 = asm_jmp_label(cg_sec);
                        asm_fixup_add(cg_sec, zj2, format(".L.zero.%d", c), 0);
                    }
                    cg_def_label(format(".L.zero_end.%d", c));
                    free_reg(cnt);
                    asm_mov_reg_mem(cg_sec, src, dst, 8); // movq src, (%dst)
#endif
                } else {
#ifdef ARCH_ARM64
                    emit_store(node->lhs->ty, src, dst, 0);
#else
                    {
                        int st_sz = node->lhs->ty->size;
                        if (st_sz < 4) st_sz = st_sz;
                        asm_mov_reg_mem(cg_sec, src, dst, st_sz); // movl/movq src, (%dst)
                    }
#endif
                }
                free_reg(src);
                return dst;
            }

            // For VLA-containing structs, use the runtime size instead of ty->size
            bool copy_is_vla_struct = (node->lhs->ty->kind == TY_STRUCT || node->lhs->ty->kind == TY_UNION) && node->lhs->ty->vla_len_expr;
            VReg r_vla_sz = -1;
            if (copy_is_vla_struct)
                r_vla_sz = gen(node->lhs->ty->vla_len_expr);
#ifdef ARCH_ARM64
            if (copy_is_vla_struct && r_vla_sz >= 0)
                asm_mov_x9_vreg(cg_sec, r_vla_sz); // mov x9, x{r_vla_sz}
            else
                emit_mov_imm64(ARM64_X9, (uint64_t)node->lhs->ty->size); // mov x9, #size
            cg_def_label(format(".L.copy.%d", c));
            arm64_subs_imm(cg_sec, 1, ARM64_XZR, ARM64_X9, 0, 0); // cmp x9, #0
            {
                size_t _cj = asm_jcc_label(cg_sec, ARM64_EQ);
                asm_fixup_add(cg_sec, _cj, format(".L.copy_end.%d", c), 1);
            }
            arm64_sub_imm(cg_sec, 1, ARM64_X9, ARM64_X9, 1, 0); // sub x9, x9, #1
            asm_ldrb_w16_x9(cg_sec, src); // ldrb w16, [x{src}, x9]
            asm_strb_w16_x9(cg_sec, dst); // strb w16, [x{dst}, x9]
            {
                size_t _jmp = asm_jmp_label(cg_sec);
                asm_fixup_add(cg_sec, _jmp, format(".L.copy.%d", c), 0);
            }
            cg_def_label(format(".L.copy_end.%d", c));
#else
            {
                VReg cnt = alloc_reg(); // dedicated counter, doesn't conflict with src/dst
                if (copy_is_vla_struct && r_vla_sz >= 0)
                    asm_mov_reg_reg(cg_sec, cnt, r_vla_sz, 8); // movq r_vla_sz, cnt
                else
                    asm_mov_imm(cg_sec, cnt, 8, node->lhs->ty->size); // movq $size, cnt
                cg_def_label(format(".L.copy.%d", c));
                x86_cmp_ri(cg_sec, 8, REG(cnt), 0); // cmpq $0, cnt
                size_t cj1 = asm_jcc_label(cg_sec, X86_E);
                asm_fixup_add(cg_sec, cj1, format(".L.copy_end.%d", c), 1);
                {
                    X86Mem msrc = x86_mem_idx(REG(src), REG(cnt), 1, -1); // -1(src, cnt)
                    X86Mem mdst = x86_mem_idx(REG(dst), REG(cnt), 1, -1); // -1(dst, cnt)
                    x86_mov_rm(cg_sec, 1, X86_RAX, msrc); // movb -1(src,cnt), %al
                    x86_mov_mr(cg_sec, 1, mdst, X86_RAX); // movb %al, -1(dst,cnt)
                }
                x86_sub_ri(cg_sec, 8, REG(cnt), 1); // subq $1, cnt
                size_t cj2 = asm_jmp_label(cg_sec);
                asm_fixup_add(cg_sec, cj2, format(".L.copy.%d", c), 0);
                cg_def_label(format(".L.copy_end.%d", c));
                free_reg(cnt);
            }
#endif
            if (r_vla_sz >= 0) free_reg(r_vla_sz);
            free_reg(src);
            return dst;
        }
        if (is_flonum(node->lhs->ty)) {
#ifdef ARCH_ARM64
            VReg r2 = gen(node->rhs);
            VReg r1 = gen_addr(node->lhs);
            asm_fmov_i2f(cg_sec, 0, r2, 1); // fmov d0, x{r2}
            if (node->lhs->ty->size == 4) {
                asm_fcvt(cg_sec, 0, 1, 0, 0); // fcvt s0, d0 (opc=0=double->single)
                asm_str_fp(cg_sec, 0, r1, 4); // str s0, [x{r1}]
            } else {
                asm_str_fp(cg_sec, 0, r1, 8); // str d0, [x{r1}]
            }
            free_reg(r1);
            return r2;
#else
#ifndef _WIN32
            if (node->lhs->ty->kind == TY_LDOUBLE) {
                // Store long double as 64-bit double (truncated)
                VReg r2 = gen(node->rhs);
                VReg r1 = gen_addr(node->lhs);
                asm_movq_r_xmm(cg_sec, 0, r2); // movq r2, %xmm0
                asm_str_fp(cg_sec, 0, r1, 8); // movsd %xmm0, (r1)
                free_reg(r1);
                // The assignment expression's value is the (truncated)
                // stored value, i.e. the double bit pattern already in
                // r2 -- exactly like the plain double path below.
                return r2;
            }
#endif
            VReg r2 = gen(node->rhs);
            VReg r1 = gen_addr(node->lhs);
            asm_movq_r_xmm(cg_sec, 0, r2); // movq r2, %xmm0
            if (node->lhs->ty->size == 4) {
                asm_cvtsd2ss(cg_sec); // cvtsd2ss %xmm0, %xmm0
                asm_str_fp(cg_sec, 0, r1, 4); // movss %xmm0, (r1)
            } else {
                asm_str_fp(cg_sec, 0, r1, 8); // movsd %xmm0, (r1)
            }
            free_reg(r1);
            return r2;
#endif
        }
        // 8/16-byte vector value copy (vector results are slot addresses,
        // so gen(rhs) would store the address instead of the bytes — the
        // 8-byte MMX case used to fall through to the scalar store below).
        if (node->lhs->ty->is_vector && (node->lhs->ty->size == 16 || node->lhs->ty->size == 8)) {
            VReg rhs_a = gen_addr(node->rhs);
            VReg lhs_a = gen_addr(node->lhs);
            if (node->lhs->ty->size == 8) {
#ifdef ARCH_ARM64
                arm64_ldr_fp(cg_sec, 3, ARM64_D0, REG(rhs_a), 0); // ldr d0, [rhs_a]
                arm64_str_fp(cg_sec, 3, ARM64_D0, REG(lhs_a), 0); // str d0, [lhs_a]
#else
                x86_movq_rm(cg_sec, x86_mem(REG(rhs_a), 0), X86_XMM0);
                x86_movq_mr(cg_sec, x86_mem(REG(lhs_a), 0), X86_XMM0);
#endif
            } else {
#ifdef ARCH_ARM64
                asm_ldr_q(cg_sec, ASM_Q0, rhs_a); // ldr q0, [x{rhs_a}]
                asm_str_q(cg_sec, ASM_Q0, lhs_a); // str q0, [x{lhs_a}]
#else
                x86_movups_rm(cg_sec, X86_XMM0, x86_mem(REG(rhs_a), 0));
                x86_movups_mr(cg_sec, x86_mem(REG(lhs_a), 0), X86_XMM0);
#endif
            }
            free_reg(rhs_a);
            return lhs_a;
        }
        if (node->lhs->kind == ND_LVAR && node->lhs->var->is_local && node->lhs->var->ty->kind != TY_ARRAY) {
            VReg r2 = gen(node->rhs);
#ifdef ARCH_ARM64
            if (node->lhs->chain_depth > 0) {
                VReg base = alloc_reg();
                gen_chain_walk_into(base, node->lhs->chain_depth);
                emit_store_offset(node->lhs->ty, r2, REG(base), -node->lhs->var->offset);
                free_reg(base);
            } else
                emit_store_offset(node->lhs->ty, r2, FRAME_PTR, -node->lhs->var->offset);
#else
            if (node->lhs->chain_depth > 0) {
                VReg base = alloc_reg();
                gen_chain_walk_into(base, node->lhs->chain_depth);
                x86_mov_mr(cg_sec, node->lhs->ty->size, x86_mem(REG(base), -node->lhs->var->offset), REG(r2));
                free_reg(base);
            } else
                asm_mov_reg_rbp(cg_sec, r2, node->lhs->ty->size, node->lhs->var->offset); // mov rr2, [rbp-node->lhs->ty->size]
#endif
            // Truncate result to match the variable's type width for unsigned narrow types
            if (node->lhs->ty->is_unsigned && node->lhs->ty->size < 4) {
                int mask = (1 << (node->lhs->ty->size * 8)) - 1;
                asm_and_imm(cg_sec, r2, 4, mask); // and $mask, reg[r2]
            }
            return r2;
        }
        // Bitfield assignment: read-modify-write
        if (node->lhs->kind == ND_MEMBER && node->lhs->member &&
            node->lhs->member->bit_width > 0) {
            int bw = node->lhs->member->bit_width;
            int bo = node->lhs->member->bit_offset;
            int unit_sz = node->lhs->member->bf_load_size
                ? node->lhs->member->bf_load_size
                : node->lhs->member->ty->size;
            unsigned long long mask = ((1ULL << bw) - 1) << bo;

            // Check if RHS reads the same bitfield (compound assignment like s.x += 1)
            bool rhs_reads_same = false;
            if (node->rhs->kind == ND_MEMBER && node->rhs->member == node->lhs->member) {
                rhs_reads_same = true;
            } else if (node->rhs->kind == ND_ADD || node->rhs->kind == ND_SUB ||
                       node->rhs->kind == ND_MUL || node->rhs->kind == ND_DIV ||
                       node->rhs->kind == ND_BITAND || node->rhs->kind == ND_BITOR ||
                       node->rhs->kind == ND_BITXOR) {
                if (node->rhs->lhs && node->rhs->lhs->kind == ND_MEMBER &&
                    node->rhs->lhs->member == node->lhs->member)
                    rhs_reads_same = true;
                if (node->rhs->rhs && node->rhs->rhs->kind == ND_MEMBER &&
                    node->rhs->rhs->member == node->lhs->member)
                    rhs_reads_same = true;
            }

            // Helper: emit unsigned load of unit_sz bytes
            // Helper: emit unsigned load of unit_sz bytes
#ifdef ARCH_ARM64
#define BF_LOAD(sz, ra, rt) do { asm_ldr_reg_off(cg_sec, rt, ra, sz, 0); } while (0)
#define BF_STORE(sz, ra, rt) do { asm_str_reg_off(cg_sec, rt, ra, sz, 0); } while (0)
#else
#define BF_LOAD(sz, ra, rt) do { \
    if ((sz) == 1) asm_movzx_mem_reg(cg_sec, rt, ra, 4, 1); \
    else if ((sz) == 2) asm_movzx_mem_reg(cg_sec, rt, ra, 4, 2); \
    else if ((sz) == 4) asm_mov_mem_reg(cg_sec, rt, ra, 4); \
    else asm_mov_mem_reg(cg_sec, rt, ra, 8); \
} while (0)
#define BF_STORE(sz, ra, rt) do { \
    if ((sz) == 1) asm_mov_reg_mem(cg_sec, rt, ra, 1); \
    else if ((sz) == 2) asm_mov_reg_mem(cg_sec, rt, ra, 2); \
    else if ((sz) == 4) asm_mov_reg_mem(cg_sec, rt, ra, 4); \
    else asm_mov_reg_mem(cg_sec, rt, ra, 8); \
} while (0)
#endif

            // Generate RHS (the new value to assign)
            VReg r2 = gen(node->rhs);

            if (rhs_reads_same) {
                VReg ra = gen_addr(node->lhs);
                VReg rt = alloc_reg();
                int eff_sz_rhs = unit_sz > 8 ? 8 : unit_sz;
                BF_LOAD(eff_sz_rhs, ra, rt);
#ifdef ARCH_ARM64
                emit_mov_imm64(ARM64_X16, ~mask);
                asm_and_reg_phy(cg_sec, rt, ARM64_X16, 8); // and rrt, r16
                emit_mov_imm64(ARM64_X16, (1ULL << bw) - 1);
                VReg rv = alloc_reg_avoid2(rt, ra);
                asm_mov_reg_reg(cg_sec, rv, r2, 8); // mov rr2 -> rrv
                asm_and_reg_phy(cg_sec, rv, ARM64_X16, 8); // and rrv, r16
                if (bo > 0) asm_shl_imm(cg_sec, rv, 8, (uint8_t)(bo)); // lsl x{rv}, x{rv}, #bo
                asm_or_reg_reg(cg_sec, rt, rv, 8); // orr x{rt}, x{rt}, x{rv} (merge new value into old)
                BF_STORE(eff_sz_rhs, ra, rt);
                free_reg(rv);
                // Reload stored bitfield value for assignment expression result
                {
                    int new_eff_sz_rhs = eff_sz_rhs;
                    if (new_eff_sz_rhs == 1)
                        asm_ldrb_uoff(cg_sec, rt, ra, 0); // ldrb w{rt}, [x{ra}]
                    else if (new_eff_sz_rhs == 2)
                        asm_ldrh_uoff(cg_sec, rt, ra, 0); // ldrh w{rt}, [x{ra}]
                    else if (new_eff_sz_rhs == 4)
                        asm_ldr_reg_off(cg_sec, rt, ra, 4, 0); // ldr w{rt}, [x{ra}]
                    else
                        asm_ldr_reg_off(cg_sec, rt, ra, 8, 0); // ldr x{rt}, [x{ra}]
                    if (bo > 0)
                        asm_shr_imm(cg_sec, rt, 8, (uint8_t)(bo)); // lsr x{rt}, x{rt}, #bo
                    if (bw < new_eff_sz_rhs * 8) {
                        if (node->lhs->member->ty->is_unsigned || node->lhs->member->ty->is_enum) {
                            emit_mov_imm64(ARM64_X16, (1ULL << bw) - 1);
                            asm_and_reg_phy(cg_sec, rt, ARM64_X16, 8); // and x{rt}, x{rt}, x16
                        } else {
                            int shift = 64 - bw;
                            asm_shl_imm(cg_sec, rt, 8, (uint8_t)(shift)); // lsl x{rt}, x{rt}, #shift
                            asm_sar_imm(cg_sec, rt, 8, (uint8_t)(shift)); // asr x{rt}, x{rt}, #shift
                        }
                    }
                    free_reg(r2);
                    VReg ret_reg = rt;
                    free_reg(ra);
                    return ret_reg;
                }
            }

            // Simple assignment: read-modify-write
            free_reg(gen_addr(node->lhs));
            VReg ra = gen_addr(node->lhs);
            VReg rt = alloc_reg();
            int eff_sz = unit_sz > 8 ? 8 : unit_sz;
            BF_LOAD(eff_sz, ra, rt);
            emit_mov_imm64(ARM64_X16, ~mask);
            asm_and_reg_phy(cg_sec, rt, ARM64_X16, 8); // and rrt, r16
            emit_mov_imm64(ARM64_X16, (1ULL << bw) - 1);
            VReg rv = alloc_reg_avoid2(rt, ra);
            asm_mov_reg_reg(cg_sec, rv, r2, 8); // mov rr2 -> rrv
            asm_and_reg_phy(cg_sec, rv, ARM64_X16, 8); // and rrv, r16
            if (bo > 0) asm_shl_imm(cg_sec, rv, 8, (uint8_t)(bo)); // lsl x{rv}, x{rv}, #bo
            asm_or_reg_reg(cg_sec, rt, rv, 8); // orr x{rt}, x{rt}, x{rv} (merge new value into old)
            BF_STORE(eff_sz, ra, rt);
            if (unit_sz > 8 && bo + bw > 64) {
                int overflow = bo + bw - 64;
                unsigned int ovf_mask = (1u << overflow) - 1;
                asm_add_imm(cg_sec, ra, 8, 8); // add ra, ra, #8
                asm_ldrb_uoff(cg_sec, rt, ra, 0); // ldrb w{rt}, [x{ra}]
                asm_and_imm(cg_sec, rt, 4, (int32_t)(~ovf_mask & 0xFF)); // and w{rt}, w{rt}, #~ovf_mask
                asm_mov_reg_reg(cg_sec, rv, r2, 8); // mov x{rv}, x{r2}
                asm_shr_imm(cg_sec, rv, 8, (uint8_t)(64 - bo)); // lsr x{rv}, x{rv}, #(64-bo)
                asm_and_imm(cg_sec, rv, 4, (int32_t)ovf_mask); // and w{rv}, w{rv}, #ovf_mask
                asm_or_reg_reg(cg_sec, rt, rv, 4); // orr w{rt}, w{rt}, w{rv}
                asm_strb_uoff(cg_sec, rt, ra, 0); // strb w{rt}, [x{ra}]
            }
#else
                if (bo > 0) asm_shr_imm(cg_sec, rt, 8, (uint8_t)(bo)); // shr $(uint8_t)(bo), rrt
                if (bw < unit_sz * 8) {
                    unsigned long long m = (1ULL << bw) - 1;
                    asm_movabs_phy(cg_sec, X86_RAX, (uint64_t)(m)); // movabs $(uint64_t)(m), rX86_RAX
                    asm_and_rax(cg_sec, rt, 8); // andq %rax, rrt
                }
                // Re-load for modify
                BF_LOAD(unit_sz, ra, rt);
                asm_movabs_phy(cg_sec, X86_RAX, (uint64_t)(~mask)); // movabs $(uint64_t)(~mask), rX86_RAX
                asm_and_rax(cg_sec, rt, 8); // andq %rax, rrt
                VReg rv = alloc_reg_avoid2(rt, ra);
                asm_mov_reg_reg(cg_sec, rv, r2, 8); // mov rr2 -> rrv
                asm_movabs_phy(cg_sec, X86_RAX, (uint64_t)((1ULL << bw) - 1)); // movabs $(uint64_t)((1ULL << bw) - 1), rX86_RAX
                asm_and_rax(cg_sec, rv, 8); // andq %rax, rrv
                if (bo > 0) asm_shl_imm(cg_sec, rv, 8, (uint8_t)(bo)); // shl $(uint8_t)(bo), rrv
                asm_or_reg_reg(cg_sec, rt, rv, 8); // or rrv, rrt
                BF_STORE(unit_sz, ra, rt);
                free_reg(rv);
                // Reload stored bitfield value for assignment expression result
                {
                    int new_eff_sz = unit_sz > 8 ? 8 : unit_sz;
                    BF_LOAD(new_eff_sz, ra, rt); // reload bitfield unit from memory
                    if (bo > 0)
                        asm_shr_imm(cg_sec, rt, 8, (uint8_t)(bo)); // shr $(uint8_t)(bo), rrt
                    if (bw < new_eff_sz * 8) {
                        if (node->lhs->member->ty->is_unsigned || node->lhs->member->ty->is_enum) {
                            unsigned long long m = (1ULL << bw) - 1;
                            asm_movabs_phy(cg_sec, X86_RAX, (uint64_t)(m)); // movabs $(uint64_t)(m), rX86_RAX
                            asm_and_rax(cg_sec, rt, 8); // andq %rax, rrt
                        } else {
                            int shift = 64 - bw;
                            asm_shl_imm(cg_sec, rt, 8, (uint8_t)(shift)); // shl $(uint8_t)(shift), rrt
                            asm_sar_imm(cg_sec, rt, 8, (uint8_t)(shift)); // sar $(uint8_t)(shift), rrt
                        }
                    }
                    free_reg(r2);
                    VReg ret_reg = rt;
                    free_reg(ra);
                    return ret_reg;
                }
            }

            // Simple assignment: read-modify-write
            free_reg(gen_addr(node->lhs)); // discard; re-gen below
            VReg ra = gen_addr(node->lhs);
            VReg rt = alloc_reg();
            int eff_sz = unit_sz > 8 ? 8 : unit_sz;
            BF_LOAD(eff_sz, ra, rt);
            asm_movabs_phy(cg_sec, X86_RAX, (uint64_t)(~mask)); // movabs $(uint64_t)(~mask), rX86_RAX
            asm_and_rax(cg_sec, rt, 8); // andq %rax, rrt
            VReg rv = alloc_reg_avoid2(rt, ra);
            asm_mov_reg_reg(cg_sec, rv, r2, 8); // mov rr2 -> rrv
            asm_movabs_phy(cg_sec, X86_RAX, (uint64_t)((1ULL << bw) - 1)); // movabs $(uint64_t)((1ULL << bw) - 1), rX86_RAX
            asm_and_rax(cg_sec, rv, 8); // andq %rax, rrv
            if (bo > 0) asm_shl_imm(cg_sec, rv, 8, (uint8_t)(bo)); // shl $(uint8_t)(bo), rrv
            asm_or_reg_reg(cg_sec, rt, rv, 8); // or rrv, rrt
            BF_STORE(eff_sz, ra, rt);
            // Handle overflow bits beyond the first 8 bytes
            if (unit_sz > 8 && bo + bw > 64) {
                int overflow = bo + bw - 64;
                unsigned int ovf_mask = (1u << overflow) - 1;
                asm_add_imm(cg_sec, ra, 8, 8); // addq $8, ra
                x86_movzx_rm(cg_sec, 4, 1, REG(rt), x86_mem(REG(ra), 0)); // movzbl (%ra), reg32[rt]
                asm_and_imm(cg_sec, rt, 4, (unsigned)(~ovf_mask & 0xFF)); // and $~ovf_mask, reg32[rt]
                asm_mov_reg_reg(cg_sec, rv, r2, 8); // mov rr2 -> rrv
                asm_shr_imm(cg_sec, rv, 8, (uint8_t)(64 - bo)); // shr $(uint8_t)(64 - bo), rrv
                asm_and_imm(cg_sec, rv, 4, ovf_mask); // and $ovf_mask, reg32[rv]
                asm_or_reg_reg(cg_sec, rt, rv, 4); // or rrv, rrt
                asm_mov_reg_mem(cg_sec, rt, ra, 1); // movb reg8[rt], (%ra)
            }
#endif
#undef BF_LOAD
#undef BF_STORE
            free_reg(rv);
#ifdef ARCH_ARM64
            // ARM64 reload for assignment expression result
            {
                int new_eff_sz = eff_sz;
                if (new_eff_sz == 1)
                    asm_ldrb_uoff(cg_sec, rt, ra, 0); // ldrb w{rt}, [x{ra}]
                else if (new_eff_sz == 2)
                    asm_ldrh_uoff(cg_sec, rt, ra, 0); // ldrh w{rt}, [x{ra}]
                else if (new_eff_sz == 4)
                    asm_ldr_reg_off(cg_sec, rt, ra, 4, 0); // ldr w{rt}, [x{ra}]
                else
                    asm_ldr_reg_off(cg_sec, rt, ra, 8, 0); // ldr x{rt}, [x{ra}]
                if (bo > 0)
                    asm_shr_imm(cg_sec, rt, 8, (uint8_t)(bo)); // lsr x{rt}, x{rt}, #bo
                if (bw < new_eff_sz * 8) {
                    if (node->lhs->member && (node->lhs->member->ty->is_unsigned || node->lhs->member->ty->is_enum)) {
                        emit_mov_imm64(ARM64_X16, (1ULL << bw) - 1);
                        asm_and_reg_phy(cg_sec, rt, ARM64_X16, 8); // and rrt, r16
                    } else {
                        int shift = 64 - bw;
                        asm_shl_imm(cg_sec, rt, 8, (uint8_t)(shift)); // lsl x{rt}, x{rt}, #shift
                        asm_sar_imm(cg_sec, rt, 8, (uint8_t)(shift)); // asr x{rt}, x{rt}, #shift
                    }
                }
                free_reg(r2);
                VReg ret_reg = rt;
                free_reg(ra);
                return ret_reg;
            }
#else
            // x86_64 reload for assignment expression result
            {
                int new_eff_sz = eff_sz;
                // Reload from [ra] into rt with appropriate size
                if (new_eff_sz == 1) asm_movzx_mem_reg(cg_sec, rt, ra, 4, 1); // movzbl (%ra), rt
                else if (new_eff_sz == 2)
                    asm_movzx_mem_reg(cg_sec, rt, ra, 4, 2); // movzwl (%ra), rt
                else if (new_eff_sz == 4)
                    asm_mov_mem_reg(cg_sec, rt, ra, 4); // movl (%ra), rt
                else
                    asm_mov_mem_reg(cg_sec, rt, ra, 8); // movq (%ra), rt
                if (bo > 0)
                    asm_shr_imm(cg_sec, rt, 8, (uint8_t)(bo)); // shr $(uint8_t)(bo), rrt
                if (bw < new_eff_sz * 8) {
                    if (node->lhs->member && (node->lhs->member->ty->is_unsigned || node->lhs->member->ty->is_enum)) {
                        unsigned long long m = (1ULL << bw) - 1;
                        asm_movabs_phy(cg_sec, X86_RAX, (uint64_t)(m)); // movabs $(uint64_t)(m), rX86_RAX
                        asm_and_rax(cg_sec, rt, 8); // andq %rax, rrt
                    } else {
                        int shift = 64 - bw;
                        asm_shl_imm(cg_sec, rt, 8, (uint8_t)(shift)); // shl $(uint8_t)(shift), rrt
                        asm_sar_imm(cg_sec, rt, 8, (uint8_t)(shift)); // sar $(uint8_t)(shift), rrt
                    }
                }
                free_reg(r2);
                VReg ret_reg = rt;
                free_reg(ra);
                return ret_reg;
            }
#endif
        }
        VReg r2 = gen(node->rhs);
        VReg r1 = gen_addr(node->lhs);
#ifdef ARCH_ARM64
        emit_store(node->lhs->ty, r2, r1, 0);
#else
        {
            int st_sz = node->lhs->ty->size;
            if (st_sz == 8) st_sz = 8;
            else if (st_sz == 4)
                st_sz = 4;
            else if (st_sz == 2)
                st_sz = 2;
            else
                st_sz = 1;
            asm_mov_reg_mem(cg_sec, r2, r1, st_sz); // movl/movq r2, (%r1)
        }
#endif
        free_reg(r1);
        return r2;
    }
    case ND_NEG: {
        VReg r = gen(node->lhs);
        if (is_flonum(node->ty)) {
#ifdef ARCH_ARM64
            asm_fmov_i2f(cg_sec, 0, r, 1); // fmov d0, x{r}
            asm_fneg_d0(cg_sec); // fneg d0, d0
            asm_fmov_f2i(cg_sec, r, 0, 1); // fmov x{r}, d0
#else
            // Negate float: xor with sign bit
            asm_movq_r_xmm(cg_sec, X86_XMM0, r); // movq r, %xmm0
            // Use pxor with sign bit mask
            asm_mov_imm(cg_sec, r, 8, (long long)0x8000000000000000LL); // movabs $0x8000...0, r
            asm_movq_r_xmm(cg_sec, X86_XMM1, r); // movq r, %xmm1
            x86_xorpd(cg_sec, X86_XMM0, X86_XMM1); // xorpd %xmm1, %xmm0
            asm_movq_xmm_r(cg_sec, r, X86_XMM0); // movq %xmm0, r
#endif
        } else {
#ifdef ARCH_ARM64
            asm_neg(cg_sec, r, op_size(node->ty)); // neg x{r}
#else
            asm_neg(cg_sec, r, op_size(node->ty)); // neg rr
#endif
        }
        return r;
    }
    case ND_NOT: {
        VReg r = gen(node->lhs);
        if (is_flonum(node->lhs->ty)) {
#ifdef ARCH_ARM64
            asm_fmov_i2f(cg_sec, 0, r, 1); // fmov d0, x{r}
            asm_fmov_d1_xzr(cg_sec); // fmov d1, xzr
            asm_fcmp(cg_sec, 1); // fcmp d0, d1
            asm_cset(cg_sec, r, ARM64_EQ); // cset r, eq
            // csel r, r, wzr, vs  (clear if unordered/NaN)
            asm_csel_vs_zero(cg_sec, r, 4); // csel wr, wr, wzr, vs
#else
            asm_movq_r_xmm(cg_sec, X86_XMM0, r); // movq %s, %%xmm0
            x86_pxor(cg_sec, X86_XMM1, X86_XMM1); // xorpd %%xmm1, %%xmm1
            asm_ucomisd(cg_sec); // ucomisd %%xmm1, %%xmm0
            asm_setcc(cg_sec, X86_RCX, X86_NP); // setnp %%cl
            asm_setcc(cg_sec, X86_RAX, X86_E); // sete %%al
            x86_and_rr(cg_sec, 1, X86_RAX, X86_RCX); // andb %%cl, %%al
            asm_movzx_phys(cg_sec, r, X86_RAX, 4, 1); // movzbl %%al, %s
#endif
        } else {
#ifdef ARCH_ARM64
            asm_cmp_zero(cg_sec, r, op_size(node->lhs->ty)); // sete %%al
            asm_cset(cg_sec, r, ARM64_EQ); // andb %%cl, %%al
#else
            asm_cmp_zero(cg_sec, r, op_size(node->lhs->ty)); // cmp $0, %s
            asm_setcc(cg_sec, X86_RAX, X86_E); // sete %%al
            asm_movzx_phys(cg_sec, r, X86_RAX, 4, 1); // movzbl %%al, %s
#endif
        }
        return r;
    }
    case ND_ZERO_INIT: {
        // Zero-fill a local variable's stack memory
        LVar *var = node->lhs->var;
        if (!var || !var->is_local || var->ty->size <= 0) return -1;
        int c = ++rcc_label_count;
#ifdef ARCH_ARM64
        // Use x16/x17 scratch registers so we don't clobber allocatable x11/x9
        // which may hold live addresses (e.g. ND_ASSIGN copy-loop destination).
        if (var->offset <= 4095) {
            arm64_sub_imm(cg_sec, 1, ARM64_X16, ARM64_X29, var->offset, 0); // sub x16, x29, #var->offset
        } else {
            emit_mov_imm64(ARM64_X17, (uint64_t)var->offset);
            arm64_sub_reg(cg_sec, 1, ARM64_X16, ARM64_X29, ARM64_X17, ARM64_LSL, 0); // sub x16, x29, x17
        }
        if (var->ty->size <= 4095) {
            emit_mov_imm64(ARM64_X17, (uint64_t)var->ty->size); // mov x17, #size
        } else {
            emit_mov_imm64(ARM64_X17, (uint64_t)var->ty->size);
        }
        cg_def_label(format(".L.zero.%d", c));
        arm64_subs_imm(cg_sec, 1, ARM64_XZR, ARM64_X17, 0, 0); // cmp x17, #0
        size_t zj1 = asm_jcc_label(cg_sec, ARM64_EQ); // b.eq .L.zero_end.c
        asm_fixup_add(cg_sec, zj1, format(".L.zero_end.%d", c), 1);
        arm64_sub_imm(cg_sec, 1, ARM64_X17, ARM64_X17, 1, 0); // sub x17, x17, #1
        arm64_str_reg(cg_sec, 0, ARM64_XZR, ARM64_X16, ARM64_X17, true, 0); // strb wzr, [x16, x17]
        size_t zj2 = asm_jmp_label(cg_sec); // b .L.zero.c
        asm_fixup_add(cg_sec, zj2, format(".L.zero.%d", c), 0); // strb wzr, [x16, x17]
        cg_def_label(format(".L.zero_end.%d", c)); // b .L.zero.%d
#else
        x86_mov_ri(cg_sec, 8, X86_RCX, var->ty->size); // movq $size, %rcx
        cg_def_label(format(".L.zero.%d", c));
        x86_cmp_ri(cg_sec, 8, X86_RCX, 0); // cmpq $0, %rcx
        size_t zj1 = asm_jcc_label(cg_sec, X86_E);
        asm_fixup_add(cg_sec, zj1, format(".L.zero_end.%d", c), 1);
        x86_mov_mi(cg_sec, 1, x86_mem_idx(X86_RBP, X86_RCX, 1, -var->offset - 1), 0); // movb $0, -off-1(%rbp,%rcx)
        x86_sub_ri(cg_sec, 8, X86_RCX, 1); // subq $1, %rcx
        size_t zj2 = asm_jmp_label(cg_sec);
        asm_fixup_add(cg_sec, zj2, format(".L.zero.%d", c), 0);
        cg_def_label(format(".L.zero_end.%d", c));
#endif
        return -1;
    }
    case ND_POST_INC:
    case ND_POST_DEC:
    case ND_PRE_INC:
    case ND_PRE_DEC: {
        bool is_pre = (node->kind == ND_PRE_INC || node->kind == ND_PRE_DEC);
        bool is_inc = (node->kind == ND_POST_INC || node->kind == ND_PRE_INC);
        // C11 6.5.3.1: for _Bool, ++/-- follow from the general "increment
        // is x=x+1" rule composed with the _Bool store normalization
        // (0/1, any nonzero -> 1): b++ always stores 1; b-- stores the
        // opposite of the current value (1-1=0, 0-1=-1 normalizes to 1).
        if (node->lhs->ty && node->lhs->ty->kind == TY_BOOL) {
            VReg addr = gen_addr(node->lhs);
            VReg old = alloc_reg();
#ifdef ARCH_ARM64
            asm_ldrb_uoff(cg_sec, old, addr, 0);
#else
            asm_movzx_mem_reg(cg_sec, old, addr, 4, 1); // movzbl (addr), old
#endif
            VReg newv = alloc_reg();
            asm_mov_imm(cg_sec, newv, 4, 1);
            if (!is_inc)
                asm_sub_reg_reg(cg_sec, newv, old, 4); // newv = 1 - old
#ifdef ARCH_ARM64
            asm_strb_uoff(cg_sec, newv, addr, 0);
#else
            x86_mov_mr(cg_sec, 1, x86_mem(REG(addr), 0), REG(newv));
#endif
            free_reg(addr);
            if (is_pre) {
                free_reg(old);
                return newv;
            }
            free_reg(newv);
            return old;
        }
        VReg r = gen_addr(node->lhs);
        VReg r2 = alloc_reg();
        int sz = node->lhs->ty->size;
        // Handle bitfield post-increment/decrement with proper read-modify-write
        if (node->lhs->kind == ND_MEMBER && node->lhs->member &&
            node->lhs->member->bit_width > 0) {
            Member *mem = node->lhs->member;
            int bw = mem->bit_width;
            int bo = mem->bit_offset;
            int unit_sz = mem->bf_load_size ? mem->bf_load_size : mem->ty->size;
            unsigned long long mask = ((1ULL << bw) - 1) << bo;
            int eff_sz = unit_sz > 8 ? 8 : unit_sz;
#ifdef ARCH_ARM64
            // Load the full container word (unsigned)
            if (eff_sz == 1)
                asm_ldrb_uoff(cg_sec, r2, r, 0); // ldrb w{r2}, [x{r}]
            else if (eff_sz == 2)
                asm_ldrh_uoff(cg_sec, r2, r, 0); // ldrh w{r2}, [x{r}]
            else if (eff_sz == 4)
                asm_ldr_reg_off(cg_sec, r2, r, 4, 0); // ldr w{r2}, [x{r}]
            else
                asm_ldr_reg_off(cg_sec, r2, r, 8, 0); // ldr x{r2}, [x{r}]
            // Extract original bitfield value into r3 (for return)
            VReg r3 = alloc_reg();
            asm_mov_reg_reg(cg_sec, r3, r2, 8); // mov rr2 -> rr3
            if (bo > 0)
                asm_shr_imm(cg_sec, r3, 8, (uint8_t)(bo)); // lsr x{r3}, x{r3}, #bo
            int load_bits = eff_sz * 8;
            if (bw < load_bits) {
                if (mem->ty->is_unsigned || mem->ty->is_enum) {
                    emit_mov_imm64(ARM64_X16, (1ULL << bw) - 1);
                    asm_and_reg_phy(cg_sec, r3, ARM64_X16, 8); // and x{r3}, x{r3}, x16
                } else {
                    int shift = 64 - bw;
                    asm_shl_imm(cg_sec, r3, 8, (uint8_t)(shift)); // lsl x{r3}, x{r3}, #shift
                    asm_sar_imm(cg_sec, r3, 8, (uint8_t)(shift)); // asr x{r3}, x{r3}, #shift
                }
            }
            // Clear the field bits in container word (r2)
            emit_mov_imm64(ARM64_X16, ~mask);
            asm_and_reg_phy(cg_sec, r2, ARM64_X16, 8); // and rr2, r16
            // Compute new field value in rn
            VReg rn = alloc_reg();
            asm_mov_reg_reg(cg_sec, rn, r3, 8); // mov rr3 -> rrn
            emit_mov_imm64(ARM64_X16, (1ULL << bw) - 1);
            asm_and_reg_phy(cg_sec, rn, ARM64_X16, 8); // and rrn, r16
            if (is_inc)
                asm_add_imm(cg_sec, rn, 8, 1); // add $1, rrn
            else
                asm_sub_imm(cg_sec, rn, 8, 1); // sub $1, rrn
            emit_mov_imm64(ARM64_X16, (1ULL << bw) - 1);
            asm_and_reg_phy(cg_sec, rn, ARM64_X16, 8); // and rrn, r16 (wrap)
            // For prefix forms, sign/zero-extend the new field value
            VReg r4 = R_NONE;
            if (is_pre) {
                r4 = alloc_reg();
                asm_mov_reg_reg(cg_sec, r4, rn, 8); // mov rrn -> rr4
                if (bw < load_bits) {
                    if (!(mem->ty->is_unsigned || mem->ty->is_enum)) {
                        int shift = 64 - bw;
                        asm_shl_imm(cg_sec, r4, 8, (uint8_t)(shift)); // lsl x{r4}, x{r4}, #shift
                        asm_sar_imm(cg_sec, r4, 8, (uint8_t)(shift)); // asr x{r4}, x{r4}, #shift
                    }
                }
            }
            if (bo > 0)
                asm_shl_imm(cg_sec, rn, 8, (uint8_t)(bo)); // lsl x{rn}, x{rn}, #bo
            asm_or_reg_reg(cg_sec, r2, rn, 8); // orr x{r2}, x{r2}, x{rn}
            // Store
            if (eff_sz == 1)
                asm_strb_uoff(cg_sec, r2, r, 0); // strb w{r2}, [x{r}]
            else if (eff_sz == 2)
                asm_strh_uoff(cg_sec, r2, r, 0); // strh w{r2}, [x{r}]
            else if (eff_sz == 4)
                asm_str_reg_off(cg_sec, r2, r, 4, 0); // str w{r2}, [x{r}]
            else
                asm_str_reg_off(cg_sec, r2, r, 8, 0); // str x{r2}, [x{r}]
#else
            // Load the full container word (unsigned)
            if (eff_sz == 1) asm_movzx_mem_reg(cg_sec, r2, r, 4, 1); // movzbl (r), r2
            else if (eff_sz == 2)
                asm_movzx_mem_reg(cg_sec, r2, r, 4, 2); // movzwl (r), r2
            else if (eff_sz == 4)
                asm_mov_mem_reg(cg_sec, r2, r, 4); // movl (r), r2
            else
                asm_mov_mem_reg(cg_sec, r2, r, 8); // movq (r), r2
            // Extract original bitfield value into r3 (for return)
            VReg r3 = alloc_reg();
            asm_mov_reg_reg(cg_sec, r3, r2, 8); // mov rr2 -> rr3
            if (bo > 0)
                asm_shr_imm(cg_sec, r3, 8, (uint8_t)(bo)); // shr $(uint8_t)(bo), rr3
            int load_bits = eff_sz * 8;
            if (bw < load_bits) {
                if (mem->ty->is_unsigned || mem->ty->is_enum) {
                    unsigned long long m = (1ULL << bw) - 1;
                    asm_movabs_phy(cg_sec, X86_RAX, (uint64_t)(m)); // movabs $(uint64_t)(m), rX86_RAX
                    asm_and_rax(cg_sec, r3, 8); // andq %rax, rr3
                } else {
                    int shift = 64 - bw;
                    asm_shl_imm(cg_sec, r3, 8, (uint8_t)(shift)); // shl $(uint8_t)(shift), rr3
                    asm_sar_imm(cg_sec, r3, 8, (uint8_t)(shift)); // sar $(uint8_t)(shift), rr3
                }
            }
            // Clear the field bits in container word (r2)
            asm_movabs_phy(cg_sec, X86_RAX, (uint64_t)(~mask)); // movabs $(uint64_t)(~mask), rX86_RAX
            asm_and_rax(cg_sec, r2, 8); // andq %rax, rr2
            // Compute new field value in rn
            VReg rn = alloc_reg();
            asm_mov_reg_reg(cg_sec, rn, r3, 8); // mov rr3 -> rrn
            asm_movabs_phy(cg_sec, X86_RAX, (uint64_t)((1ULL << bw) - 1)); // movabs mask, %rax
            asm_and_rax(cg_sec, rn, 8); // andq %rax, rrn (mask to bw bits)
            if (is_inc)
                asm_add_imm(cg_sec, rn, 8, 1); // addq $1, rn
            else
                asm_sub_imm(cg_sec, rn, 8, 1); // subq $1, rn
            asm_movabs_phy(cg_sec, X86_RAX, (uint64_t)((1ULL << bw) - 1)); // movabs mask, %rax
            asm_and_rax(cg_sec, rn, 8); // andq %rax, rrn (wrap)
            // For prefix forms, sign/zero-extend the new field value
            VReg r4 = R_NONE;
            if (is_pre) {
                r4 = alloc_reg();
                asm_mov_reg_reg(cg_sec, r4, rn, 8); // mov rrn -> rr4
                if (bw < load_bits) {
                    if (!(mem->ty->is_unsigned || mem->ty->is_enum)) {
                        int shift = 64 - bw;
                        asm_shl_imm(cg_sec, r4, 8, (uint8_t)(shift)); // shl $(uint8_t)(shift), rr4
                        asm_sar_imm(cg_sec, r4, 8, (uint8_t)(shift)); // sar $(uint8_t)(shift), rr4
                    }
                }
            }
            if (bo > 0)
                asm_shl_imm(cg_sec, rn, 8, (uint8_t)(bo)); // shl $(uint8_t)(bo), rrn
            asm_or_reg_reg(cg_sec, rn, r2, 8); // or rrn, rr2
            // Store
            if (eff_sz == 1)
                asm_mov_reg_mem(cg_sec, rn, r, 1); // movb rn, (%r)
            else if (eff_sz == 2)
                asm_mov_reg_mem(cg_sec, rn, r, 2); // movw rn, (%r)
            else if (eff_sz == 4)
                asm_mov_reg_mem(cg_sec, rn, r, 4); // movl rn, (%r)
            else
                asm_mov_reg_mem(cg_sec, rn, r, 8); // movq rn, (%r)
#endif
            free_reg(rn);
            free_reg(r2);
            free_reg(r);
            if (is_pre) {
                free_reg(r3);
                return r4; // Return new value (prefix semantics)
            }
            if (r4 != R_NONE)
                free_reg(r4);
            return r3; // Return original value (post-increment semantics)
        }
#ifdef ARCH_ARM64
        // Load current value (correct load width for type)
        emit_load(node->lhs->ty, r2, r, 0);
        // Update in-place: load into temp, add/sub, store back
        VReg r3 = alloc_reg();
        emit_load(node->lhs->ty, r3, r, 0);
        if (is_flonum(node->lhs->ty)) {
            // Float inc/dec: load r3 bits into d0/s0, add/sub 1.0, store back.
            int id = add_float_literal(1.0, sz);
            int tmp = alloc_reg();
            emit_adrp_add(tmp, format(".LF%d", id));
            if (sz == 4) {
                asm_fmov_i2f(cg_sec, 0, r3, 0); // fmov s0, w{r3}
                asm_ldr_fp(cg_sec, 1, tmp, 4); // ldr s1, [x{tmp}]
                (is_inc ? asm_fadd(cg_sec, 0) : asm_fsub(cg_sec, 0)); // fadd/fsub s0, s0, s1
                asm_fmov_f2i(cg_sec, r3, 0, 0); // fmov w{r3}, s0
            } else {
                asm_fmov_i2f(cg_sec, 0, r3, 1); // fmov d0, x{r3}
                asm_ldr_fp(cg_sec, 1, tmp, 8); // ldr d1, [x{tmp}]
                (is_inc ? asm_fadd(cg_sec, 1) : asm_fsub(cg_sec, 1)); // fadd/fsub d0, d0, d1
                asm_fmov_f2i(cg_sec, r3, 0, 1); // fmov x{r3}, d0
            }
            free_reg(tmp);
        } else {
            int delta = 1;
            if (node->lhs->ty->kind == TY_PTR || node->lhs->ty->kind == TY_ARRAY)
                delta = node->lhs->ty->base->size;
            if (is_inc)
                asm_add_imm(cg_sec, r3, sz, delta); // add r3, r3, #delta
            else
                asm_sub_imm(cg_sec, r3, sz, delta); // sub r3, r3, #delta
            // Narrow unsigned/signed types: extend so that comparisons against
            // the returned register (e.g. --z > 0, unsigned char z) see the
            // correctly truncated value, not the raw 32-bit add/sub result.
            if (node->lhs->ty->size < 4) {
                if (node->lhs->ty->is_unsigned)
                    zero_extend_to(r3, node->lhs->ty->size, 4);
                else
                    sign_extend_to(r3, node->lhs->ty->size, 4);
            }
        }
        emit_store(node->lhs->ty, r3, r, 0);
        // r3 already holds the new value; r2 holds the original. Return
        // whichever the prefix/postfix semantics require and free the other.
        if (is_pre) {
            free_reg(r2);
            r2 = r3;
        } else {
            free_reg(r3);
        }
#else
        // Sign/zero-extend narrow loads into the full register — rcc keeps
        // sub-int values extended in their register, and a plain narrow `mov`
        // would leave garbage in the upper bits, breaking direct uses of the
        // result (e.g. `x-- < 0` or `--x < 0`).
        emit_load(node->lhs->ty, r2, r, 0); // mov/movsx/movzx (%r), r2
        bool is_float = is_flonum(node->lhs->ty);
        if (is_float) {
            int id = add_float_literal(1.0, sz);
            asm_movq_r_xmm(cg_sec, X86_XMM0, r2); // movq r2, %xmm0
            VReg tmp = alloc_reg();
            asm_lea_rip_reg(cg_sec, tmp, format(".LF%d", id)); // lea .LFid(%rip), tmp
            if (sz == 4) {
                x86_movss_rm(cg_sec, X86_XMM1, x86_mem(REG(tmp), 0)); // movss (tmp), %xmm1
                if (is_inc) x86_addss(cg_sec, X86_XMM0, X86_XMM1); // addss %xmm1, %xmm0
                else
                    x86_subss(cg_sec, X86_XMM0, X86_XMM1); // subss %xmm1, %xmm0
            } else {
                x86_movsd_rm(cg_sec, X86_XMM1, x86_mem(REG(tmp), 0)); // movsd (tmp), %xmm1
                if (is_inc) x86_addsd(cg_sec, X86_XMM0, X86_XMM1); // addsd %xmm1, %xmm0
                else
                    x86_subsd(cg_sec, X86_XMM0, X86_XMM1); // subsd %xmm1, %xmm0
            }
            free_reg(tmp);
            if (sz == 4)
                x86_movss_mr(cg_sec, x86_mem(REG(r), 0), X86_XMM0); // movss %xmm0, (%r)
            else
                x86_movsd_mr(cg_sec, x86_mem(REG(r), 0), X86_XMM0); // movsd %xmm0, (%r)
        } else {
            int delta = 1;
            if (node->lhs->ty->kind == TY_PTR || node->lhs->ty->kind == TY_ARRAY)
                delta = node->lhs->ty->base->size;
            if (is_inc)
                x86_add_mi(cg_sec, sz, x86_mem(REG(r), 0), delta); // add%c $delta, (%r)
            else
                x86_sub_mi(cg_sec, sz, x86_mem(REG(r), 0), delta); // sub%c $delta, (%r)
        }
        // The new value lives only in memory now (computed in-place); for
        // prefix semantics, reload it (sign/zero-extended) into r2.
        if (is_pre)
            emit_load(node->lhs->ty, r2, r, 0);
#endif
        free_reg(r);
        return r2;
    }
    case ND_MEMBER: {
        VReg r = gen_addr(node);
        if (node->ty->kind == TY_ARRAY || node->ty->kind == TY_VLA) {
            return r; // array/VLA decays to pointer
        }
        Type *load_ty = (node->member && node->member->bit_width > 0) ? node->member->ty : node->ty;
        if (is_flonum(load_ty)) {
#ifdef ARCH_ARM64
            if (load_ty->size == 4) {
                asm_ldr_fp(cg_sec, 0, r, 4); // ldr s0, [x{r}]
                asm_fcvt(cg_sec, 1, 0, 0, 0); // fcvt d0, s0
            } else {
                asm_ldr_fp(cg_sec, 0, r, 8); // ldr d0, [x{r}]
            }
            asm_fmov_f2i(cg_sec, r, 0, 1); // fmov x{r}, d0
#else
            if (load_ty->size == 4) {
                x86_movss_rm(cg_sec, X86_XMM0, x86_mem(REG(r), 0)); // movss (%s), %%xmm0
                asm_cvtss2sd(cg_sec); // cvtss2sd %%xmm0, %%xmm0
            } else {
                x86_movsd_rm(cg_sec, X86_XMM0, x86_mem(REG(r), 0)); // movq %%xmm0, %s
            }
            asm_movq_xmm_r(cg_sec, r, X86_XMM0); // movq x{r}, xmm0
#endif
        } else if (node->member && node->member->bit_width > 0 && node->member->bf_load_size) {
            int ls = node->member->bf_load_size;
            int bw = node->member->bit_width;
            int bo = node->member->bit_offset;
            int eff_ls = ls > 8 ? 8 : ls;
            VReg r_addr = -1;
            if (ls > 8 && bo + bw > 64) {
                r_addr = alloc_reg();
#ifdef ARCH_ARM64
                asm_mov_reg_reg(cg_sec, r_addr, r, 8); // mov rr -> rr_addr
#else
                asm_mov_reg_reg(cg_sec, r_addr, r, 8); // mov rr -> rr_addr
#endif
            }
#ifdef ARCH_ARM64
            if (eff_ls == 1)
                asm_ldrb_uoff(cg_sec, r, r, 0); // ldrb w{r}, [x{r}]
            else if (eff_ls == 2)
                asm_ldrh_uoff(cg_sec, r, r, 0); // ldrh w{r}, [x{r}]
            else if (eff_ls == 4)
                asm_ldr_reg_off(cg_sec, r, r, 4, 0); // ldr w{r}, [x{r}]
            else
                asm_ldr_reg_off(cg_sec, r, r, 8, 0); // ldr x{r}, [x{r}]
            if (bo > 0)
                asm_shr_imm(cg_sec, r, 8, (uint8_t)(bo)); // lsr x{r}, x{r}, #bo
            if (r_addr >= 0) {
                int overflow = bo + bw - 64;
                int tmp = alloc_reg();
                asm_ldrb_uoff(cg_sec, tmp, r_addr, 8); // ldrb w{tmp}, [x{r_addr}, #8]
                int ovf_mask = (1 << overflow) - 1;
                asm_and_imm(cg_sec, tmp, 4, ovf_mask); // and w{tmp}, w{tmp}, #ovf_mask
                asm_shl_imm(cg_sec, tmp, 8, (uint8_t)(64 - bo)); // lsl x{tmp}, x{tmp}, #(64-bo)
                asm_or_reg_reg(cg_sec, r, tmp, 8); // orr x{r}, x{r}, x{tmp}
                free_reg(tmp);
                free_reg(r_addr);
            }
            int load_bits = ls * 8;
            if (bw < load_bits) {
                if (node->member->ty->is_unsigned || node->member->ty->is_enum) {
                    unsigned long long mask = (1ULL << bw) - 1;
                    emit_mov_imm64(ARM64_X16, mask);
                    asm_and_reg_phy(cg_sec, r, ARM64_X16, 8); // and rr, r16
                } else {
                    int shift = 64 - bw;
                    asm_shl_imm(cg_sec, r, 8, (uint8_t)(shift)); // lsl x{r}, x{r}, #shift
                    asm_sar_imm(cg_sec, r, 8, (uint8_t)(shift)); // asr x{r}, x{r}, #shift
                }
            }
            return r;
#else
            {
                asm_mov_mem_reg(cg_sec, r, r, eff_ls); // movl/movq (%r), rr
            }
            if (bo > 0)
                asm_shr_imm(cg_sec, r, 8, (uint8_t)(bo)); // shr $(uint8_t)(bo), rr
            if (r_addr >= 0) {
                int overflow = bo + bw - 64;
                int tmp = alloc_reg();
                asm_movzx_base_off_reg(cg_sec, tmp, r_addr, 8, 8, 1); // movzbl 8(r_addr), tmp
                unsigned long long mask = (1ULL << overflow) - 1;
                asm_movabs_phy(cg_sec, X86_RAX, (uint64_t)(mask)); // movabs $(uint64_t)(mask), rX86_RAX
                asm_and_rax(cg_sec, tmp, 8); // andq %rax, rtmp
                asm_shl_imm(cg_sec, tmp, 8, (uint8_t)(64 - bo)); // shl $(uint8_t)(64 - bo), rtmp
                asm_or_reg_reg(cg_sec, tmp, r, 8); // or rtmp, rr
                free_reg(tmp);
                free_reg(r_addr);
            }
            int load_bits = ls * 8;
            if (bw < load_bits) {
                if (node->member->ty->is_unsigned || node->member->ty->is_enum) {
                    unsigned long long mask = (1ULL << bw) - 1;
                    asm_movabs_phy(cg_sec, X86_RAX, (uint64_t)(mask)); // movabs $(uint64_t)(mask), rX86_RAX
                    asm_and_rax(cg_sec, r, 8); // andq %rax, rr
                } else {
                    int shift = 64 - bw;
                    asm_shl_imm(cg_sec, r, 8, (uint8_t)(shift)); // shl $(uint8_t)(shift), rr
                    asm_sar_imm(cg_sec, r, 8, (uint8_t)(shift)); // sar $(uint8_t)(shift), rr
                }
            }
            return r;
#endif
        } else {
#ifdef ARCH_ARM64
            emit_load(load_ty, r, r, 0);
#else
            emit_load(load_ty, r, r, 0);
#endif
        }
        if (node->member && node->member->bit_width > 0) {
            int bw = node->member->bit_width;
            int bo = node->member->bit_offset;
            int load_bits = node->member->bf_load_size
                ? node->member->bf_load_size * 8
                : node->member->ty->size * 8;
#ifdef ARCH_ARM64
            if (bo > 0)
                asm_shr_imm(cg_sec, r, 8, (uint8_t)(bo)); // lsr x{r}, x{r}, #bo
            if (bw < load_bits) {
                if (node->member->ty->is_unsigned || node->member->ty->is_enum) {
                    unsigned long long mask = (1ULL << bw) - 1;
                    emit_mov_imm64(ARM64_X16, mask);
                    asm_and_reg_phy(cg_sec, r, ARM64_X16, 8); // and x{r}, x{r}, x16
                } else {
                    int shift = 64 - bw;
                    asm_shl_imm(cg_sec, r, 8, (uint8_t)(shift)); // lsl x{r}, x{r}, #shift
                    asm_sar_imm(cg_sec, r, 8, (uint8_t)(shift)); // asr x{r}, x{r}, #shift
                }
            }
#else
            if (bo > 0)
                asm_shr_imm(cg_sec, r, 8, (uint8_t)(bo)); // shr $(uint8_t)(bo), rr
            if (bw < load_bits) {
                if (node->member->ty->is_unsigned || node->member->ty->is_enum) {
                    unsigned long long mask = (1ULL << bw) - 1;
                    asm_movabs_phy(cg_sec, X86_RAX, (uint64_t)(mask)); // movabs $(uint64_t)(mask), rX86_RAX
                    asm_and_rax(cg_sec, r, 8); // andq %rax, rr
                } else {
                    int shift = 64 - bw;
                    asm_shl_imm(cg_sec, r, 8, (uint8_t)(shift)); // shl $(uint8_t)(shift), rr
                    asm_sar_imm(cg_sec, r, 8, (uint8_t)(shift)); // sar $(uint8_t)(shift), rr
                }
            }
#endif
        }
        return r;
    }
    case ND_ADDR:
        return gen_addr(node->lhs);
    case ND_CAST: {
        if (node->ty && node->ty->is_vector)
            return gen_addr(node); // vectors are slot-resident
        // 8-byte vector -> scalar bitcast (e.g. `(long long)__m64`):
        // the vector VALUE is a slot address, so load the bytes out.
        if (node->lhs && node->lhs->ty && node->lhs->ty->is_vector &&
            node->lhs->ty->size <= 8 && !is_complex(node->ty)) {
            VReg a = gen_addr(node->lhs);
            VReg r = alloc_reg();
#ifndef ARCH_ARM64
            x86_mov_rm(cg_sec, node->lhs->ty->size <= 4 ? 4 : 8, REG(r), x86_mem(REG(a), 0));
#else
            arm64_ldr_uoff(cg_sec, node->lhs->ty->size <= 4 ? 2 : 3, REG(r), REG(a), 0);
#endif
            free_reg(a);
            return r;
        }
        // Decimal casts: _Decimal* <-> integer / binary float / _Decimal*.
        // The result type may be non-decimal (int/float), so dispatch on
        // operand types here before the generic scalar path.
        if ((node->lhs && node->lhs->ty && is_decimal(node->lhs->ty)) ||
            (node->ty && is_decimal(node->ty)))
            return gen_decimal(node);
        VReg r = gen(node->lhs);
        return gen_cast_reg(r, node->lhs->ty, node->ty);
    }
    case ND_BITNOT: {
#ifndef ARCH_ARM64
        if (is_complex(node->ty)) {
            int complex_sz = node->ty->size, base_sz = node->ty->base->size;
            VReg addr = gen_addr(node->lhs), need_free = 0;
            if (addr < 0) {
                need_free = 1;
                addr = alloc_reg();
                asm_sub_rsp_imm(cg_sec, (complex_sz + 15) & ~15); // subq $N, %rsp
                x86_mov_rr(cg_sec, 8, REG(addr), X86_RSP); // movq %rsp, addr
                VReg v = gen(node->lhs);
                if (is_complex(node->lhs->ty)) {
                    x86_mov_rm(cg_sec, 8, X86_RAX, x86_mem(REG(v), 0)); // movq (v), %rax
                    x86_mov_mr(cg_sec, 8, x86_mem(REG(addr), 0), X86_RAX); // movq %rax, (addr)
                    if (complex_sz > 8) {
                        x86_mov_rm(cg_sec, 8, X86_RAX, x86_mem(REG(v), base_sz)); // movq base_sz(v), %rax
                        x86_mov_mr(cg_sec, 8, x86_mem(REG(addr), base_sz), X86_RAX); // movq %rax, base_sz(addr)
                    }
                } else {
                    x86_mov_mr(cg_sec, 8, x86_mem(REG(addr), 0), REG(v)); // movq v, (addr)
                    if (complex_sz > 8) asm_movq_zero(cg_sec, addr);
                }
                free_reg(v);
            }
            int alloc = (complex_sz + 7) & ~7;
            fn_struct_ret_off += alloc;
            if (fn_struct_ret_off > fn_struct_ret_total) fn_struct_ret_total = fn_struct_ret_off;
            int result_off = current_fn_stack_size + fn_struct_ret_off;
            VReg result = alloc_reg();
            asm_lea_rbp_reg(cg_sec, result, 8, result_off);
            if (is_flonum(node->ty->base)) {
                asm_mov_fp_rm(cg_sec, base_sz, X86_XMM0, x86_mem(REG(addr), 0)); // movss/sd (addr), %xmm0
                asm_mov_fp_rm(cg_sec, base_sz, X86_XMM1, x86_mem(REG(addr), base_sz)); // movss/sd base_sz(addr), %xmm1
                asm_movabs_phy(cg_sec, X86_RAX, 0x8000000000000000ULL); // movabs $0x8000000000000000, %rax
                x86_movq_r_xmm(cg_sec, X86_XMM2, X86_RAX); // movq %rax, %xmm2
                if (base_sz == 4) x86_xorps(cg_sec, X86_XMM1, X86_XMM2); // xorps %xmm2, %xmm1
                else
                    x86_xorpd(cg_sec, X86_XMM1, X86_XMM2); // xorpd %xmm2, %xmm1
                asm_mov_fp_mr(cg_sec, base_sz, x86_mem(REG(result), 0), X86_XMM0); // movss/sd %xmm0, (result)
                asm_mov_fp_mr(cg_sec, base_sz, x86_mem(REG(result), base_sz), X86_XMM1); // movss/sd %xmm1, base_sz(result)
            } else {
                x86_mov_rm(cg_sec, 8, X86_RAX, x86_mem(REG(addr), 0)); // mov (addr), %rax
                x86_mov_rm(cg_sec, 8, X86_RDX, x86_mem(REG(addr), base_sz)); // mov base_sz(addr), %rdx
                x86_neg_r(cg_sec, 8, X86_RDX); // neg %rdx
                x86_mov_mr(cg_sec, 8, x86_mem(REG(result), 0), X86_RAX); // mov %rax, (result)
                x86_mov_mr(cg_sec, 8, x86_mem(REG(result), base_sz), X86_RDX); // mov %rdx, base_sz(result)
            }
            free_reg(addr);
            if (need_free) asm_add_rsp_imm(cg_sec, (complex_sz + 15) & ~15); // addq $N, %rsp
            return result;
        }
#endif
        VReg r = gen(node->lhs);
#ifdef ARCH_ARM64
        asm_not(cg_sec, r, 8); // not rr
#else
        asm_not(cg_sec, r, node->ty->size); // not rr
#endif
        return r;
    }
    case ND_STR: {
        VReg r = alloc_reg();
#ifdef ARCH_ARM64
        emit_adrp_add(r, format(".LC%d", node->str_id));
#else
        asm_lea_rip_reg(cg_sec, r, format(".LC%d", node->str_id)); // lea rip, rr
#endif
        return r;
    }
    case ND_DEREF: {
        if (node->ty->kind == TY_FUNC || node->ty->kind == TY_ARRAY || node->ty->kind == TY_VLA)
            return gen(node->lhs);
        if (node->lhs->kind == ND_ADD && node->lhs->lhs->ty &&
            node->lhs->lhs->ty->base && !is_flonum(node->ty) &&
            node->lhs->lhs->kind == ND_LVAR) {
            VReg idx = gen(node->lhs->rhs);
            VReg base = gen(node->lhs->lhs);
            // When register pressure causes idx and base to share the
            // same VReg, idx's original value is in its spill slot while
            // the register now holds base.  Reload idx before the add,
            // otherwise asm_add_reg_reg emits a self-add (e.g. add %rsi,%rsi).
            if (idx == base && (spilled_regs & (1 << idx))) {
#ifdef ARCH_ARM64
                asm_ldur_fp_phy(cg_sec, ARM64_X16, spill_offset(idx)); // ldr x16, [x29, #-spill]
                spilled_regs &= ~(1 << idx);
                arm64_add_reg(cg_sec, 1, REG(base), REG(base), ARM64_X16, ARM64_LSL, 0); // add base, base, x16
                free_reg(idx); // idx and base share a VReg
                emit_load(node->ty, base, base, 0);
                return base;
#else
                // On x86-64 the register holds base after the spill;
                // add idx's spilled value directly from the spill slot.
                asm_add_spill_reg(cg_sec, idx, 8, spill_offset(idx)); // add spill, reg
                spilled_regs &= ~(1 << idx);
                // idx and base are the same VReg; free once (via idx).
                free_reg(idx);
                emit_load(node->ty, base, base, 0);
                return base;
#endif
            }
#ifdef ARCH_ARM64
            asm_add_reg_reg(cg_sec, base, idx, 8); // add base, base, idx
            free_reg(idx);
            // For aggregate types, return the address, not the loaded value
            if (node->ty->kind != TY_STRUCT && node->ty->kind != TY_UNION &&
                node->ty->kind != TY_COMPLEX)
                emit_load(node->ty, base, base, 0); // load from [base]
            return base;
#else
            asm_add_reg_reg(cg_sec, idx, base, 8); // add idx, base
            free_reg(base);
            // For aggregate types, return the address, not the loaded value
            if (node->ty->kind != TY_STRUCT && node->ty->kind != TY_UNION &&
                node->ty->kind != TY_COMPLEX)
                emit_load(node->ty, idx, idx, 0); // load from (idx)
            return idx;
#endif
        }
        VReg r = gen(node->lhs);
        if (is_flonum(node->ty)) {
#ifdef ARCH_ARM64
            if (node->ty->size == 4) {
                asm_ldr_fp(cg_sec, 0, r, 4); // asm_ldr_fp(0, r, 4)
                asm_fcvt(cg_sec, 1, 0, 0, 0); // fcvt d0, s0
            } else {
                asm_ldr_fp(cg_sec, 0, r, 8); // asm_ldr_fp(0, r, 8)
            }
            asm_fmov_f2i(cg_sec, r, 0, 1); // asm_fmov_f2i(r, 0, 1)
#else
            if (node->ty->size == 4) {
                asm_ldr_fp(cg_sec, 0, r, 4); // movss (%r), %xmm0
                asm_cvtss2sd(cg_sec); // cvtss2sd %xmm0, %xmm0
            } else {
                asm_ldr_fp(cg_sec, 0, r, 8); // movsd (%r), %xmm0
            }
            asm_movq_xmm_r(cg_sec, r, X86_XMM0); // asm_movq_xmm_r(r, X86_XMM0)
#endif
        } else {
#ifdef ARCH_ARM64
            emit_load(node->ty, r, r, 0);
#else
            emit_load(node->ty, r, r, 0);
#endif
        }
        return r;
    }
    case ND_RETURN: {
        if (node->lhs) {
            // Returning a _Complex value from a function whose declared
            // return type is a plain scalar (GNU extension: discards the
            // imaginary part). Wrap in an ND_CAST to the return type so the
            // scalar-return path below loads just the real component,
            // instead of mistaking this for a complex-returning function
            // and copying raw bytes into __retbuf.
            Type *ret_ty = current_fn_def->ty->return_ty;
            if (node->lhs->ty && is_complex(node->lhs->ty) && ret_ty &&
                !is_complex(ret_ty) && ret_ty->kind != TY_STRUCT && ret_ty->kind != TY_UNION) {
                Node *cast = arena_alloc(sizeof(Node));
                cast->kind = ND_CAST;
                cast->lhs = node->lhs;
                cast->ty = ret_ty;
                cast->tok = node->lhs->tok;
                node->lhs = cast;
            }
            // Implicit conversion from the expression type to the function's
            // declared return type. The parser does not insert this cast, so
            // we must perform it here to get correct sign/zero extension.
            if (node->lhs->ty && ret_ty &&
                ((is_integer(node->lhs->ty) && is_integer(ret_ty)) ||
                 (is_flonum(node->lhs->ty) && is_flonum(ret_ty))) &&
                (node->lhs->ty->size != ret_ty->size || node->lhs->ty->is_unsigned != ret_ty->is_unsigned)) {
                Node *cast = arena_alloc(sizeof(Node));
                cast->kind = ND_CAST;
                cast->lhs = node->lhs;
                cast->ty = ret_ty;
                cast->tok = node->lhs->tok;
                node->lhs = cast;
            }
            if (node->lhs->ty && is_int128_like(node->lhs->ty)) {
                // Return int128 (or _BitInt(65..128)) in rax:rdx (lo:hi) on
                // x86-64, x0:x1 on ARM64
                int src = gen(node->lhs); // dispatches to gen_int128/gen_bitint
#ifdef ARCH_ARM64
                arm64_ldr_uoff(cg_sec, 3, ARM64_X0, REG(src), 0); // ldr x0, [src]
                arm64_ldr_uoff(cg_sec, 3, ARM64_X1, REG(src), 1); // ldr x1, [src, #8]
                free_reg(src);
#else
                x86_mov_rm(cg_sec, 8, X86_RAX, x86_mem(REG(src), 0)); // movq (src), %rax
                x86_mov_rm(cg_sec, 8, X86_RDX, x86_mem(REG(src), 8)); // movq 8(src), %rdx
                free_reg(src);
#endif
#ifdef _WIN32
            } else if (node->lhs->ty && is_complex(node->lhs->ty) && node->lhs->ty->size <= 8 && current_fn_def && current_fn_def->ty &&
                       is_complex(current_fn_def->ty->return_ty) && current_fn_def->ty->return_ty->size <= 8) {
                // Win64: an 8-byte _Complex return value is returned by value
                // in RAX — there is no hidden return pointer for it (see the
                // param_index computation in the prologue).
                int src = gen_addr(node->lhs);
                if (src < 0)
                    src = gen(node->lhs);
                x86_mov_rm(cg_sec, 8, X86_RAX, x86_mem(REG(src), 0)); // movq (src), %rax
                free_reg(src);
#endif
#if !defined(_WIN32) && !defined(ARCH_ARM64)
            } else if (node->lhs->ty && is_complex(node->lhs->ty) && current_fn_def && current_fn_def->ty &&
                       is_complex(current_fn_def->ty->return_ty)) {
                // Linux SysV: return _Complex double in xmm0:xmm1,
                // _Complex float in xmm0 (packed).
                int src = gen_addr(node->lhs);
                if (src < 0) src = gen(node->lhs);
                int base_sz = node->lhs->ty->base ? node->lhs->ty->base->size : 8;
                if (node->lhs->ty->size <= 8) {
                    x86_movsd_rm(cg_sec, X86_XMM0, x86_mem(REG(src), 0));
                } else {
                    x86_movsd_rm(cg_sec, X86_XMM0, x86_mem(REG(src), 0));
                    x86_movsd_rm(cg_sec, X86_XMM1, x86_mem(REG(src), base_sz));
                }
                free_reg(src);
#endif
#if !defined(_WIN32) && !defined(ARCH_ARM64)
            } else if (current_fn_def && current_fn_def->ty && struct_returns_in_gp_regs(current_fn_def->ty->return_ty)) {
                // Linux/SysV x86-64: a struct/union <=16 bytes with no
                // floating member (struct_returns_in_gp_regs) returns in
                // RAX (first eightbyte) and, if bigger than 8 bytes, RDX
                // (second eightbyte) -- no hidden return pointer. Mirrors
                // the has_retbuf/param_index classification in the
                // function prologue and gen_funcall's call-site handling.
                int src = gen_addr(node->lhs);
                if (src < 0) src = gen(node->lhs);
                int64_t sz = current_fn_def->ty->return_ty->size;
                x86_mov_rm(cg_sec, sz <= 4 ? 4 : 8, X86_RAX, x86_mem(REG(src), 0)); // mov (src), %rax
                if (sz > 8)
                    x86_mov_rm(cg_sec, sz - 8 <= 4 ? 4 : 8, X86_RDX, x86_mem(REG(src), 8)); // mov 8(src), %rdx
                free_reg(src);
#endif
#ifdef ARCH_ARM64
            } else if (current_fn_def && current_fn_def->ty && struct_returns_in_gp_regs(current_fn_def->ty->return_ty)) {
                // AAPCS64: a struct/union <=16 bytes with no floating
                // member (struct_returns_in_gp_regs) returns raw bits in
                // X0 (first eightbyte) and, if bigger than 8 bytes, X1
                // (second eightbyte) -- no indirect result register (X8).
                // Mirrors the has_hidden_retbuf/prologue classification
                // and gen_funcall's call-site handling.
                int src = gen_addr(node->lhs);
                if (src < 0) src = gen(node->lhs);
                int64_t sz = current_fn_def->ty->return_ty->size;
                int lo_sz = sz <= 4 ? 4 : 8;
                arm64_ldr_uoff(cg_sec, lo_sz <= 4 ? 2 : 3, ARM64_X0, REG(src), 0); // ldr x0, [src]
                if (sz > 8) {
                    int hi_sz = sz - 8 <= 4 ? 4 : 8;
                    arm64_ldr_uoff(cg_sec, hi_sz <= 4 ? 2 : 3, ARM64_X1, REG(src), 8 / hi_sz); // ldr x1, [src, #8]
                }
                free_reg(src);
#endif
            } else if (node->lhs->ty && (node->lhs->ty->kind == TY_STRUCT || node->lhs->ty->kind == TY_UNION || is_complex(node->lhs->ty) || (node->lhs->ty->kind == TY_BITINT && node->lhs->ty->size > 16)) && current_fn_def && current_fn_def->ty && (current_fn_def->ty->return_ty->kind == TY_STRUCT || current_fn_def->ty->return_ty->kind == TY_UNION || is_complex(current_fn_def->ty->return_ty) || (current_fn_def->ty->return_ty->kind == TY_BITINT && current_fn_def->ty->return_ty->size > 16))) {
                int src = gen_addr(node->lhs);
                if (src < 0)
                    src = gen(node->lhs);
                int c = ++rcc_label_count;
                int retbuf_offset = 0;
                for (LVar *var = current_fn_def->locals; var; var = var->next) {
                    if (var->name && var->name == kw_retbuf) {
                        retbuf_offset = var->offset;
                        break;
                    }
                }
#ifdef ARCH_ARM64
                // Save src to x12 before loading retbuf into x11
                asm_mov_phy_reg(cg_sec, ARM64_X12, src, 1); // mov x12, src
                if (retbuf_offset <= 4095)
                    asm_ldur_phy(cg_sec, ARM64_X11, ARM64_X29, 3, -retbuf_offset); // ldur x11, [x29, #-retbuf_offset]
                else {
                    emit_mov_imm64(ARM64_X16, (uint64_t)retbuf_offset); // mov x16, #retbuf_offset
                    asm_sub_x11_fp_x16(cg_sec); // sub x11, x29, x16
                }
                emit_mov_imm64(ARM64_X9, (uint64_t)node->lhs->ty->size); // mov x9, #size
                cg_def_label(format(".L.retcopy.%d", c));
                arm64_subs_imm(cg_sec, 1, ARM64_XZR, ARM64_X9, 0, 0); // cmp x9, #0
                {
                    size_t _cj = asm_jcc_label(cg_sec, ARM64_EQ);
                    asm_fixup_add(cg_sec, _cj, format(".L.retcopy_end.%d", c), 1);
                }
                arm64_sub_imm(cg_sec, 1, ARM64_X9, ARM64_X9, 1, 0); // sub x9, x9, #1
                asm_ldrb_w16_x9_phy(cg_sec, ARM64_X12); // ldrb w16, [x12, x9]
                asm_strb_w16_x9_phy(cg_sec, ARM64_X11); // strb w16, [x11, x9]
                {
                    size_t _jmp = asm_jmp_label(cg_sec);
                    asm_fixup_add(cg_sec, _jmp, format(".L.retcopy.%d", c), 0);
                }
                cg_def_label(format(".L.retcopy_end.%d", c));
                asm_mov_x0_reg(cg_sec, ARM64_X11); // mov x0, x11
#else
#ifdef _WIN32
                if (node->lhs->ty->size <= 8) {
                    // Win64: struct ≤8 bytes returned by value in RAX.
                    // Load the struct value into RAX directly, no sret copy.
                    int sz = node->lhs->ty->size <= 4 ? 4 : 8;
                    x86_mov_rm(cg_sec, sz, X86_RAX, x86_mem(REG(src), 0)); // movl/movq (src), %rax
                } else
#endif
                {
                    asm_mov_rbp(cg_sec, X86_R9, 8, retbuf_offset); // mov [rbp-retbuf_offset], X86_R9
                    x86_mov_ri(cg_sec, 8, X86_RCX, node->lhs->ty->size); // movq $size, %%rcx
                    cg_def_label(format(".L.retcopy.%d", c));
                    x86_cmp_ri(cg_sec, 8, X86_RCX, 0); // cmp $0, rcx
                    {
                        size_t o = asm_jcc_label(cg_sec, X86_E);
                        asm_fixup_add(cg_sec, o, format(".L.retcopy_end.%d", c), 1);
                    }
                    x86_mov_rm(cg_sec, 1, X86_RAX, x86_mem_idx(REG(src), X86_RCX, 1, -1)); // movb -1(src, rcx), %%al
                    x86_mov_mr(cg_sec, 1, x86_mem_idx(X86_R9, X86_RCX, 1, -1), X86_RAX); // movb %%al, -1(r9, rcx)
                    x86_sub_ri(cg_sec, 8, X86_RCX, 1); // dec rcx
                    {
                        size_t o = asm_jmp_label(cg_sec);
                        asm_fixup_add(cg_sec, o, format(".L.retcopy.%d", c), 0);
                    }
                    cg_def_label(format(".L.retcopy_end.%d", c));
                    x86_mov_rr(cg_sec, 8, X86_RAX, X86_R9); // movq %%r9, %%rax
                }
                free_reg(src);
#endif
            } else if (node->lhs->ty && is_complex(node->lhs->ty)) {
                // Complex expression returned as scalar: extract real part
                int addr = gen_addr(node->lhs);
                if (addr < 0) addr = gen(node->lhs);
                Type *cplx_ret_ty = current_fn_def->ty->return_ty;
#ifdef ARCH_ARM64
                arm64_ldr_uoff(cg_sec, 2, ARM64_X0, REG(addr), 0); // ldr w0, [addr]
#else
                x86_mov_rm(cg_sec, 4, X86_RAX, x86_mem(REG(addr), 0)); // movl (addr), %eax
#endif
                if (cplx_ret_ty && cplx_ret_ty->size == 1) {
#ifdef ARCH_ARM64
                    arm64_uxtb(cg_sec, ARM64_X0, ARM64_X0); // uxtb w0, w0
#else
                    asm_movzx(cg_sec, 0, 0, 4, 1); // movzbl %al, %eax
#endif
                }
                free_reg(addr);
            } else {
                VReg r = gen(node->lhs);
                Type *scalar_ret_ty = current_fn_def->ty->return_ty;
                if (scalar_ret_ty && is_flonum(scalar_ret_ty)) {
                    if (node->lhs->ty && is_flonum(node->lhs->ty)) {
#ifdef ARCH_ARM64
                        asm_fmov_i2f(cg_sec, 0, r, 1); // fmov d0, x{r}
                        if (scalar_ret_ty->kind == TY_LDOUBLE) {
#ifndef __APPLE__
                            // Linux AAPCS64 only: Apple's arm64 ABI
                            // defines `long double` as identical to
                            // `double` (type.c's ty_ldouble, size=8
                            // under __APPLE__) -- no widening needed,
                            // and calling __extenddftf2 on an
                            // already-plain-double value would corrupt
                            // it. On Linux, AAPCS64 returns `long
                            // double` as a genuine 128-bit binary128
                            // quad in v0, not a plain double in d0
                            // (mirrors the identical ND_CALL/gen_funcall
                            // widening in the long-double argument
                            // pre-pass above, and the narrowing
                            // counterpart in gen_funcall's own
                            // return-value read). Widen via libgcc's
                            // __extenddftf2, consuming the double just
                            // placed in d0 and leaving the widened quad
                            // in v0 -- exactly where the caller expects
                            // to find the return value.
                            emit_direct_call("__extenddftf2", false); // bl __extenddftf2 (d0 -> quad in v0)
#endif
                        } else if (scalar_ret_ty->kind == TY_FLOAT)
                            asm_fcvt(cg_sec, 0, 1, 0, 0); // fcvt s0, d0 (opc=0=single dest)
#else
                        if (scalar_ret_ty->kind == TY_LDOUBLE) {
                            // SysV x86-64 ABI: a `long double` return
                            // value goes back to the caller in ST0, not
                            // XMM0 -- the generic movq-to-xmm0 path below
                            // is only correct for float/double. Widen
                            // rr's double bit pattern (rcc's internal
                            // long-double representation) onto the x87
                            // stack via a red-zone scratch round-trip:
                            // `fldl` leaves it in ST0, exactly where the
                            // `ret` instruction's caller expects to find
                            // it (mirrors the call-site argument-passing
                            // widen sequence in gen_funcall, and the
                            // narrowing counterpart in gen_funcall's own
                            // ST0-popping return-value read).
                            X86Mem scratch = x86_mem(X86_RSP, -8);
                            x86_mov_mr(cg_sec, 8, scratch, REG(r)); // movq rr, -8(%rsp)
                            x86_fldl_m(cg_sec, scratch); // fldl -8(%rsp)
                        } else {
                            asm_movq_r_xmm(cg_sec, X86_XMM0, r); // movq r, %xmm0
                            if (scalar_ret_ty->kind == TY_FLOAT)
                                asm_cvtsd2ss(cg_sec); // cvtsd2ss %xmm0, %xmm0
                        }
#endif
                    } else if (scalar_ret_ty->size == 4) {
#ifdef ARCH_ARM64
                        {
                            int src_sf = (node->lhs->ty && node->lhs->ty->size >= 8) ? 1 : 0;
                            if (node->lhs->ty && node->lhs->ty->is_unsigned)
                                arm64_ucvtf(cg_sec, src_sf, 0, 0, REG(r)); // ucvtf s0, reg(r, size)
                            else
                                arm64_scvtf(cg_sec, src_sf, 0, 0, REG(r)); // scvtf s0, reg(r, size)
                        }
#else
                        {
                            int src_sz = node->lhs->ty ? node->lhs->ty->size : 4;
                            bool src_u = node->lhs->ty && node->lhs->ty->is_unsigned;
                            if (src_u && src_sz == 8) {
                                // unsigned long long → float: handle high bit
                                int c = ++rcc_label_count;
                                asm_test(cg_sec, REG(r), REG(r), 8); // test r, r
                                {
                                    size_t o = asm_jcc_label(cg_sec, X86_S); // jcc label
                                    asm_fixup_add(cg_sec, o, format(".L.u2f.high.%d", c), 1);
                                }
                                asm_cvtsi2ss(cg_sec, REG(r), 8); // cvtsi2ss rr, %xmm0
                                {
                                    size_t o = asm_jmp_label(cg_sec); // jmp .L.u2f.end.%d
                                    asm_fixup_add(cg_sec, o, format(".L.u2f.end.%d", c), 0);
                                }
                                cg_def_label(format(".L.u2f.high.%d", c)); // .L.u2f.high.%d:
                                x86_mov_rr(cg_sec, 8, X86_RCX, REG(r)); // movq r, %rcx
                                x86_shr_ri(cg_sec, 8, X86_RCX, 1); // shrq $1, %rcx
                                asm_cvtsi2ss(cg_sec, X86_RCX, 8); // cvtsi2ss %rcx, %xmm0
                                x86_addss(cg_sec, X86_XMM0, X86_XMM0); // addss %xmm0, %xmm0 (double it)
                                cg_def_label(format(".L.u2f.end.%d", c)); // .L.u2f.end.%d:
                            } else if (src_u && src_sz <= 4) {
                                // unsigned int/short/char → float: zero-extend to 64-bit,
                                // then cvtsi2ss with 64-bit reg (value is non-negative 64-bit int)
                                asm_cvtsi2ss(cg_sec, REG(r), 8); // cvtsi2ss rr, %xmm0
                            } else {
                                int cssz = (src_sz >= 8) ? 8 : (src_sz < 4 ? 4 : src_sz);
                                asm_cvtsi2ss(cg_sec, REG(r), cssz); // cvtsi2ss rr, %xmm0
                            }
                        }
#endif
                    } else {
#ifdef ARCH_ARM64
                        {
                            int src_sf = (node->lhs->ty && node->lhs->ty->size >= 8) ? 1 : 0;
                            if (node->lhs->ty && node->lhs->ty->is_unsigned)
                                asm_ucvtf(cg_sec, 0, r, src_sf); // ucvtf d0, w/x{r}
                            else
                                asm_scvtf(cg_sec, 0, r, src_sf); // scvtf d0, w/x{r}
                            // Integer-to-long-double return: the conversion
                            // above always lands in d0 as a genuine double.
#ifndef __APPLE__
                            // On Linux, AAPCS64 returns `long double` as a
                            // 128-bit binary128 quad in v0 -- widen via the
                            // same __extenddftf2 call used by the
                            // flonum-to-flonum branch above. Apple's arm64
                            // ABI defines `long double` as identical to
                            // `double` (size 8, see type.c's ty_ldouble) --
                            // no widening needed there.
                            if (scalar_ret_ty->kind == TY_LDOUBLE)
                                emit_direct_call("__extenddftf2", false); // bl __extenddftf2 (d0 -> quad in v0)
#endif
                        }
#else
                        {
                            int src_sz = node->lhs->ty ? node->lhs->ty->size : 8;
                            bool src_u = node->lhs->ty && node->lhs->ty->is_unsigned;
                            if (src_u && src_sz == 8) {
                                // unsigned long long → double: handle high bit
                                int c = ++rcc_label_count;
                                asm_test(cg_sec, REG(r), REG(r), 8); // test r, r
                                {
                                    size_t o = asm_jcc_label(cg_sec, X86_S); // jcc label
                                    asm_fixup_add(cg_sec, o, format(".L.u2f.high.%d", c), 1);
                                }
                                asm_cvtsi2sd(cg_sec, r, 8); // cvtsi2sd rr, xmm0
                                {
                                    size_t o = asm_jmp_label(cg_sec); // jmp .L.u2f.end.%d
                                    asm_fixup_add(cg_sec, o, format(".L.u2f.end.%d", c), 0);
                                }
                                cg_def_label(format(".L.u2f.high.%d", c)); // .L.u2f.high.%d:
                                x86_mov_rr(cg_sec, 8, X86_RCX, REG(r)); // movq r, %rcx
                                x86_shr_ri(cg_sec, 8, X86_RCX, 1); // shrq $1, %rcx
                                x86_cvtsi2sd(cg_sec, 8, X86_XMM0, X86_RCX); // cvtsi2sd %rcx, %xmm0
                                x86_addsd(cg_sec, X86_XMM0, X86_XMM0); // addsd %xmm0, %xmm0 (double it)
                                cg_def_label(format(".L.u2f.end.%d", c)); // .L.u2f.end.%d:
                            } else if (src_u && src_sz <= 4) {
                                // unsigned int/short/char → double: zero-extend to 64-bit
                                asm_cvtsi2sd(cg_sec, r, 8); // cvtsi2sd rr, %xmm0
                            } else {
                                int cssz = (src_sz >= 8) ? 8 : (src_sz < 4 ? 4 : src_sz);
                                asm_cvtsi2sd(cg_sec, r, cssz); // cvtsi2sd rr, %xmm0
                            }
                            // Integer-to-long-double return: the conversion above
                            // always lands in XMM0 (as a genuine double), but a
                            // `long double` return value must go back to the
                            // caller in ST0 -- widen it via the same red-zone
                            // round-trip used by the flonum-to-flonum branch
                            // above (movsd to scratch memory, then `fldl` onto
                            // the x87 stack).
                            if (scalar_ret_ty->kind == TY_LDOUBLE) {
                                X86Mem scratch = x86_mem(X86_RSP, -8);
                                x86_movsd_mr(cg_sec, scratch, X86_XMM0); // movsd %xmm0, -8(%rsp)
                                x86_fldl_m(cg_sec, scratch); // fldl -8(%rsp)
                            }
                        }
#endif
                    }
                } else if (node->lhs->ty && is_flonum(node->lhs->ty)) {
                    // Float expression returned as integer: convert from xmm0/d0
                    int sz = scalar_ret_ty ? scalar_ret_ty->size : 8;
#ifdef ARCH_ARM64
                    {
                        bool ret_unsigned = scalar_ret_ty && scalar_ret_ty->is_unsigned;
                        if (ret_unsigned)
                            asm_fcvtzu(cg_sec, r, sz); // fcvtzu w/x{r}, d0
                        else
                            asm_fcvtzs(cg_sec, r, sz); // fcvtzs w/x{r}, d0
                    }
                    if (sz >= 8)
                        asm_mov_x0_reg(cg_sec, REG(r)); // mov x0, reg64[r]
                    else if (scalar_ret_ty && scalar_ret_ty->is_unsigned)
                        asm_mov_w0_reg32(cg_sec, REG(r)); // mov w0, reg32[r]
                    else
                        asm_sxtw(cg_sec, ARM64_X0, REG(r)); // sxtw x0, reg32[r]
#else
                    asm_cvttsd2si(cg_sec, r, sz); // cvttsd2si xmm0, rr
                    asm_mov_reg_to_retval(cg_sec, r, 8); // movq rr, %rax
#endif
                } else {
#ifdef ARCH_ARM64
                    // Truncate return value to match function return type width
                    if (scalar_ret_ty && scalar_ret_ty->size < 4) {
                        if (scalar_ret_ty->is_unsigned)
                            asm_and_imm(cg_sec, r, 4, (1 << (scalar_ret_ty->size * 8)) - 1);
                        else if (scalar_ret_ty->size == 1)
                            asm_movsx(cg_sec, r, r, 4, 1); // movsx4->r rr, rr
                        else
                            asm_movsx(cg_sec, r, r, 4, 2); // movsx4->r rr, rr
                    }
                    asm_mov_x0_reg(cg_sec, REG(r)); // mov x0, x{r}
#else
                    // Truncate BEFORE moving to %rax so return register has correct value
                    if (scalar_ret_ty && scalar_ret_ty->size < 4 && scalar_ret_ty->is_unsigned)
                        asm_and_imm(cg_sec, r, 4, (1 << (scalar_ret_ty->size * 8)) - 1); // and mask, rr
                    asm_mov_reg_to_retval(cg_sec, r, 8); // movq rr, %rax
#endif
                }
                free_reg(r);
            }
        }
        // A pending cleanup/defer runs arbitrary code (calls, etc.)
        // between here and the actual jump to the epilogue -- protect
        // whatever's already sitting in the ABI return register(s) (see
        // parser.c's `return` handling, which only allocates
        // defer_retspill when cleanup_begin..cleanup_end is non-empty).
        if (node->defer_retspill) {
            int off = node->defer_retspill->offset;
#ifdef ARCH_ARM64
            arm64_orr_reg(cg_sec, 1, ARM64_X16, ARM64_XZR, ARM64_X0, ARM64_LSL, 0); // mov x16, x0
            arm64_store_to_fp_minus(off);
            arm64_orr_reg(cg_sec, 1, ARM64_X16, ARM64_XZR, ARM64_X1, ARM64_LSL, 0); // mov x16, x1
            arm64_store_to_fp_minus(off - 8);
#else
            asm_mov_phyreg_rbp(cg_sec, X86_RAX, 8, off); // mov [rbp-off], %rax
            asm_mov_phyreg_rbp(cg_sec, X86_RDX, 8, off - 8); // mov [rbp-off+8], %rdx
#endif
        }
        emit_cleanup_range(node->cleanup_begin, node->cleanup_end);
        if (node->defer_retspill) {
            int off = node->defer_retspill->offset;
#ifdef ARCH_ARM64
            arm64_load_from_fp_minus(off, ARM64_X0);
            arm64_load_from_fp_minus(off - 8, ARM64_X1);
#else
            asm_mov_rbp(cg_sec, X86_RAX, 8, off); // mov %rax, [rbp-off]
            asm_mov_rbp(cg_sec, X86_RDX, 8, off - 8); // mov %rdx, [rbp-off+8]
#endif
        }
        size_t ret_off = asm_jmp_label(cg_sec); /* jmp/b .L.return.%s */ /* andl $%d, %eax\n */
        char *ret_lbl = format(".L.return.%s", current_fn_def->name);
        asm_fixup_add(cg_sec, ret_off, ret_lbl, 0); // fixup add for forward branch
        return -1;
    }
    case ND_NULL:
        return -1;
    case ND_COMMA:
    case ND_CHAIN: {
        VReg r1 = gen(node->lhs);
        if (r1 != -1) free_reg(r1);
        return gen(node->rhs);
    }
    case ND_ALLOCA:
    case ND_ALLOCA_ZINIT: {
#ifdef ARCH_ARM64
        if (node->kind == ND_ALLOCA_ZINIT && node->lhs && node->lhs->kind == ND_NUM && node->lhs->val == 0) {
            arm64_load_from_fp_minus(node->var->offset, ARM64_X16);
            asm_mov_sp_reg(cg_sec, ARM64_X16); // mov sp, x16
            return -1;
        }
        VReg r = gen(node->lhs);
        // Save current SP into VLA save slot
        asm_mov_reg_sp(cg_sec, ARM64_X16); // mov x16, sp
        arm64_store_to_fp_minus(node->var->offset);
        // Round size up to 16-byte alignment, keep in x16 (scratch, not in pool)
        asm_add_imm(cg_sec, r, 8, 15); // add r, r, #15
        asm_and64_imm(cg_sec, r, ~(uint64_t)15); // and r, r, #-16
        asm_mov_x16_vreg(cg_sec, r); // mov x16, x{r}
        free_reg(r);
        asm_sub_sp_sp_x16_v2(cg_sec); // sub sp, sp, x16
        if (node->kind == ND_ALLOCA_ZINIT) {
            cg_def_label(format(".L.alloca.zero.%d", rcc_label_count));
            // subs x16, x16, #8  (sets flags)
            asm_subs_x16_imm(cg_sec, 8); // subs x16, x16, #8
            {
                size_t _cj = asm_jcc_label(cg_sec, ARM64_LT);
                asm_fixup_add(cg_sec, _cj, format(".L.alloca.done.%d", rcc_label_count), 1);
            }
            // str xzr, [sp, x16]
            asm_str_xzr_sp_x16(cg_sec); // str xzr, [sp, x16]
            {
                size_t _jmp = asm_jmp_label(cg_sec);
                asm_fixup_add(cg_sec, _jmp, format(".L.alloca.zero.%d", rcc_label_count), 0);
            }
        } else {
            cg_def_label(format(".L.alloca.probe.%d", rcc_label_count));
            // subs x16, x16, #4096
            asm_subs_x16_imm(cg_sec, 4096); // subs x16, x16, #4096
            {
                size_t _cj = asm_jcc_label(cg_sec, ARM64_LT);
                asm_fixup_add(cg_sec, _cj, format(".L.alloca.done.%d", rcc_label_count), 1);
            }
            // ldrb w17, [sp, x16]
            asm_ldrb_w17_sp_x16(cg_sec); // ldrb w17, [sp, x16]
            {
                size_t _jmp = asm_jmp_label(cg_sec);
                asm_fixup_add(cg_sec, _jmp, format(".L.alloca.probe.%d", rcc_label_count), 0);
            }
        }
        cg_def_label(format(".L.alloca.done.%d", rcc_label_count));
        rcc_label_count++;
        // Save VLA data pointer
        asm_mov_reg_sp(cg_sec, ARM64_X16); // mov x16, sp
        arm64_store_to_fp_minus(node->var->offset - 8);
#else
        if (node->kind == ND_ALLOCA_ZINIT && node->lhs && node->lhs->kind == ND_NUM && node->lhs->val == 0) {
            asm_mov_rbp(cg_sec, X86_RSP, 8, node->var->offset); // mov -offset(%rbp), %rsp
            return -1;
        }
        VReg r = gen(node->lhs);
        asm_mov_phyreg_rbp(cg_sec, X86_RSP, 8, node->var->offset); // movq %rsp, -offset(%rbp)
        x86_mov_rr(cg_sec, 8, X86_RAX, REG(r)); // movq %s, %rax
        free_reg(r);
        x86_mov_rr(cg_sec, 8, X86_RCX, X86_RSP); // movq %rsp, %rcx
        x86_sub_rr(cg_sec, 8, X86_RSP, X86_RAX); // subq %rax, %rsp
        x86_and_ri(cg_sec, 8, X86_RSP, -16); // andq $-16, %rsp
        x86_sub_rr(cg_sec, 8, X86_RCX, X86_RSP); // subq %rsp, %rcx
        if (node->kind == ND_ALLOCA_ZINIT) {
            x86_pxor(cg_sec, X86_XMM0, X86_XMM0); // pxor %%xmm0, %%xmm0
            cg_def_label(format(".L.alloca.zero.%d", rcc_label_count));
            x86_sub_ri(cg_sec, 8, X86_RCX, 16); // subq $16, %rcx
            {
                size_t o = asm_jcc_label(cg_sec, X86_S); // js .L.alloca.done.%d
                asm_fixup_add(cg_sec, o, format(".L.alloca.done.%d", rcc_label_count), 1);
            }
            {
                X86Mem m = {X86_RSP, X86_RCX, 1, 0};
                x86_movaps_mr(cg_sec, m, X86_XMM0); // movaps %%xmm0, (%rsp,%rcx)
            }
            {
                size_t o = asm_jmp_label(cg_sec); // jmp .L.alloca.zero.%d
                asm_fixup_add(cg_sec, o, format(".L.alloca.zero.%d", rcc_label_count), 0);
            }
        } else {
            cg_def_label(format(".L.alloca.probe.%d", rcc_label_count));
            x86_sub_ri(cg_sec, 8, X86_RCX, 4096); // subq $4096, %rcx
            {
                size_t o = asm_jcc_label(cg_sec, X86_S); // js .L.alloca.done.%d
                asm_fixup_add(cg_sec, o, format(".L.alloca.done.%d", rcc_label_count), 1);
            }
            {
                X86Mem m = {X86_RSP, X86_RCX, 1, 0};
                x86_or_mi(cg_sec, 1, m, 0); // orb $0, (%rsp,%rcx)
            }
            {
                size_t o = asm_jmp_label(cg_sec); // jmp .L.alloca.probe.%d
                asm_fixup_add(cg_sec, o, format(".L.alloca.probe.%d", rcc_label_count), 0);
            }
        }
        cg_def_label(format(".L.alloca.done.%d", rcc_label_count));
        rcc_label_count++;
        if (node->var)
            asm_mov_phyreg_rbp(cg_sec, X86_RSP, 8, node->var->offset - 8); // movq %rsp, -offset(%rbp)
        else
            x86_mov_rr(cg_sec, 8, X86_RAX, X86_RSP); // movq %rsp, %rax
#endif
        return -1;
    }
    case ND_BLOCK: {
        // At -O1, statements after __builtin_unreachable() in a block are dead;
        // skip emitting them until the next label (a jump target makes code
        // reachable again — basic-block boundary).
        bool dead = false;
        for (Node *n = node->body; n; n = n->next) {
            if (dead) {
                if (n->kind == ND_LABEL || n->kind == ND_CASE ||
                    n->kind == ND_LABEL_VAL) {
                    dead = false;
                } else {
                    if (opt_v && opt_W && !cg_dry_run && n->tok)
                        fprintf(stderr, "%s:%d: warning: unreachable statement elided\n",
                                n->tok->filename ? n->tok->filename
                                                 : (current_filename ? current_filename : "<unknown>"),
                                n->tok->lineno);
                    continue;
                }
            }
            VReg r = gen(n);
            if (r != -1) free_reg(r);
            if (opt_O1 && stmt_is_unreachable(n))
                dead = true;
        }
        return -1;
    }
    case ND_STMT_EXPR: {
        VReg result = -1;
        for (Node *n = node->body; n; n = n->next) {
            if (node->stmt_expr_result && n->kind == ND_EXPR_STMT && n->lhs == node->stmt_expr_result) {
                result = gen(node->stmt_expr_result);
            } else {
                VReg r = gen(n);
                if (r != -1)
                    free_reg(r);
            }
        }
        if (result == -1) {
            result = alloc_reg();
#ifdef ARCH_ARM64
            asm_movq_zero(cg_sec, result); // xor rresult, rresult
#else
            asm_movl_zero(cg_sec, result); // xor rresult, rresult
#endif
        }
        return result;
    }
    case ND_EXPR_STMT: {
        VReg r = gen(node->lhs);
        if (r != -1) free_reg(r);
        return -1;
    }
    case ND_FUNCALL: {
        return gen_funcall(node, -1);
    }
    case ND_LOGAND: {
        int c = ++rcc_label_count;
#ifdef ARCH_ARM64
        VReg r = alloc_reg();
        asm_movq_zero(cg_sec, r); // xor r, r — result is 0 when lhs short-circuits
        VReg lhs = gen(node->lhs);
        if (is_flonum(node->lhs->ty))
            gen_flonum_branch_if_zero(lhs, NULL, format(".L.end.%d", c));
        else {
            asm_cmp_zero(cg_sec, lhs, op_size(node->lhs->ty));
            {
                size_t o = asm_jcc_label(cg_sec, ARM64_EQ);
                asm_fixup_add(cg_sec, o, format(".L.end.%d", c), 1);
            }
        }
        free_reg(lhs);
        VReg rhs = gen(node->rhs);
        if (is_flonum(node->rhs->ty)) {
            gen_flonum_truthiness(rhs); // 0/1 in rhs
            asm_mov_reg_reg(cg_sec, r, rhs, 4); // mov w{rhs} -> w{r}
        } else {
            asm_cmp_zero(cg_sec, rhs, op_size(node->rhs->ty));
            asm_cset(cg_sec, r, ARM64_NE);
        }
        free_reg(rhs);
        cg_def_label(format(".L.end.%d", c));
#else
        VReg lhs = gen(node->lhs);
        asm_mov_rbp_imm(cg_sec, 1, spill_logand, 0); // movb $0, -spill_logand(%rbp)
        if (is_flonum(node->lhs->ty)) {
            // IEEE truthiness: -0.0 == 0.0 (false), NaN truthy; a bitwise
            // GP-register test would misclassify -0.0.
            gen_flonum_branch_if_zero(lhs, NULL, format(".L.end.%d", c));
        } else {
            asm_cmp_zero(cg_sec, lhs, op_size(node->lhs->ty));
            {
                size_t o = asm_jcc_label(cg_sec, X86_E);
                asm_fixup_add(cg_sec, o, format(".L.end.%d", c), 1);
            }
        }
        free_reg(lhs);
        VReg rhs = gen(node->rhs);
        if (is_flonum(node->rhs->ty)) {
            gen_flonum_truthiness(rhs); // 0/1 in rhs
            asm_mov_reg_rbp(cg_sec, rhs, 1, spill_logand); // movb r{rhs}b, -spill_logand(%rbp)
        } else {
            asm_cmp_zero(cg_sec, rhs, op_size(node->rhs->ty));
            asm_setcc(cg_sec, X86_RAX, X86_NE); // setne %al
            asm_mov_phyreg_rbp(cg_sec, X86_RAX, 1, spill_logand); // movb %al, -spill_logand(%rbp)
        }
        free_reg(rhs);
        cg_def_label(format(".L.end.%d", c));
        VReg r = alloc_reg();
        asm_movzx_rbp_reg(cg_sec, r, 4, 1, spill_logand); // movzbl -spill_logand(%rbp), rr
#endif
        return r;
    }
    case ND_LOGOR: {
        int c = ++rcc_label_count;
#ifdef ARCH_ARM64
        VReg lhs = gen(node->lhs);
        if (is_flonum(node->lhs->ty)) {
            int c2 = ++rcc_label_count;
            const char *skip = format(".L.logor.fskip.%d", c2);
            // Truthy (nonzero or NaN) lhs short-circuits with result 1.
            arm64_movz(cg_sec, 0, ARM64_X16, 1, 0); // mov w16, #1 (true-path result)
            gen_flonum_branch_if_nonzero(lhs, skip);
            free_reg(lhs);
            VReg rhs = gen(node->rhs);
            if (is_flonum(node->rhs->ty)) {
                gen_flonum_truthiness(rhs); // 0/1 in rhs
                arm64_orr_reg(cg_sec, 0, ARM64_X16, ARM64_XZR, REG(rhs), ARM64_LSL, 0); // mov w16, w{rhs}
            } else {
                asm_cmp_zero(cg_sec, rhs, op_size(node->rhs->ty)); // cmp $0, rrhs
                arm64_cset(cg_sec, 0, ARM64_X16, ARM64_NE); // cset w16, ne (false-path result in x16)
            }
            free_reg(rhs);
            cg_def_label(skip);
            VReg r = alloc_reg();
            arm64_orr_reg(cg_sec, 0, REG(r), ARM64_XZR, ARM64_X16, ARM64_LSL, 0); // mov w{r}, w16
            return r;
        }
        asm_cmp_zero(cg_sec, lhs, op_size(node->lhs->ty)); // cmp $0, rlhs
        arm64_movz(cg_sec, 0, ARM64_X16, 1, 0); // mov w16, #1 (true-path result in x16)
        size_t o = asm_jcc_label(cg_sec, ARM64_NE); // b.ne .L.end.%d
        asm_fixup_add(cg_sec, o, format(".L.end.%d", c), 1);
        free_reg(lhs);
        // False path: evaluate rhs; x16 may be clobbered here, re-set after
        VReg rhs = gen(node->rhs);
        asm_cmp_zero(cg_sec, rhs, op_size(node->rhs->ty)); // cmp $0, rrhs
        arm64_cset(cg_sec, 0, ARM64_X16, ARM64_NE); // cset w16, ne (false-path result in x16)
        free_reg(rhs);
        cg_def_label(format(".L.end.%d", c));
        // Result is in x16 on both paths; move to a fresh VReg
        VReg r = alloc_reg();
        arm64_orr_reg(cg_sec, 0, REG(r), ARM64_XZR, ARM64_X16, ARM64_LSL, 0); // mov w{r}, w16
        return r;
#else
        VReg lhs = gen(node->lhs);
        asm_mov_rbp_imm(cg_sec, 1, spill_logand, 1); // movb $1, -spill_logand(%rbp)
        if (is_flonum(node->lhs->ty)) {
            // IEEE truthiness: -0.0 == 0.0 (false), NaN truthy; a bitwise
            // GP-register test would misclassify -0.0.
            gen_flonum_branch_if_nonzero(lhs, format(".L.end.%d", c));
        } else {
            asm_cmp_zero(cg_sec, lhs, op_size(node->lhs->ty)); // cmp $0, rlhs
            {
                size_t o = asm_jcc_label(cg_sec, X86_NE); // jne .L.end.%d
                asm_fixup_add(cg_sec, o, format(".L.end.%d", c), 1);
            }
        }
        free_reg(lhs);
        VReg rhs = gen(node->rhs);
        if (is_flonum(node->rhs->ty)) {
            gen_flonum_truthiness(rhs); // 0/1 in rhs
            asm_mov_reg_rbp(cg_sec, rhs, 1, spill_logand); // movb r{rhs}b, -spill_logand(%rbp)
        } else {
            asm_cmp_zero(cg_sec, rhs, op_size(node->rhs->ty)); // cmp $0, rrhs
            asm_setcc(cg_sec, X86_RAX, X86_NE); // setne %al
            asm_mov_phyreg_rbp(cg_sec, X86_RAX, 1, spill_logand); // movb %al, -spill_logand(%rbp)
        }
        free_reg(rhs);
        cg_def_label(format(".L.end.%d", c));
        VReg r2 = alloc_reg();
        asm_movzx_rbp_reg(cg_sec, r2, 4, 1, spill_logand); // movzbl -spill_logand(%rbp), r2
        return r2;
#endif
    }
    /*
    case ND_LOGOR: {
        int c = ++rcc_label_count;
        const char *true_label = format(".L.logor_true.%d", c);
        const char *end_label = format(".L.logor_end.%d", c);
#ifdef ARCH_ARM64
        VReg r = alloc_reg();
        VReg lhs = gen(node->lhs);
        asm_cmp_zero(cg_sec, lhs, node->lhs->ty->size); // cmp $0, rlhs
        asm_mov_imm(cg_sec, r, 4, 1); // mov $1, rr
        size_t jo1 = asm_jcc_label(cg_sec, ARM64_NE); // jcc label
        asm_fixup_add(cg_sec, jo1, true_label, 1); // fixup add for forward branch
        free_reg(lhs);
        VReg rhs = gen(node->rhs);
        asm_cmp_zero(cg_sec, rhs, node->rhs->ty->size); // cmp $0, rrhs
        asm_cset(cg_sec, r, ARM64_NE); // cset rr
        size_t jo2 = asm_jmp_label(cg_sec); // jne .L.end.%d
        asm_fixup_add(cg_sec, jo2, end_label, 0); // fixup add for forward branch
        cg_def_label(true_label); // setne %%al
#else
        VReg r = alloc_reg();
        VReg lhs = gen(node->lhs);
        asm_cmp_zero(cg_sec, lhs, node->lhs->ty->size); // cmp $0, rlhs
        asm_mov_imm(cg_sec, r, 4, 1); // mov $1, rr
        size_t jo1 = asm_jcc_label(cg_sec, X86_NE); // jcc label
        asm_fixup_add(cg_sec, jo1, true_label, 1); // fixup add for forward branch
        free_reg(lhs);
        VReg rhs = gen(node->rhs);
        asm_cmp_zero(cg_sec, rhs, node->rhs->ty->size); // cmp $0, rrhs
        asm_setcc(cg_sec, REG(r), X86_NE); // setcc rr
        asm_movzx(cg_sec, r, r, 4, 1); // movzx4->r rr, rr
        size_t jo2 = asm_jmp_label(cg_sec); // .L.else.%d:
        asm_fixup_add(cg_sec, jo2, end_label, 0); // fixup add for forward branch
        cg_def_label(true_label); // cmp $0, %s
#endif
        free_reg(rhs);
        cg_def_label(end_label); // je .L.else.%d
        return r;
    }
*/
    case ND_COND: {
        int c = ++rcc_label_count;
        const char *else_label = format(".L.cond_else.%d", c);
        const char *end_label = format(".L.cond_end.%d", c);
        // Deferred: allocate the result register only once `then_r` is
        // known (just before it's actually needed), not up front. An
        // early alloc_reg() here would hold `r`'s slot reserved through
        // the ENTIRE recursive evaluation of cond/then/els -- for a
        // deeply nested ternary (each branch itself another ND_COND,
        // e.g. radiotap's BITNO_32/16/8/4/2 bit-counting macro chain),
        // every nesting level's own `r` stacks up simultaneously even
        // though none of them holds a real value until the very end,
        // exhausting the register pool several levels sooner than
        // necessary and forcing spills under pressure that a plain,
        // unnested expression of the same total complexity wouldn't hit.
        VReg r;
        int cond = gen(node->cond);
        int cond_sz = (node->cond->ty && is_flonum(node->cond->ty)) ? 8 : node->cond->ty->size;
        // GNU `a ?: b` (omitted then-operand): node->then is either literally
        // node->cond, or a cast wrapping it (inserted by check_type's usual
        // arithmetic conversion to the ND_COND's promoted result type) —
        // either way it is the SAME subexpression as node->cond and must not
        // be re-evaluated (side effects like `++x` would otherwise fire
        // twice). Reuse the already-computed `cond` register instead.
        bool gnu_omitted = node->then == node->cond ||
            (node->then->kind == ND_CAST && node->then->lhs == node->cond);
        if (node->cond->ty && is_flonum(node->cond->ty)) {
            // IEEE truthiness: -0.0 == 0.0 (false), NaN truthy; a bitwise
            // GP-register test would misclassify -0.0.
            gen_flonum_branch_if_zero(cond, NULL, else_label);
        } else {
#ifdef ARCH_ARM64
            asm_cmp_zero(cg_sec, cond, cond_sz); // cmp $0, rcond
            {
                size_t cj1 = asm_jcc_label(cg_sec, ARM64_EQ); // jcc label
                asm_fixup_add(cg_sec, cj1, else_label, 1); // fixup add for forward branch
            }
#else
            asm_cmp_zero(cg_sec, cond, cond_sz); // cmp $0, rcond
            {
                size_t cj1 = asm_jcc_label(cg_sec, X86_E); // jcc label
                asm_fixup_add(cg_sec, cj1, else_label, 1); // fixup add for forward branch
            }
#endif
        }
        VReg then_r;
        if (gnu_omitted) {
            then_r = (node->then->kind == ND_CAST)
                ? gen_cast_reg(cond, node->cond->ty, node->then->ty)
                : cond;
        } else {
            free_reg(cond);
            then_r = gen(node->then);
        }
        // A void-typed branch (e.g. `(void)0`, __builtin_unreachable()) yields
        // no value register; there is nothing to move into the result.
        r = R_NONE;
        if (then_r != R_NONE) {
            r = alloc_reg();
            asm_mov_reg_reg(cg_sec, r, then_r, 8); // mov rthen_r -> rr
            free_reg(then_r);
        }
        size_t cj2 = asm_jmp_label(cg_sec); // .L.end.%d:
        asm_fixup_add(cg_sec, cj2, end_label, 0); // fixup add for forward branch
        cg_def_label(else_label); // mov %s, #0
        VReg else_r = gen(node->els);
        if (else_r != R_NONE) {
            if (r == R_NONE)
                r = alloc_reg();
            asm_mov_reg_reg(cg_sec, r, else_r, 8); // mov relse_r -> rr
            free_reg(else_r);
        }
        cg_def_label(end_label); // .L.end.%d:
        return r;
    }
    case ND_IF: {
        // Same multi-declarator chain issue as ND_FOR's init (see below):
        // C23 `if (init-statement; cond)` also reuses declaration().
        for (Node *n = node->init; n; n = n->next) {
            VReg r = gen(n);
            if (r != -1) free_reg(r);
        }
        // Fold constant integer conditions to avoid dead code emission,
        // but keep blocks that contain labels (may be targets of goto).
        if (node->cond->kind == ND_NUM) {
            // Recursively check if a node tree contains any label or goto
            bool has_label = false;
            {
                Node *stack[512];
                int sp = 0;
                for (Node *n = node->then; n && sp < 512; n = n->next)
                    stack[sp++] = n;
                if (node->els)
                    for (Node *n = node->els; n && sp < 512; n = n->next)
                        stack[sp++] = n;
                while (sp > 0 && !has_label) {
                    Node *n = stack[--sp];
                    if (n->kind == ND_LABEL || n->kind == ND_GOTO || n->kind == ND_GOTO_IND ||
                        n->kind == ND_CASE || n->kind == ND_LABEL_VAL) {
                        has_label = true;
                    } else if (n->kind == ND_FOR || n->kind == ND_DO || n->kind == ND_IF) {
                        if (n->then && sp < 512) stack[sp++] = n->then;
                        if (n->cond && sp < 512) stack[sp++] = n->cond;
                        if (n->kind == ND_FOR) {
                            if (n->body && sp < 512) stack[sp++] = n->body;
                            if (n->init && sp < 512) stack[sp++] = n->init;
                            if (n->inc && sp < 512) stack[sp++] = n->inc;
                        }
                        if (n->els && sp < 512) stack[sp++] = n->els;
                    } else if (n->kind == ND_BLOCK) {
                        for (Node *c = n->body; c && sp < 512; c = c->next)
                            stack[sp++] = c;
                    }
                }
            }
            if (!has_label) {
                if (node->cond->val) {
                    VReg r = gen(node->then);
                    if (r != -1) free_reg(r);
                } else if (node->els) {
                    VReg r = gen(node->els);
                    if (r != -1) free_reg(r);
                }
                return -1;
            }
            // if(0) with labels: the labeled body is a Duff's-device
            // target -- reachable ONLY via a switch/goto jumping directly
            // to its label, NEVER via this if's own (always-false)
            // condition. Normal SEQUENTIAL fall-through into this source
            // position (from whatever precedes the if-statement) must
            // still skip the body, exactly like a real `if(0) goto after;`
            // would; a switch's own dispatch bypasses this skip-jump
            // entirely by branching straight to the case label inside the
            // body, past the jump instruction. Emitting the case node(s)
            // directly with NO skip-jump (as this used to do) executed
            // the "dead" body unconditionally on fall-through, corrupting
            // any code reached by ordinary sequential execution before
            // the first real switch dispatch into it. Found via LZ4's
            // frame decompression state machine
            // (`if (0) case dstage_storeCBlock: {...}`), which silently
            // executed the storeCBlock body on every single call instead
            // of only when explicitly dispatched there.
            //
            // Plain if(1) with labels (cond->val != 0) intentionally
            // falls through to the general codegen path below instead:
            // `if(1)` unconditionally enters its then-branch anyway, so a
            // plain fall-through already has the right semantics with no
            // special-casing needed.
            if (!node->cond->val) {
                // Find outermost block body
                Node *list = node->then;
                while (list && list->kind == ND_BLOCK && list->body &&
                       list->body->kind == ND_BLOCK && !list->body->next)
                    list = list->body;
                if (list && list->kind == ND_BLOCK)
                    list = list->body;
                // Skip to first label/case
                Node *n = list;
                while (n && n->kind != ND_LABEL && n->kind != ND_CASE && n->kind != ND_LABEL_VAL)
                    n = n->next;
                // If the first label is a case (Duff), skip dead code before it.
                // Otherwise generate full body to preserve goto labels.
                if (n && n->kind == ND_CASE) {
                    int duff_c = ++rcc_label_count;
                    const char *duff_after = format(".L.duff_after.%d", duff_c);
                    size_t skip_jmp = asm_jmp_label(cg_sec); // jmp duff_after (fall-through skips the dead body)
                    asm_fixup_add(cg_sec, skip_jmp, duff_after, 0);
                    for (; n; n = n->next) {
                        VReg r = gen(n);
                        if (r != -1) free_reg(r);
                    }
                    cg_def_label(duff_after);
                    return -1;
                }
            }
        }
        int c = ++rcc_label_count;
        const char *end_label = format(".L.end.%d", c);
        size_t else_jmp = (size_t)-1;
        size_t end_jmp = (size_t)-1;
        if (node->els) {
            const char *else_label = format(".L.else.%d", c);
            gen_cond_branch_inv(node->cond, &else_jmp, else_label);
            VReg r1 = gen(node->then);
            if (r1 != -1) free_reg(r1);
            end_jmp = asm_jmp_label(cg_sec); // jmp end
            asm_patch_branch(cg_sec, else_jmp, cg_sec->len, 1); // else:
            cg_def_label(else_label);
            VReg r2 = gen(node->els);
            if (r2 != -1) free_reg(r2);
            asm_patch_branch(cg_sec, end_jmp, cg_sec->len, 0); // end:
            cg_def_label(end_label);
        } else {
            gen_cond_branch_inv(node->cond, &end_jmp, end_label);
            VReg r1 = gen(node->then);
            if (r1 != -1) free_reg(r1);
            asm_patch_branch(cg_sec, end_jmp, cg_sec->len, 1); // end:
            cg_def_label(end_label);
        }
        return -1;
    }
    case ND_FOR: {
        int c = ++rcc_label_count;
        const char *cont_label = format(".L.continue.%d", c);
        const char *end_label = format(".L.end.%d", c);

        // node->init is head.next from declaration(): a multi-declarator
        // init clause (`for (T a = x, b = y; ...)`) chains one ND_EXPR_STMT
        // per declarator via ->next, exactly like a compound statement's
        // body. gen() on a single ND_EXPR_STMT only generates that node's
        // own ->lhs and never walks ->next, so calling gen() once on just
        // the head silently dropped every declarator after the first —
        // e.g. `for (size_t i = 0, copied = 0; ...)` left `copied`
        // uninitialized. Walk the whole chain, mirroring how function/
        // block bodies are generated elsewhere in this file.
        for (Node *n = node->init; n; n = n->next) {
            VReg r = gen(n);
            if (r != -1) free_reg(r);
        }
        size_t begin_pos = cg_sec->len; // begin label
        break_stack[ctrl_depth] = c;
        continue_stack[ctrl_depth] = c;
        ctrl_depth++;
        size_t end_jmp = (size_t)-1;
        if (node->cond) {
            gen_cond_branch_inv(node->cond, &end_jmp, end_label);
        }
        VReg r_then = gen(node->then);
        if (r_then != -1) free_reg(r_then);
        cg_def_label(cont_label); // continue label
        if (node->inc) {
            VReg r_inc = gen(node->inc);
            if (r_inc != -1) free_reg(r_inc);
        }
        asm_b_back(cg_sec, begin_pos); // b begin
        asm_patch_branch(cg_sec, end_jmp, cg_sec->len, 1); // patch cond branch
        cg_def_label(end_label); // end label (for break)
        ctrl_depth--;
        return -1;
    }
    case ND_DO: {
        int c = ++rcc_label_count;
        const char *cont_label = format(".L.continue.%d", c);
        size_t begin_pos = cg_sec->len; // begin label
        break_stack[ctrl_depth] = c;
        continue_stack[ctrl_depth] = c;
        ctrl_depth++;
        VReg r_then = gen(node->then);
        if (r_then != -1) free_reg(r_then);
        cg_def_label(cont_label); // continue:
        VReg r = gen(node->cond);
        if (node->cond->ty && is_flonum(node->cond->ty)) {
            // IEEE truthiness: -0.0 == 0.0 (false), NaN truthy. A bitwise
            // GP-register test would misclassify -0.0 (nonzero bit
            // pattern, value == 0.0) as truthy. Branch forward past the
            // backward jump when the condition is FALSE, else loop back.
            int c2 = ++rcc_label_count;
            const char *fskip = format(".L.do.fskip.%d", c2);
            gen_flonum_branch_if_zero(r, NULL, fskip);
            free_reg(r);
            asm_b_back(cg_sec, begin_pos); // b begin
            cg_def_label(fskip);
        } else {
#ifdef ARCH_ARM64
            asm_cmp_zero(cg_sec, r, op_size(node->cond->ty)); // cmp $0, rr
            free_reg(r);
            asm_bcond_back(cg_sec, ARM64_NE, begin_pos); // b.ne begin
#else
            asm_cmp_zero(cg_sec, r, op_size(node->cond->ty)); // cmp $0, rr
            free_reg(r);
            asm_bcond_back(cg_sec, X86_NE, begin_pos); // jne begin
#endif
        }
        cg_def_label(format(".L.end.%d", c)); // end:
        ctrl_depth--;
        return -1;
    }
    case ND_SWITCH: {
        // Same multi-declarator chain issue as ND_FOR's init above (C23
        // `switch (init-statement; cond)` reuses declaration()).
        for (Node *n = node->init; n; n = n->next) {
            VReg r = gen(n);
            if (r != -1) free_reg(r);
        }
        int c = ++rcc_label_count;
        VReg cond = gen(node->cond);
        int sz = op_size(node->cond->ty);
        bool is_uns = node->cond->ty && node->cond->ty->is_unsigned;
        for (Node *cs = node->case_next; cs; cs = cs->case_next) {
            if (!cs->label_id)
                cs->label_id = ++rcc_label_count;
            if (cs->is_case_range) {
                int skip_lbl = ++rcc_label_count;
#ifdef ARCH_ARM64
                if ((cs->case_val >= 0 && cs->case_val <= 4095) ||
                    (cs->case_val > 0 && cs->case_val <= 0xffffff && (cs->case_val % 4096) == 0))
                    asm_cmp_imm(cg_sec, cond, sz, (long long)cs->case_val); // cmp $(long long)cs->case_val, rcond
                else {
                    VReg tmp = alloc_reg();
                    asm_mov_imm(cg_sec, tmp, 8, (int64_t)cs->case_val); // mov tmp, #case_val
                    asm_cmp_reg_reg(cg_sec, cond, tmp, sz); // cmp rtmp, rcond
                    free_reg(tmp);
                }
                {
                    size_t o = asm_jcc_label(cg_sec, is_uns ? ARM64_LO : ARM64_LT); // jcc label
                    asm_fixup_add(cg_sec, o, format(".L.skip.%d", skip_lbl), 1);
                } /* cmp %s, #%lld\n */
                if ((cs->case_end >= 0 && cs->case_end <= 4095) ||
                    (cs->case_end > 0 && cs->case_end <= 0xffffff && (cs->case_end % 4096) == 0))
                    asm_cmp_imm(cg_sec, cond, sz, (long long)cs->case_end); // cmp $(long long)cs->case_end, rcond
                else {
                    VReg tmp = alloc_reg();
                    asm_mov_imm(cg_sec, tmp, 8, (int64_t)cs->case_end); // mov tmp, #case_end
                    asm_cmp_reg_reg(cg_sec, cond, tmp, 8); // cmp rtmp, rcond
                    free_reg(tmp);
                }
                {
                    size_t o = asm_jcc_label(cg_sec, is_uns ? ARM64_LS : ARM64_LE); // jcc label
                    asm_fixup_add(cg_sec, o, format(".L.case.%d", (int)cs->label_id), 1);
                } /* movabs $%lld, %s\n */
#else
                if (cs->case_val == (int32_t)cs->case_val)
                    asm_cmp_imm(cg_sec, cond, sz, (long long)cs->case_val); // cmp $(long long)cs->case_val, rcond
                else {
                    VReg tmp = alloc_reg();
                    asm_mov_imm(cg_sec, tmp, 8, (long long)cs->case_val); // mov $(long long)cs->case_val, rtmp
                    asm_cmp_reg_reg(cg_sec, cond, tmp, 8); // cmp rtmp, rcond
                    free_reg(tmp);
                }
                {
                    size_t o = asm_jcc_label(cg_sec, is_uns ? X86_B : X86_L); // jcc label
                    asm_fixup_add(cg_sec, o, format(".L.skip.%d", skip_lbl), 1);
                } /* cmp %s, %s\n */
                if (cs->case_end == (int32_t)cs->case_end)
                    asm_cmp_imm(cg_sec, cond, sz, (long long)cs->case_end); // cmp $(long long)cs->case_end, rcond
                else {
                    VReg tmp = alloc_reg();
                    asm_mov_imm(cg_sec, tmp, 8, (long long)cs->case_end); // mov $(long long)cs->case_end, rtmp
                    asm_cmp_reg_reg(cg_sec, cond, tmp, 8); // cmp rtmp, rcond
                    free_reg(tmp);
                }
                {
                    size_t o = asm_jcc_label(cg_sec, is_uns ? X86_BE : X86_LE); // jcc label
                    asm_fixup_add(cg_sec, o, format(".L.case.%d", (int)cs->label_id), 1);
                } /* b.eq .L.case.%d\n */
#endif
                cg_def_label(format(".L.skip.%d", skip_lbl)); // cmp $%lld, %s
            } else {
#ifdef ARCH_ARM64
                if ((cs->case_val >= 0 && cs->case_val <= 4095) ||
                    (cs->case_val > 0 && cs->case_val <= 0xffffff && (cs->case_val % 4096) == 0)) {
                    asm_cmp_imm(cg_sec, cond, sz, (long long)cs->case_val); // cmp $(long long)cs->case_val, rcond
                } else {
                    VReg tmp = alloc_reg();
                    asm_mov_imm(cg_sec, tmp, 8, (int64_t)cs->case_val); // mov tmp, #case_val
                    asm_cmp_reg_reg(cg_sec, cond, tmp, sz); // cmp rtmp, rcond
                    free_reg(tmp);
                }
                {
                    size_t case_jmp = asm_jcc_label(cg_sec, ARM64_EQ); // jcc label
                    asm_fixup_add(cg_sec, case_jmp, format(".L.case.%d", (int)cs->label_id), 1);
                }
#else
                if (cs->case_val == (int32_t)cs->case_val) {
                    asm_cmp_imm(cg_sec, cond, sz, (long long)cs->case_val); // cmp $(long long)cs->case_val, rcond
                } else {
                    VReg tmp = alloc_reg();
                    asm_mov_imm(cg_sec, tmp, 8, (long long)cs->case_val); // mov $(long long)cs->case_val, rtmp
                    asm_cmp_reg_reg(cg_sec, tmp, cond, 8); // cmp rcond, rtmp
                    free_reg(tmp);
                }
                size_t case_jmp = asm_jcc_label(cg_sec, X86_E); // jcc label
                asm_fixup_add(cg_sec, case_jmp, format(".L.case.%d", cs->label_id), 1); // fixup label
#endif
            }
        }
        if (node->default_case) {
            if (!node->default_case->label_id)
                node->default_case->label_id = ++rcc_label_count;
            size_t sw_jmp = asm_jmp_label(cg_sec); // cmp %s, %s
            asm_fixup_add(cg_sec, sw_jmp, format(".L.case.%d", node->default_case->label_id), 0); // fixup label
        } else {
            size_t sw_jmp = asm_jmp_label(cg_sec); // cmp %s, #%lld
            asm_fixup_add(cg_sec, sw_jmp, format(".L.end.%d", c), 0); // fixup label
        }
        free_reg(cond);
        break_stack[ctrl_depth] = c;
        continue_stack[ctrl_depth] = ctrl_depth > 0 ? continue_stack[ctrl_depth - 1] : c;
        ctrl_depth++;
        VReg r_body = gen(node->then);
        if (r_body != -1) free_reg(r_body);
        ctrl_depth--;
        cg_def_label(format(".L.end.%d", c)); // .L.end.%d:
        return -1;
    }
    case ND_CASE: {
        if (!node->label_id)
            node->label_id = ++rcc_label_count;
        cg_def_label(format(".L.case.%d", node->label_id)); // .L.case.%d:
        return gen(node->lhs);
    }
    case ND_BREAK:
        if (ctrl_depth == 0)
            error("stray break");
        emit_cleanup_range(node->cleanup_begin, node->cleanup_end);
        emit_vla_dealloc(node->cleanup_begin, node->cleanup_end);
        {
            size_t brk_off = asm_jmp_label(cg_sec); // b .L.end.%d
            asm_fixup_add(cg_sec, brk_off, format(".L.end.%d", break_stack[ctrl_depth - 1]), 0); // fixup label
        }
        return -1;
    case ND_CONTINUE:
        if (ctrl_depth == 0)
            error("stray continue");
        emit_cleanup_range(node->cleanup_begin, node->cleanup_end);
        emit_vla_dealloc(node->cleanup_begin, node->cleanup_end);
        {
            size_t cont_off = asm_jmp_label(cg_sec); // b .L.continue.%d
            asm_fixup_add(cg_sec, cont_off, format(".L.continue.%d", continue_stack[ctrl_depth - 1]), 0); // fixup add for forward branch
        }
        return -1;
    case ND_GOTO:
        emit_cleanup_range(node->cleanup_begin, node->cleanup_end);
        emit_vla_dealloc(node->cleanup_begin, node->cleanup_end);
#ifdef ARCH_ARM64
        if (node->funcname) {
            // Nonlocal goto (GNU __label__ + goto crossing a nested
            // function's frame boundary back into an enclosing function):
            // restore that ancestor frame's x29 AND sp exactly. x29 comes
            // from the usual chain-walk (gen_chain_walk_into); sp is read
            // straight from the SAME target frame's own CHAIN_RSP_OFFSET
            // slot - which that ancestor recorded about ITSELF, once, at
            // its own prologue (see the unconditional stur right after
            // the `sub sp,sp,#frame_size` above). A generous fixed sp
            // margin does NOT work: every function's epilogue is `add
            // sp, sp, #frame_size; ldp fp,lr; ret`, and frame_size is a
            // per-function, codegen-time value this site (compiled
            // independently, often *before* the target ancestor - see
            // parser.c's nested-function-definition path) cannot know.
            // Nor can the CALLER propagate its own live sp at the call
            // site: call-site-local register/argument staging transiently
            // perturbs sp around the very call that would capture it.
            //
            // The jump itself goes indirect through a computed address
            // (adrp+add, exactly ND_LABEL_VAL's mechanism below) rather
            // than a direct branch+fixup: nested functions are appended
            // to the TLItem list — and therefore codegen'd — BEFORE their
            // enclosing function (see parser.c's declaration()), so a
            // direct branch's symbolic fixup to the enclosing function's
            // not-yet-emitted label never resolves; adrp+add's ELF
            // relocation is order-independent and already proven correct
            // for this exact cross-function case (a nested function's
            // `&&label` referring to an enclosing label already relies on
            // it — see ND_LABEL_VAL immediately below).
            VReg base = alloc_reg();
            gen_chain_walk_into(base, node->chain_depth); // base = target's x29
            VReg tmp = alloc_reg();
            arm64_ldur(cg_sec, 1, REG(tmp), REG(base), -CHAIN_RSP_OFFSET); // tmp = [base, #-CHAIN_RSP_OFFSET] (target's own recorded sp)
            arm64_orr_reg(cg_sec, 1, ARM64_X29, ARM64_XZR, REG(base), ARM64_LSL, 0); // mov x29, base
            asm_mov_sp_reg(cg_sec, REG(tmp)); // mov sp, tmp
            free_reg(tmp);
            free_reg(base);
            VReg tgt = alloc_reg();
            emit_adrp_add(tgt, format(".L.label.%s.%s", node->funcname, node->label_name));
            asm_jmp_reg(cg_sec, tgt); // br tgt
            free_reg(tgt);
        } else {
            size_t goto_off = asm_jmp_label(cg_sec); // b .L.label.%s.%s
            asm_fixup_add(cg_sec, goto_off, format(".L.label.%s.%s", current_fn, node->label_name), 0); // fixup label
        }
#else
        if (node->funcname) {
            // Nonlocal goto: see the ARM64 branch above for the full
            // rationale on why rsp must be read from the target's own
            // self-recorded CHAIN_RSP_OFFSET slot (not a guessed margin,
            // not caller-propagated), and why the jump must be indirect
            // through a computed (lea-resolved) address rather than a
            // direct jmp+fixup.
            VReg base = alloc_reg();
            gen_chain_walk_into(base, node->chain_depth); // base = target's rbp
            VReg tmp = alloc_reg();
            x86_mov_rm(cg_sec, 8, REG(tmp), x86_mem(REG(base), -CHAIN_RSP_OFFSET)); // tmp = [base - CHAIN_RSP_OFFSET] (target's own recorded rsp)
            x86_mov_rr(cg_sec, 8, X86_RBP, REG(base)); // rbp = base
            x86_mov_rr(cg_sec, 8, X86_RSP, REG(tmp)); // rsp = tmp
            free_reg(tmp);
            free_reg(base);
            VReg tgt = alloc_reg();
            asm_lea_rip_reg(cg_sec, tgt, format(".L.label.%s.%s", node->funcname, node->label_name));
            asm_jmp_reg(cg_sec, tgt); // jmp *tgt
            free_reg(tgt);
        } else {
            size_t goto_off = asm_jmp_label(cg_sec); // b .L.label.%s.%s
            asm_fixup_add(cg_sec, goto_off, format(".L.label.%s.%s", current_fn, node->label_name), 0); // fixup label
        }
#endif
        return -1;
    case ND_GOTO_IND: {
        VReg r = gen(node->lhs);
#ifdef ARCH_ARM64
        asm_jmp_reg(cg_sec, r); // jmp rr
#else
        asm_jmp_reg(cg_sec, r); // jmp rr
#endif
        free_reg(r);
        return -1;
    }
    case ND_LABEL: {
        cg_def_label(format(".L.label.%s.%s", current_fn, node->label_name)); // .L.label.%s.%s:
        return gen(node->lhs);
    }
    case ND_LABEL_VAL: {
        VReg r = alloc_reg();
        // For a nested function's &&label referring to an ENCLOSING
        // function's label, node->funcname (set by find_label_scope_owner
        // in parser.c) names the owning function; current_fn only applies
        // when the label belongs to the function being codegen'd here.
        const char *label_fn = node->funcname ? node->funcname : current_fn;
#ifdef ARCH_ARM64
        emit_adrp_add(r, format(".L.label.%s.%s", label_fn, node->label_name));
#else
        asm_lea_rip_reg(cg_sec, r, format(".L.label.%s.%s", label_fn, node->label_name)); // lea label(%rip), r
#endif
        return r;
    }
    case ND_ASM: {
#ifdef ARCH_ARM64
        // ARM64 extended inline assembly handler.
        //
        // For each operand:
        //   - Memory (m, Q, Ump): gen_addr → asm_str = "[xN]"
        //   - Output-only register (=r, =w, =x, =y, =&r):
        //       gen_addr for store-back, alloc fresh register for asm use
        //   - Read-write register (+r): gen_addr, alloc fresh reg, load value, store back
        //   - Input register (r, w, x): gen value into register
        //   - Matching constraint (digit): same register as referenced output
        //   - Immediate (I,K,L,M,N): gen value, emit as register (immediate in template)
        //   - Zero (Z): use xzr
        //   - Symbol (S): gen address

        VReg op_regs[MAX_ASM_OPERANDS]; // GPR for asm template operand (-1 if unused/FP)
        VReg op_addr[MAX_ASM_OPERANDS]; // Address register for store-back (-1 if none)
        int op_is_fp[MAX_ASM_OPERANDS]; // FP constraint (use d0 as output)
        for (int i = 0; i < node->asm_noperands; i++) {
            op_regs[i] = -1;
            op_addr[i] = -1;
            op_is_fp[i] = 0;
        }

        for (int i = 0; i < node->asm_noperands; i++) {
            AsmOperand *op = &node->asm_ops[i];
            const char *c = op->constraint;
            // skip leading modifiers =, +, &
            while (*c == '=' || *c == '+' || *c == '&') c++;

            // Matching constraint: digit refers to another operand
            if (*c >= '0' && *c <= '9') {
                // defer: will resolve after all others are evaluated
                op_regs[i] = -2; // sentinel: matching
                continue;
            }

            // Determine constraint class
            bool is_mem = op->is_memory || *c == 'Q' ||
                (*c == 'U' && c[1] == 'm' && c[2] == 'p');
            bool is_fp = (*c == 'w' || *c == 'x' || *c == 'y') && !is_mem;
            bool is_imm = (*c == 'I' || *c == 'K' || *c == 'L' ||
                           *c == 'M' || *c == 'N');
            bool is_zero = (*c == 'Z');

            if (is_mem) {
                VReg r = gen_addr(op->expr);
                op_regs[i] = r;
                op->reg = r;
                snprintf(op->asm_str, sizeof(op->asm_str), "[%s]", reg64[r]);
            } else if (is_fp) {
                if (op->is_output) {
                    // FP output: get address for store-back, use d0 in template
                    VReg r = gen_addr(op->expr);
                    op_addr[i] = r;
                    op_is_fp[i] = 1;
                    op->reg = -1;
                    snprintf(op->asm_str, sizeof(op->asm_str), "d0");
                } else {
                    // FP input treated as integer register for now
                    VReg r = gen(op->expr);
                    op_regs[i] = r;
                    op->reg = r;
                    snprintf(op->asm_str, sizeof(op->asm_str), "%s", reg64[r]);
                }
            } else if (is_zero) {
                // Zero constraint: always xzr, no register allocation needed
                snprintf(op->asm_str, sizeof(op->asm_str), "xzr");
                // op_regs[i] stays -1
            } else if (is_imm) {
                // Immediate constraint (I,K,L,M,N): emit as #value in the template.
                // Try to extract compile-time constant; fall back to register.
                int64_t cval = 0;
                if (try_const_int(op->expr, &cval)) {
                    snprintf(op->asm_str, sizeof(op->asm_str), "#0x%llx",
                             (unsigned long long)(uint64_t)cval);
                    // op_regs[i] stays -1, no register needed
                } else {
                    VReg r = gen(op->expr);
                    op_regs[i] = r;
                    op->reg = r;
                    snprintf(op->asm_str, sizeof(op->asm_str), "%s", reg64[r]);
                }
            } else if (op->is_output && !op->is_rw) {
                // Output-only register (=r, =&r, etc.):
                // Allocate a fresh x register for the asm, store back after.
                // Always use x (64-bit) register in the template; %wN modifier handles 32-bit.
                VReg r_addr = gen_addr(op->expr);
                op_addr[i] = r_addr;
                VReg r = alloc_reg();
                op_regs[i] = r;
                op->reg = r;
                snprintf(op->asm_str, sizeof(op->asm_str), "%s", reg64[r]);
            } else if (op->is_rw) {
                // Read-write register (+r):
                // Allocate fresh x register, load current value, store back after.
                VReg r_addr = gen_addr(op->expr);
                op_addr[i] = r_addr;
                VReg r = alloc_reg();
                op_regs[i] = r;
                op->reg = r;
                emit_load(op->expr->ty, r, r_addr, 0);
                snprintf(op->asm_str, sizeof(op->asm_str), "%s", reg64[r]);
            } else {
                // Input register (r) or symbol address (S):
                VReg r = gen(op->expr);
                op_regs[i] = r;
                op->reg = r;
                snprintf(op->asm_str, sizeof(op->asm_str), "%s", reg64[r]);
            }
        }

        // Resolve matching constraints (digit): share register with referenced output.
        // Also load the input value into the shared register before the asm.
        for (int i = 0; i < node->asm_noperands; i++) {
            if (op_regs[i] != -2) continue;
            AsmOperand *op = &node->asm_ops[i];
            const char *c = op->constraint;
            while (*c == '=' || *c == '+' || *c == '&') c++;
            int ref = *c - '0';
            if (ref >= 0 && ref < node->asm_noperands) {
                VReg r = op_regs[ref];
                // For output operands that compute an address (gen_addr), the
                // matching input needs a SEPARATE value register, not the address reg.
                bool ref_is_addr = node->asm_ops[ref].is_output && !node->asm_ops[ref].is_rw;
                if (ref_is_addr) {
                    // Allocate a fresh register for the input value,
                    // then move it into the output register so the asm
                    // finds it in the shared register.
                    VReg r_in = gen(op->expr);
                    asm_mov_reg_reg(cg_sec, r, r_in, 8); // mov rr_in -> rr
                    free_reg(r_in);
                    op_regs[i] = r;
                    op->reg = r;
                    snprintf(op->asm_str, sizeof(op->asm_str), "%s", reg64[r]);
                } else {
                    op_regs[i] = r;
                    op->reg = r;
                    strncpy(op->asm_str, node->asm_ops[ref].asm_str, sizeof(op->asm_str) - 1);
                    op->asm_str[sizeof(op->asm_str) - 1] = '\0';
                    if (r >= 0) {
                        VReg r_in = gen(op->expr);
                        if (r_in != r)
                            asm_mov_reg_reg(cg_sec, r, r_in, 8); // mov rr_in -> rr
                        free_reg(r_in);
                    }
                }
            }
        }

        // Validate the asm template (catches invalid mnemonics, ranges, etc.)
        arm64_validate_asm_template(node->asm_template, node->tok);

        // Translate TCC-specific {$}N to ARM64 immediate #N
        char adj[4096];
        int alen = 0;
        const char *tp = node->asm_template;
        while (*tp && alen < (int)sizeof(adj) - 1) {
            if (*tp == '{' && *(tp + 1) == '$' && *(tp + 2) == '}') {
                adj[alen++] = '#';
                tp += 3;
            } else {
                adj[alen++] = *tp++;
            }
        }
        adj[alen] = '\0';

        // Substitute %N, %wN, %xN, %dN, %l[label], %[name] in template
        char out[4096];
        int olen = 0;
        const char *p = adj;
        while (*p && olen < (int)sizeof(out) - 1) {
            if (*p != '%') {
                out[olen++] = *p++;
                continue;
            }
            p++;
            if (*p == '%') {
                out[olen++] = '%';
                p++;
                continue;
            }
            // Check for modifier letter(s): w, x, d, s, l, h
            char mod = 0;
            if (*p == 'w' || *p == 'x' || *p == 'd' || *p == 's' ||
                *p == 'l' || *p == 'h') {
                // peek: if next char is '[' or digit, it's a modifier
                if (*(p + 1) == '[' || (*(p + 1) >= '0' && *(p + 1) <= '9'))
                    mod = *p++;
            }

            if (mod == 'l' && *p == '[') {
                // %l[name] → goto label
                p++;
                const char *end = strchr(p, ']');
                if (!end) {
                    out[olen++] = '%';
                    out[olen++] = 'l';
                    out[olen++] = '[';
                    continue;
                }
                const char *prefix = ".L.label.";
                for (const char *s = prefix; *s && olen < (int)sizeof(out) - 1;) out[olen++] = *s++;
                for (const char *s = current_fn; *s && olen < (int)sizeof(out) - 1;) out[olen++] = *s++;
                if (olen < (int)sizeof(out) - 1) out[olen++] = '.';
                for (const char *s = p; s < end && olen < (int)sizeof(out) - 1;) out[olen++] = *s++;
                p = end + 1;
            } else if (*p == '[') {
                // %[name] → named operand (not a goto label)
                p++;
                const char *end = strchr(p, ']');
                if (!end) {
                    out[olen++] = '%';
                    out[olen++] = '[';
                    continue;
                }
                // find operand with matching name
                char namebuf[32];
                int nlen = (int)(end - p) < 31 ? (int)(end - p) : 31;
                memcpy(namebuf, p, nlen);
                namebuf[nlen] = '\0';
                p = end + 1;
                const char *sub = NULL;
                for (int j = 0; j < node->asm_noperands; j++) {
                    if (strcmp(node->asm_ops[j].name, namebuf) == 0) {
                        sub = node->asm_ops[j].asm_str;
                        break;
                    }
                }
                if (!sub) sub = "";
                // Apply modifier to named operand
                if (sub[0] && (mod == 'w' || mod == 'x') && sub[0] == 'x') {
                    if (mod == 'w') {
                        if (olen < (int)sizeof(out) - 1) out[olen++] = 'w';
                        sub++; // skip leading 'x'
                    }
                }
                while (*sub && olen < (int)sizeof(out) - 1) out[olen++] = *sub++;
            } else if (mod == 'l' && *p >= '0' && *p <= '9') {
                // %lN: positional "asm goto" label reference, numbered
                // continuing right after the in/out operand indices (see
                // the matching x86 branch below for the full rationale).
                int n = *p - '0';
                p++;
                int li = n - node->asm_noperands;
                if (li >= 0 && li < node->asm_ngoto) {
                    const char *prefix = ".L.label.";
                    for (const char *s = prefix; *s && olen < (int)sizeof(out) - 1;) out[olen++] = *s++;
                    for (const char *s = current_fn; *s && olen < (int)sizeof(out) - 1;) out[olen++] = *s++;
                    if (olen < (int)sizeof(out) - 1) out[olen++] = '.';
                    for (const char *s = node->asm_goto_labels[li]; *s && olen < (int)sizeof(out) - 1;) out[olen++] = *s++;
                }
            } else if (*p >= '0' && *p <= '9') {
                int n = *p - '0';
                p++;
                if (n < node->asm_noperands) {
                    const char *s = node->asm_ops[n].asm_str;
                    // Apply modifier
                    if (mod == 'w' && s[0] == 'x') {
                        if (olen < (int)sizeof(out) - 1) out[olen++] = 'w';
                        s++; // skip 'x'
                    } else if (mod == 'd') {
                        // FP double form: dN where N is the reg number
                        if (op_is_fp[n]) {
                            // already stored as "d0" etc
                        } else {
                            if (olen < (int)sizeof(out) - 1) out[olen++] = 'd';
                            // s points to the register name like "x5" → "d5"
                            if (*s == 'x' || *s == 'w') s++;
                            while (*s && olen < (int)sizeof(out) - 1) out[olen++] = *s++;
                            goto next_char;
                        }
                    }
                    while (*s && olen < (int)sizeof(out) - 1) out[olen++] = *s++;
                }
            } else {
                out[olen++] = '%';
                if (mod) out[olen++] = mod;
                // pass through the character that wasn't a modifier/operand
                if (*p && *p != '%') out[olen++] = *p++;
            }
        next_char:;
        }
        out[olen] = '\0';
        if (olen > 0 && !cg_dry_run) {
            // Split on ';' for multi-instruction inline asm (e.g. "mov %0, #1; mov %1, #2")
            char *inst = out;
            while (*inst) {
                char *semi = strchr(inst, ';');
                if (semi) *semi = '\0';
                assemble_inline(cg_obj, inst, cg_inline_fixup_cb, NULL);
                if (!semi) break;
                inst = semi + 1;
                inst += strspn(inst, " \t");
            }
        }

        for (int i = 0; i < node->asm_noperands; i++) {
            AsmOperand *op = &node->asm_ops[i];
            if (op_addr[i] < 0) continue;
            if (op_is_fp[i]) {
                // Store FP result (d0) back to variable
                int sz = op->expr->ty ? op->expr->ty->size : 8;
                if (sz <= 4)
                    asm_str_fp(cg_sec, 0, op_addr[i], 4); // asm_str_fp(0, op_addr[i], 4)
                else
                    asm_str_fp(cg_sec, 0, op_addr[i], 8); // asm_str_fp(0, op_addr[i], 8)
            } else {
                emit_store(op->expr->ty, op_regs[i], op_addr[i], 0);
            }
        }

        // Free registers: value regs first, then address regs
        for (int i = 0; i < node->asm_noperands; i++)
            if (op_regs[i] >= 0) free_reg(op_regs[i]);
        for (int i = 0; i < node->asm_noperands; i++)
            if (op_addr[i] >= 0) free_reg(op_addr[i]);
        return -1;
#else
        // x86-64 inline asm operand evaluation.
        // op_regs[i]: register holding value (or address for memory operands)
        // op_addr[i]: address register for store-back after asm (-1 if none)
        VReg op_regs[MAX_ASM_OPERANDS];
        VReg op_addr[MAX_ASM_OPERANDS];
        for (int i = 0; i < node->asm_noperands; i++) {
            op_regs[i] = -1;
            op_addr[i] = -1;
        }

        // x86-64 specific register constraints: "a"=rax, "b"=rbx, "c"=rcx,
        // "d"=rdx, "S"=rsi, "D"=rdi.  We encode them as sentinel values
        // -(X86Reg+3) so they don't collide with virtual regs (0..7).
        // Sentinels: X86_RAX(0)=-3, RCX(1)=-4, RDX(2)=-5, RBX(3)=-6,
        //            RSI(6)=-9, RDI(7)=-10.
#define ASM_X86_REG(r) (-((int)(r)) - 3)
#define ASM_IS_X86_REG(v) ((v) <= -3 && (v) >= -18)
#define ASM_GET_X86_REG(v) ((X86Reg)(-((v)) - 3))
        static const char *x86_phy_reg64[] = {
            "%rax", "%rcx", "%rdx", "%rbx", "%rsp", "%rbp", "%rsi", "%rdi",
            "%r8", "%r9", "%r10", "%r11", "%r12", "%r13", "%r14", "%r15"};
        static const char *x86_phy_reg32[] = {
            "%eax", "%ecx", "%edx", "%ebx", "%esp", "%ebp", "%esi", "%edi",
            "%r8d", "%r9d", "%r10d", "%r11d", "%r12d", "%r13d", "%r14d", "%r15d"};

        // Helper: map constraint char to X86Reg; returns true on match.
        // Inline switch avoids block/nested-function portability issues.
#define X86REG_CASE(ch, reg) case ch: xreg = reg; is_x86_reg = true; break
        // First pass: allocate registers for outputs and memory operands.
        // Defer matching input constraints (digit) to second pass.
        for (int i = 0; i < node->asm_noperands; i++) {
            AsmOperand *op = &node->asm_ops[i];
            const char *c = op->constraint;
            while (*c == '=' || *c == '+' || *c == '&') c++;

            // Check for x86-specific register constraints
            X86Reg xreg = X86_RAX;
            bool is_x86_reg = false;
            switch (*c) {
                X86REG_CASE('a', X86_RAX);
                X86REG_CASE('b', X86_RBX);
                X86REG_CASE('c', X86_RCX);
                X86REG_CASE('d', X86_RDX);
                X86REG_CASE('S', X86_RSI);
                X86REG_CASE('D', X86_RDI);
            default: break;
            }

            if (is_x86_reg) {
                int sz = op->expr->ty ? op->expr->ty->size : 4;
                const char *rname = (sz == 4) ? x86_phy_reg32[xreg]
                                              : x86_phy_reg64[xreg];
                if (op->is_output && !op->is_rw) {
                    // Output-only x86 reg: capture address for store-back
                    VReg r_addr = gen_addr(op->expr);
                    op_addr[i] = r_addr;
                    op_regs[i] = ASM_X86_REG(xreg);
                    op->reg = -1;
                    snprintf(op->asm_str, sizeof(op->asm_str), "%s", rname);
                } else if (op->is_rw) {
                    // Read-write x86 reg: load current value into physical reg
                    VReg r_addr = gen_addr(op->expr);
                    op_addr[i] = r_addr;
                    x86_mov_rm(cg_sec, sz, xreg, x86_mem(REG(r_addr), 0));
                    op_regs[i] = ASM_X86_REG(xreg);
                    op->reg = -1;
                    snprintf(op->asm_str, sizeof(op->asm_str), "%s", rname);
                } else {
                    // Input x86 reg: move value into physical register
                    VReg r_in = gen(op->expr);
                    x86_mov_rr(cg_sec, (sz > 4 ? 8 : 4), xreg, REG(r_in));
                    free_reg(r_in);
                    op_regs[i] = ASM_X86_REG(xreg);
                    op->reg = -1;
                    snprintf(op->asm_str, sizeof(op->asm_str), "%s", rname);
                }
            } else if (op->is_memory) {
                // "m" constraint: compute address, use as memory ref in template.
                VReg r = gen_addr(op->expr);
                if (r < 0) {
                    // Not an lvalue -- e.g. GMP's own longlong.h count_
                    // leading_zeros/count_trailing_zeros macros pass
                    // "rm" ((UDItype)(x)): the explicit cast makes the
                    // operand expression a genuine rvalue (gen_addr()
                    // correctly refuses an address for a scalar ND_CAST,
                    // "caller can fall back to gen()"), but this is a
                    // memory-alternative-only path with no register
                    // fallback of its own. Real GCC materializes a
                    // temporary and uses its address; do the same via
                    // the shared staging-slot scheme (current_fn_stack_size
                    // + fn_struct_ret_off) other rvalue-needs-an-address
                    // sites in this file already use.
                    VReg val = gen(op->expr);
                    int sz = op->expr->ty ? op->expr->ty->size : 8;
                    int alloc = (sz + 7) & ~7;
                    fn_struct_ret_off += alloc;
                    if (fn_struct_ret_off > fn_struct_ret_total) fn_struct_ret_total = fn_struct_ret_off;
                    int result_off = current_fn_stack_size + fn_struct_ret_off;
                    asm_mov_reg_rbp(cg_sec, val, sz, result_off); // mov val, -result_off(%rbp)
                    free_reg(val);
                    r = alloc_reg();
                    asm_lea_rbp_reg(cg_sec, r, 8, result_off); // lea -result_off(%rbp), r
                }
                op_regs[i] = r;
                op->reg = r;
                snprintf(op->asm_str, sizeof(op->asm_str), "(%s)", reg64[r]);
            } else if (op->is_output && !op->is_rw) {
                // Output-only: "=r" -> allocate fresh register, store back after asm.
                // "=m" is handled above (is_memory).
                VReg r_addr = gen_addr(op->expr);
                op_addr[i] = r_addr;
                VReg r = alloc_reg();
                op_regs[i] = r;
                op->reg = r;
                int sz = op->expr->ty ? op->expr->ty->size : 4;
                snprintf(op->asm_str, sizeof(op->asm_str), "%s", reg(r, sz));
            } else if (op->is_rw) {
                // Read-write "+r": load current value, store back after asm.
                VReg r_addr = gen_addr(op->expr);
                op_addr[i] = r_addr;
                VReg r = alloc_reg();
                op_regs[i] = r;
                op->reg = r;
                int sz = op->expr->ty ? op->expr->ty->size : 4;
                emit_load(op->expr->ty, r, r_addr, 0);
                snprintf(op->asm_str, sizeof(op->asm_str), "%s", reg(r, sz));
            } else if (*c == 'i' || *c == 'n') {
                // Immediate constraint: emit numeric value, or (for an
                // address-of-global, e.g. jump_label.h's arch_static_branch
                // passing "i" (key) where key = &some_static_key) the
                // symbol name as an immediate address, in the template.
                int64_t cval = 0;
                const char *sym = NULL;
                if (try_const_int(op->expr, &cval)) {
                    snprintf(op->asm_str, sizeof(op->asm_str), "$%lld", (long long)cval);
                    op_regs[i] = -1; // no register needed
                } else if (try_const_addr_sym(op->expr, &sym)) {
                    snprintf(op->asm_str, sizeof(op->asm_str), "$%s", sym);
                    op_regs[i] = -1;
                } else {
                    VReg r = gen(op->expr);
                    op_regs[i] = r;
                    op->reg = r;
                    snprintf(op->asm_str, sizeof(op->asm_str), "%s", reg64[r]);
                }
            } else if (*c >= '0' && *c <= '9') {
                // Matching constraint: defer to second pass
                op_regs[i] = -2; // sentinel
            } else {
                // Regular input "r": load value into a register
                VReg r = gen(op->expr);
                op_regs[i] = r;
                op->reg = r;
                int sz = op->expr->ty ? op->expr->ty->size : 4;
                snprintf(op->asm_str, sizeof(op->asm_str), "%s", reg(r, sz));
            }
        }

        // Second pass: resolve matching constraints ("0".."9").
        // Load the input value into the same register as the referenced output.
        for (int i = 0; i < node->asm_noperands; i++) {
            if (op_regs[i] != -2) continue;
            AsmOperand *op = &node->asm_ops[i];
            const char *c = op->constraint;
            while (*c == '=' || *c == '+' || *c == '&') c++;
            int ref = *c - '0';
            if (ref >= 0 && ref < node->asm_noperands) {
                int sz = op->expr->ty ? op->expr->ty->size : 4;
                if (ASM_IS_X86_REG(op_regs[ref])) {
                    // Referenced output uses an x86 physical register
                    X86Reg xreg = ASM_GET_X86_REG(op_regs[ref]);
                    VReg r_in = gen(op->expr);
                    x86_mov_rr(cg_sec, (sz > 4 ? 8 : 4), xreg, REG(r_in));
                    free_reg(r_in);
                    op_regs[i] = ASM_X86_REG(xreg);
                    op->reg = -1;
                    strncpy(op->asm_str, node->asm_ops[ref].asm_str,
                            sizeof(op->asm_str) - 1);
                    op->asm_str[sizeof(op->asm_str) - 1] = '\0';
                } else if (op_regs[ref] >= 0) {
                    VReg r = op_regs[ref];
                    // Load the input value into the shared register
                    VReg r_in = gen(op->expr);
                    if (r_in != r) {
                        if (sz <= 4)
                            asm_mov_reg_reg(cg_sec, r, r_in, 4); // mov r_in -> r (load input into output reg)
                        else
                            asm_mov_reg_reg(cg_sec, r, r_in, 8); // mov r_in -> r
                    }
                    free_reg(r_in);
                    op_regs[i] = r;
                    op->reg = r;
                    strncpy(op->asm_str, node->asm_ops[ref].asm_str,
                            sizeof(op->asm_str) - 1);
                    op->asm_str[sizeof(op->asm_str) - 1] = '\0';
                }
            }
        }

        // Save address registers for x86 register outputs that may be
        // clobbered by the asm (e.g. cpuid overwrites %rbx which may hold
        // the address of an output variable).
        for (int i = 0; i < node->asm_noperands; i++) {
            if (op_addr[i] >= 0 && ASM_IS_X86_REG(op_regs[i]))
                asm_push(cg_sec, REG(op_addr[i]));
        }

        // Emit template with operand substitution, wrapped in AT&T syntax
        /* inline asm already in AT&T */

        // Translate TCC-specific {$}N to GAS-immediate $N in AT&T syntax
        char adj[4096];
        int alen = 0;
        const char *tp = node->asm_template;
        while (*tp && alen < (int)sizeof(adj) - 1) {
            if (*tp == '{' && *(tp + 1) == '$' && *(tp + 2) == '}') {
                adj[alen++] = '$';
                tp += 3;
            } else {
                adj[alen++] = *tp++;
            }
        }
        adj[alen] = '\0';

        // Substitute %N and %l[name] in template
        char out[4096];
        int olen = 0;
        const char *p = adj;
        while (*p && olen < (int)sizeof(out) - 1) {
            if (*p != '%') {
                out[olen++] = *p++;
                continue;
            }
            p++;
            if (*p == '%') {
                out[olen++] = '%';
                p++;
                continue;
            }
            // check for modifier: 'l' (label operand), 'c' (bare constant,
            // no leading '$' — used in data-directive expressions like
            // jump_label.h's ".quad %c0 + %c1 - ." inside its __jump_table
            // entry), or an x86 operand size override (b/w/k/q =
            // 8/16/32/64-bit, h = legacy high byte) — GCC extended-asm
            // syntax used constantly in kernel uaccess/atomic templates to
            // force a specific sub-register name regardless of the
            // operand's natural width, e.g. %k1 to get the 32-bit name of
            // a 64-bit pointer operand.
            char mod = 0;
            if (*p == 'l' || *p == 'c' || *p == 'b' || *p == 'w' || *p == 'k' || *p == 'q' || *p == 'h') { mod = *p++; }
            if (mod == 'l' && *p == '[') {
                // %l[name] -> goto label
                p++;
                const char *end = strchr(p, ']');
                if (!end) {
                    out[olen++] = '%';
                    out[olen++] = 'l';
                    out[olen++] = '[';
                    continue;
                }
                // emit .L.label.<fn>.<name>
                const char *prefix = ".L.label.";
                for (const char *s = prefix; *s && olen < (int)sizeof(out) - 1;) out[olen++] = *s++;
                for (const char *s = current_fn; *s && olen < (int)sizeof(out) - 1;) out[olen++] = *s++;
                if (olen < (int)sizeof(out) - 1) out[olen++] = '.';
                for (const char *s = p; s < end && olen < (int)sizeof(out) - 1;) out[olen++] = *s++;
                p = end + 1;
            } else if (*p == '[') {
                // %[name] / %mod[name] -> named operand reference (GCC
                // extended-asm symbolic names, e.g. `[errout] "+r" (err)`
                // declared in the constraint list) — used pervasively in
                // modern kernel templates instead of positional %0/%1 so
                // adding/reordering operands doesn't require renumbering
                // every reference. parse_asm_stmt already records each
                // operand's bracketed name in AsmOperand.name; look it up
                // here the same way a plain %N looks up by position.
                p++;
                const char *end = strchr(p, ']');
                if (!end) {
                    out[olen++] = '%';
                    if (mod) out[olen++] = mod;
                    out[olen++] = '[';
                    continue;
                }
                char namebuf[64];
                size_t nlen = (size_t)(end - p);
                if (nlen >= sizeof(namebuf)) nlen = sizeof(namebuf) - 1;
                memcpy(namebuf, p, nlen);
                namebuf[nlen] = '\0';
                p = end + 1;
                int found = -1;
                for (int oi = 0; oi < node->asm_noperands; oi++)
                    if (node->asm_ops[oi].name[0] && !strcmp(node->asm_ops[oi].name, namebuf)) {
                        found = oi;
                        break;
                    }
                if (found >= 0) {
                    const char *s = node->asm_ops[found].asm_str;
                    if (mod == 'c' && *s == '$') s++; // bare constant, drop '$'
#ifndef ARCH_ARM64
                    else if (mod && mod != 'l') {
                        int size_class = mod == 'q' ? 0 : mod == 'k' ? 1
                            : mod == 'w'                             ? 2
                            : mod == 'h'                             ? 4
                                                                     : 3;
                        const char *resized = x86_reg_resize(s, size_class);
                        if (resized) s = resized;
                    }
#endif
                    while (*s && olen < (int)sizeof(out) - 1) out[olen++] = *s++;
                }
                // else: unresolved name — drop it, matching the existing
                // %N-out-of-range behavior below rather than corrupting
                // the rest of the template.
            } else if (mod == 'l' && *p >= '0' && *p <= '9') {
                // %lN: positional "asm goto" label reference. GCC numbers
                // goto labels continuing right after the in/out operand
                // indices (not a separate zero-based space), e.g. with 2
                // operands the first goto label is %l2 — see
                // arch/x86/include/asm/uaccess.h's __put_user_goto()
                // ("mov %0,%1" then "_ASM_EXTABLE_UA(1b, %l2)"). Falling
                // through to the plain %N operand path below silently
                // dropped this (out of range for asm_noperands), leaving a
                // fault-handler extable entry with no relocation at all.
                int n = *p - '0';
                p++;
                int li = n - node->asm_noperands;
                if (li >= 0 && li < node->asm_ngoto) {
                    const char *prefix = ".L.label.";
                    for (const char *s = prefix; *s && olen < (int)sizeof(out) - 1;) out[olen++] = *s++;
                    for (const char *s = current_fn; *s && olen < (int)sizeof(out) - 1;) out[olen++] = *s++;
                    if (olen < (int)sizeof(out) - 1) out[olen++] = '.';
                    for (const char *s = node->asm_goto_labels[li]; *s && olen < (int)sizeof(out) - 1;) out[olen++] = *s++;
                }
            } else if (*p >= '0' && *p <= '9') {
                int n = *p - '0';
                p++;
                if (n < node->asm_noperands) {
                    const char *s = node->asm_ops[n].asm_str;
                    if (mod == 'c' && *s == '$') s++; // bare constant, drop '$'
#ifndef ARCH_ARM64
                    else if (mod && mod != 'l') {
                        int size_class = mod == 'q' ? 0 : mod == 'k' ? 1
                            : mod == 'w'                             ? 2
                            : mod == 'h'                             ? 4
                                                                     : 3;
                        const char *resized = x86_reg_resize(s, size_class);
                        if (resized) s = resized;
                    }
#endif
                    while (*s && olen < (int)sizeof(out) - 1) out[olen++] = *s++;
                }
            } else {
                out[olen++] = '%';
                if (mod) out[olen++] = mod;
                // leave other %x as-is
            }
        }
        out[olen] = '\0';
        if (olen > 0 && !cg_dry_run) {
            const char *a = strstr(out, ".ascii");
            if (a) {
                a += 6;
                while (*a == ' ') a++;
                if (*a == '"') {
                    a++;
                    const char *end = strchr(a, '"');
                    if (end) {
                        size_t n = (size_t)(end - a);
                        char *s = arena_alloc(n + 1);
                        memcpy(s, a, n);
                        s[n] = '\0';
                        cg_ascii_add(s);
                    }
                }
            }
            assemble_inline(cg_obj, out, cg_inline_fixup_cb, NULL);
        }

        // Capture x86-physical-register outputs (eax/ebx/ecx/edx/...) into
        // scratch vregs *before* restoring any address registers below.
        // Bug this guards against: op_addr[j] (the address of some OTHER
        // output variable) can be allocated to the very same physical
        // register as this operand's fixed-register constraint (e.g. a
        // "=c" operand's address register happens to land in %rbx, while a
        // *different* operand uses "=b"). The address-register save/restore
        // below exists to protect op_addr[j] from being clobbered by the
        // asm itself (cpuid always writes eax/ebx/ecx/edx); but popping
        // that address back into %rbx would overwrite the still-unread
        // "=b" result first. Reading every fixed-register output out to a
        // scratch vreg here, while the hardware register still holds the
        // asm's real result, makes the store-back below immune to whatever
        // the address-register restore does to those same registers.
        VReg op_saved[MAX_ASM_OPERANDS];
        for (int i = 0; i < node->asm_noperands; i++) {
            if (op_addr[i] >= 0 && ASM_IS_X86_REG(op_regs[i])) {
                X86Reg xreg = ASM_GET_X86_REG(op_regs[i]);
                int sz = node->asm_ops[i].expr->ty ? node->asm_ops[i].expr->ty->size : 4;
                VReg tmp = alloc_reg();
                x86_mov_rr(cg_sec, (sz > 4 ? 8 : 4), REG(tmp), xreg);
                op_saved[i] = tmp;
            } else {
                op_saved[i] = -1;
            }
        }

        // Restore saved address registers (reverse order) so that
        // the store-back below uses the correct address values.
        for (int i = node->asm_noperands - 1; i >= 0; i--) {
            if (op_addr[i] >= 0 && ASM_IS_X86_REG(op_regs[i]))
                asm_pop(cg_sec, REG(op_addr[i]));
        }
        // Store back register outputs ("=r", "+r") to their C variables
        for (int i = 0; i < node->asm_noperands; i++) {
            if (op_addr[i] < 0) continue;
            AsmOperand *op = &node->asm_ops[i];
            int sz = op->expr->ty ? op->expr->ty->size : 4;
            if (ASM_IS_X86_REG(op_regs[i])) {
                // Store the value captured above, not the live physical
                // register (which the address-register pop may have
                // overwritten with an unrelated operand's address).
                x86_mov_mr(cg_sec, sz, x86_mem(REG(op_addr[i]), 0), REG(op_saved[i]));
                free_reg(op_saved[i]);
            } else if (sz == 1)
                asm_mov_reg_mem(cg_sec, op_regs[i], op_addr[i], 1); // movb op_regs[i], (%op_addr[i])
            else if (sz == 2)
                asm_mov_reg_mem(cg_sec, op_regs[i], op_addr[i], 2); // movw op_regs[i], (%op_addr[i])
            else if (sz <= 4)
                asm_mov_reg_mem(cg_sec, op_regs[i], op_addr[i], 4); // movl op_regs[i], (%op_addr[i])
            else
                asm_mov_reg_mem(cg_sec, op_regs[i], op_addr[i], 8); // movq op_regs[i], (%op_addr[i])
        }


        // Free registers (skip x86 physical reg sentinels)
        for (int i = 0; i < node->asm_noperands; i++)
            if (op_regs[i] >= 0) free_reg(op_regs[i]);
        for (int i = 0; i < node->asm_noperands; i++)
            if (op_addr[i] >= 0) free_reg(op_addr[i]);
        return -1;
#endif
    }
    case ND_VA_START: {
#if defined(ARCH_ARM64) && defined(__APPLE__)
        // Apple ARM64: va_list is char*. Point ap to first variadic arg on stack.
        VReg r = gen_addr(node->lhs); // address of the char* variable ap
        asm_add_x16_fp_imm(cg_sec, va_st_start); // add x16, fp, #va_st_start
        asm_str_x16_uoff(cg_sec, r, 0); // str x16, [x{r}]
#elif defined(ARCH_ARM64)
        // AAPCS64: initialize 5-field va_list struct
        VReg r = gen(node->lhs); // va_list is array-of-struct, decays to pointer
        // __stack: pointer to first stack overflow argument
        asm_add_x16_fp_imm(cg_sec, va_st_start); // add x16, x29, #va_st_start
        asm_str_x16_uoff(cg_sec, r, 0); // str x16, [x{r}]
        // __gr_top: end of GP reg save area = saved_sp + 64
        asm_ldur_fp_phy(cg_sec, ARM64_X16, 8); // ldur x16, [x29, #-8]
        asm_add_x16_imm(cg_sec, 64); // add x16, x16, #64
        asm_str_x16_uoff(cg_sec, r, 1); // str x16, [x{r}, #8]
        // __vr_top: end of FP reg save area = saved_sp + 192
        asm_ldur_fp_phy(cg_sec, ARM64_X16, 8); // ldur x16, [x29, #-8]
        asm_add_x16_imm(cg_sec, 192); // add x16, x16, #192
        asm_str_x16_uoff(cg_sec, r, 2); // str x16, [x{r}, #16]
        // __gr_offs: -(8 - gp_param) * 8
        emit_mov_imm64(ARM64_X16, (uint64_t)va_gp_start); // mov w16, #va_gp_start
        asm_str_w16_uoff(cg_sec, r, 6); // str w16, [x{r}, #24]
        // __vr_offs: -(8 - fp_param) * 16
        emit_mov_imm64(ARM64_X16, (uint64_t)va_fp_start); // mov w16, #va_fp_start
        asm_str_w16_uoff(cg_sec, r, 7); // str w16, [x{r}, #28]
#elif defined(_WIN32)
        // Windows x64: va_list is char *. Point to first variadic arg.
        // If named params use < 4 reg slots, the first variadic arg is in the
        // shadow space (rbp+16 for slot 0, +24 for slot 1, etc.).
        // If all 4 reg slots are consumed by named params, the first variadic
        // arg is on the stack at rbp+48 + n_stack*8.
        VReg r = gen_addr(node->lhs); // va_list is char *, need its address to write
        {
            // Use va_gp_start (already includes hidden struct ret ptr) — matches codegen.c.main
            int va_first = (va_gp_start < 32) ? (16 + va_gp_start) : va_st_start;
            // asm_lea_rbp negates offset, so pass -va_first to get +va_first
            asm_lea_rbp(cg_sec, X86_RDX, 8, -va_first); // leaq va_first(%rbp), %rdx
        }
        x86_mov_mr(cg_sec, 8, x86_mem(REG(r), 0), X86_RDX); // movq %rdx, (%r)
#else
        VReg r = gen(node->lhs); // va_list is array-of-struct, decays to pointer
        x86_mov_mi(cg_sec, 4, x86_mem(REG(r), 0), va_gp_start); // movl $va_gp_start, (r)
        x86_mov_mi(cg_sec, 4, x86_mem(REG(r), 4), va_fp_start); // movl $va_fp_start, 4(r)
        asm_lea_rbp(cg_sec, X86_RDX, 8, -va_st_start); // leaq va_st_start(%rbp), %rdx
        x86_mov_mr(cg_sec, 8, x86_mem(REG(r), 8), X86_RDX); // movq %rdx, 8(r)
        asm_lea_rbp(cg_sec, X86_RDX, 8, va_reg_save_ofs); // leaq -va_reg_save_ofs(%rbp), %rdx
        x86_mov_mr(cg_sec, 8, x86_mem(REG(r), 16), X86_RDX); // movq %rdx, 16(r)
#endif
        free_reg(r);
        return -1;
    }
    case ND_VA_COPY: {
#if defined(ARCH_ARM64) && defined(__APPLE__)
        // Apple ARM64: va_list is char*. Just copy the pointer.
        VReg rd = gen_addr(node->lhs); // address of dst char* variable
        VReg rs = gen(node->rhs); // value of src char* (the pointer itself)
        asm_str_reg_off(cg_sec, rs, rd, 8, 0); // str rs, [rd]
        free_reg(rd);
        free_reg(rs);
#elif defined(ARCH_ARM64)
        VReg rd = gen(node->lhs);
        VReg rs = gen(node->rhs);
        // AAPCS64 va_list is 32 bytes: copy all 4 x 8-byte words (use x16 as temp)
        for (int _vi = 0; _vi < 4; _vi++) {
            asm_ldr_x16_uoff(cg_sec, rs, _vi); // ldr x16, [x{rs}, #_vi*8]
            asm_str_x16_uoff(cg_sec, rd, _vi); // str x16, [x{rd}, #_vi*8]
        }
        free_reg(rd);
        free_reg(rs);
#elif defined(_WIN32)
        // Windows x64: va_list is char* (8 bytes). Just copy the pointer.
        VReg rd = gen_addr(node->lhs); // address of dst char* variable
        VReg rs = gen(node->rhs); // value of src char* (the pointer itself)
        x86_mov_mr(cg_sec, 8, x86_mem(REG(rd), 0), REG(rs)); // movq rs, (%rd)
        free_reg(rd);
        free_reg(rs);
#else
        VReg rd = gen(node->lhs);
        asm_push(cg_sec, REG(rd)); // pushq rd
        free_reg(rd);
        VReg rs = gen(node->rhs);
        VReg rpop = alloc_reg();
        asm_pop(cg_sec, REG(rpop)); // popq rpop
        x86_mov_rm(cg_sec, 8, X86_RCX, x86_mem(REG(rs), 0)); // movq (rs), %rcx
        asm_mov_phy_base_off(cg_sec, X86_RCX, rpop, 0, 8); // movq %rcx, 0(%rpop)
        x86_mov_rm(cg_sec, 8, X86_RCX, x86_mem(REG(rs), 8)); // movq 8(rs), %rcx
        asm_mov_phy_base_off(cg_sec, X86_RCX, rpop, 8, 8); // movq %rcx, 8(%rpop)
        x86_mov_rm(cg_sec, 8, X86_RCX, x86_mem(REG(rs), 16)); // movq 16(rs), %rcx
        asm_mov_phy_base_off(cg_sec, X86_RCX, rpop, 16, 8); // movq %rcx, 16(%rpop)
        free_reg(rs);
        free_reg(rpop);
#endif
        return -1;
    }
    case ND_VA_ARG_PACK:
    case ND_VA_ARG_PACK_LEN: {
        // Only valid inside a function inlined by the call-site
        // __builtin_va_arg_pack expander in the parser; if one reaches
        // codegen (e.g. unused inline-pack function), emit a harmless 0.
        int r = alloc_reg();
#ifdef ARCH_ARM64
        asm_movq_zero(cg_sec, r);
#else
        asm_movl_zero(cg_sec, r);
#endif
        return r;
    }
    case ND_VA_ARG: {
        Type *ty = node->ty->base;
#if defined(ARCH_ARM64) && defined(__APPLE__)
        // Apple ARM64: va_list is char*; all variadic args are 8-byte slots on the stack.
        // slot_sz = ceil(size / 8) * 8; zero-size types advance by 0.
        VReg apple_r = gen_addr(node->lhs); // address of the char* variable ap
        int slot_sz = (ty->size + 7) & ~7;
        // Use x17 (reserved scratch) to hold &ap before loading x12.
        // Without this, if gen_addr returns x12, "ldr x12,[x12]" clobbers the
        // address and "str x16,[x12]" writes new_ap into the data area instead
        // of back into the ap variable.
        asm_mov_phy_reg(cg_sec, ARM64_X17, apple_r, 1); // mov x17, x{apple_r}
        arm64_ldr_uoff(cg_sec, 3, ARM64_X12, ARM64_X17, 0); // ldr x12, [x17]
        // __int128 requires 16-byte alignment; the caller pads stack_args to even
        // before placing it, so va_arg must align ap up to 16 to match.
        if (ty->kind == TY_INT128) {
            arm64_add_imm(cg_sec, 1, ARM64_X16, ARM64_X12, 15, 0); // add x16, x12, #15
            arm64_and_imm(cg_sec, 1, ARM64_X12, ARM64_X16, ~15ULL); // and x12, x16, #-16
        }
        arm64_add_imm(cg_sec, 1, ARM64_X16, ARM64_X12, slot_sz, 0); // add x16, x12, #slot_sz
        asm_str_reg(cg_sec, ARM64_X16, ARM64_X17); // str x16, [x17]
        free_reg(apple_r);
        VReg ret = alloc_reg();
        asm_mov_vreg_x12(cg_sec, ret); // mov x{ret}, x12
        return ret;
#else
#ifndef _WIN32
        bool is_fp = is_flonum(ty);
#endif
        VReg r;
#ifdef ARCH_ARM64
        r = gen(node->lhs);
        bool is_ptr_struct = (ty->kind == TY_STRUCT || ty->kind == TY_UNION) && ty->size > 16;
        // rcc passes ALL structs >8 bytes by pointer (not by value).
        // va_arg must read 8-byte pointer from reg save area, then dereference.
        bool is_ptr_val_struct = (ty->kind == TY_STRUCT || ty->kind == TY_UNION) && ty->size > 8;

        // AArch64 ABI va_list: [__stack(8), __gr_top(8), __vr_top(8), __gr_offs(4), __vr_offs(4)]
        // gr_offs < 0 means reg arg available; addr = __gr_top + __gr_offs
        // Only use FP path for bare float types; structs (incl. HFA) always go in GP regs.
        // AArch64 ABI va_list: [__stack(8), __gr_top(8), __vr_top(8), __gr_offs(4), __vr_offs(4)]
        // gr_offs < 0 means reg arg available; addr = __gr_top + __gr_offs
        // Only use FP path for bare float types; structs (incl. HFA) always go in GP regs.
        if (is_fp) {
            int fp_size = 16;
            // ldr w16, [x{r}, #28]  — __vr_offs
            asm_ldr_w16_uoff(cg_sec, r, 7); // ldr w16, [x{r}, #28]
            arm64_subs_imm(cg_sec, 0, ARM64_XZR, ARM64_X16, 0, 0); // cmp w16, #0
            {
                size_t _cj = asm_jcc_label(cg_sec, ARM64_GE);
                asm_fixup_add(cg_sec, _cj, format(".L.va_overflow.%d", rcc_label_count), 1);
            }
            // Store new vr_offs before loading vr_top (may clobber reg holding r)
            asm_add_w17_w16_imm(cg_sec, fp_size); // add w17, w16, #fp_size
            // str w17, [x{r}, #28]
            asm_str_w17_uoff(cg_sec, r, 7); // str w17, [x{r}, #28]
            // ldr x12, [x{r}, #16]  — __vr_top (safe to clobber r now)
            asm_ldr_x12_uoff(cg_sec, r, 2); // ldr x12, [x{r}, #16]
            asm_sxtw(cg_sec, ARM64_X17, ARM64_X16); // sxtw x17, w16
            asm_add_x12_x12_x17(cg_sec); // add x12, x12, x17
            if (ty->kind == TY_LDOUBLE) {
                // x12 now addresses the 16-byte VR-save-area slot holding a
                // genuine binary128 value (the call-site pre-pass widens
                // through __extenddftf2 before every variadic call - see
                // above). rcc's internal `long double` is a plain 64-bit
                // double everywhere else, so narrow the incoming quad back
                // down via libgcc's __trunctfdf2 before handing the
                // address back: reading the raw quad's low 64 bits as a
                // double would reinterpret binary128's mantissa/exponent
                // layout as binary64 and produce garbage (typically ~0.0).
                // x12 must survive the `bl` (a caller-saved scratch reg
                // isn't guaranteed to), so park it on the stack around it.
                arm64_ldr_fp(cg_sec, 4, ASM_Q0, ARM64_X12, 0); // ldr q0, [x12]
                arm64_sub_imm(cg_sec, 1, ARM64_SP, ARM64_SP, 16, 0); // sub sp, sp, #16
                arm64_str_uoff(cg_sec, 3, ARM64_X12, ARM64_SP, 0); // str x12, [sp]
                emit_direct_call("__trunctfdf2", false); // bl __trunctfdf2 (quad in v0 -> d0)
                arm64_ldr_uoff(cg_sec, 3, ARM64_X12, ARM64_SP, 0); // ldr x12, [sp]
                arm64_add_imm(cg_sec, 1, ARM64_SP, ARM64_SP, 16, 0); // add sp, sp, #16
                arm64_str_fp(cg_sec, 3, ARM64_D0, ARM64_X12, 0); // str d0, [x12]
            }
            {
                size_t _jmp = asm_jmp_label(cg_sec);
                asm_fixup_add(cg_sec, _jmp, format(".L.va_done.%d", rcc_label_count), 0);
            }
        } else {
            int gp_size = 8;
#if defined(__APPLE__) && defined(ARCH_ARM64)
            // Apple ARM64 variadic: ALL struct args go on overflow stack, not GP regs.
            // HFA structs ≤8 bytes (e.g. {float}, {double}) are not is_ptr_val_struct
            // but still skip the GP save area because they went directly to stack.
            bool is_struct = (ty->kind == TY_STRUCT || ty->kind == TY_UNION);
            if (is_struct && !is_ptr_val_struct) {
                // Force branch to overflow — gp_size irrelevant, struct on stack
                // cmp w16, #0; b.ge .L.va_overflow — but we want ALWAYS overflow
                size_t _force_jmp = asm_jmp_label(cg_sec);
                asm_fixup_add(cg_sec, _force_jmp, format(".L.va_overflow.%d", rcc_label_count), 0);
            } else
#endif
            {
                // ldr w16, [x{r}, #24]  — __gr_offs
                asm_ldr_w16_uoff(cg_sec, r, 6); // ldr w16, [x{r}, #24]
                arm64_subs_imm(cg_sec, 0, ARM64_XZR, ARM64_X16, 0, 0); // cmp w16, #0
                {
                    size_t _cj = asm_jcc_label(cg_sec, ARM64_GE);
                    asm_fixup_add(cg_sec, _cj, format(".L.va_overflow.%d", rcc_label_count), 1);
                }
                // Store new gr_offs before loading gr_top (may clobber reg holding r)
                asm_add_w17_w16_imm(cg_sec, gp_size); // add w17, w16, #gp_size
                // str w17, [x{r}, #24]
                asm_str_w17_uoff(cg_sec, r, 6); // str w17, [x{r}, #24]
                // ldr x12, [x{r}, #8]  — __gr_top (safe to clobber r now)
                asm_ldr_x12_uoff(cg_sec, r, 1); // ldr x12, [x{r}, #8]
                asm_sxtw(cg_sec, ARM64_X17, ARM64_X16); // sxtw x17, w16
                asm_add_x12_x12_x17(cg_sec); // add x12, x12, x17
                if (is_ptr_val_struct) {
                    asm_ldr_x12_0(cg_sec); // ldr x12, [x12]
                }
                {
                    size_t _jmp = asm_jmp_label(cg_sec);
                    asm_fixup_add(cg_sec, _jmp, format(".L.va_done.%d", rcc_label_count), 0);
                }
            }
        }

        cg_def_label(format(".L.va_overflow.%d", rcc_label_count));
        // ldr x16, [x{r}]  — __stack (use x16 so va_list ptr r is not clobbered)
        asm_ldr_x16_uoff(cg_sec, r, 0); // ldr x16, [x{r}]
#if defined(__APPLE__) && defined(ARCH_ARM64)
        if (is_ptr_val_struct) {
            // Apple ARM64: variadic struct by value on overflow stack — data is inline, no ptr deref.
            // Just advance __stack past the struct data and set x12 to original __stack.
            int ovf_sz = (ty->size + 7) & ~7;
            if (ovf_sz <= 4095)
                arm64_add_imm(cg_sec, 1, ARM64_X17, ARM64_X16, ovf_sz, 0); // add x17, x16, #ovf_sz
            else {
                emit_mov_imm64(ARM64_X17, (uint64_t)ovf_sz);
                arm64_add_reg(cg_sec, 1, ARM64_X17, ARM64_X16, ARM64_X17, ARM64_LSL, 0); // add x17, x16, x17
            }
            asm_str_x17_uoff(cg_sec, r, 0); // str x17, [x{r}]  — store updated __stack
            asm_mov_x12_x16(cg_sec); // mov x12, x16  — result = original __stack
        } else if (is_ptr_struct)
#else
        if (is_ptr_struct || is_ptr_val_struct)
#endif
        {
            // Pointer-passed struct: load pointer from overflow stack, advance __stack by 8
            asm_ldr_x12_x16_0(cg_sec); // ldr x12, [x16]  — deref __stack
            arm64_add_imm(cg_sec, 1, ARM64_X17, ARM64_X16, 8, 0); // add x17, x16, #8
            asm_str_x17_uoff(cg_sec, r, 0); // str x17, [x{r}]  — store new __stack
            // x12 now holds the dereferenced pointer = result
        } else {
            // Non-struct arg on overflow stack: advance __stack by alignment
            int align = ty->align;
            int ovf_size = ty->size <= 8 ? 8 : (ty->size + 7) & ~7;
            if (align > 8) {
                arm64_add_imm(cg_sec, 1, ARM64_X17, ARM64_X16, align - 1, 0); // add x17, x16, #align-1
                arm64_and_imm(cg_sec, 1, ARM64_X17, ARM64_X17, (uint64_t)(int64_t)(-align)); // and x17, x17, #-align
                arm64_add_imm(cg_sec, 1, ARM64_X17, ARM64_X17, ovf_size, 0); // add x17, x17, #ovf_size
            } else {
                arm64_add_imm(cg_sec, 1, ARM64_X17, ARM64_X16, ovf_size, 0); // add x17, x16, #ovf_size
            }
            asm_str_x17_uoff(cg_sec, r, 0); // str x17, [x{r}]  — store new __stack
            asm_mov_x12_x16(cg_sec); // mov x12, x16  — result = original __stack
        }

        cg_def_label(format(".L.va_done.%d", rcc_label_count));
        cg_def_label(format(".L.va_done_x.%d", rcc_label_count)); // extra for jmp fixup compatibility
        asm_mov_vreg_x12(cg_sec, r); // mov x{r}, x12
#elif defined(_WIN32)
        // Windows x64: va_list is char *. Read arg from current ap, advance by 8.
        r = gen_addr(node->lhs); // va_list is char *, need its address to advance
        bool is_ptr_struct = (ty->kind == TY_STRUCT || ty->kind == TY_UNION) && ty->size > 8;
        // __int128 is also passed by pointer on Windows x64 (like structs > 8 bytes)
        bool is_by_ptr = is_ptr_struct || ty->kind == TY_INT128;
        if (is_by_ptr) {
            // Struct/int128 >8 bytes passed by pointer: slot holds data pointer
            x86_mov_rm(cg_sec, 8, X86_RCX, x86_mem(REG(r), 0)); // movq (%r), %rcx  [rcx = ap]
            x86_mov_rm(cg_sec, 8, X86_RCX, x86_mem(X86_RCX, 0)); // movq (%rcx), %rcx [rcx = data ptr]
            x86_add_mi(cg_sec, 8, x86_mem(REG(r), 0), 8); // addq $8, (%r) [ap += 8]
            x86_mov_rr(cg_sec, 8, REG(r), X86_RCX); // movq %rcx, r
        } else {
            // Return old ap (address of arg slot), then advance
            x86_mov_rm(cg_sec, 8, X86_RCX, x86_mem(REG(r), 0)); // movq (%r), %rcx  [rcx = ap]
            x86_add_mi(cg_sec, 8, x86_mem(REG(r), 0), 8); // addq $8, (%r) [ap += 8]
            x86_mov_rr(cg_sec, 8, REG(r), X86_RCX); // movq %rcx, r  [result = old ap]
        }
#else
        r = gen(node->lhs);
        bool is_ptr_struct = (ty->kind == TY_STRUCT || ty->kind == TY_UNION) && ty->size > 8;
        // va_arg x86: r points to va_list struct {gr_offs, fp_offs, overflow_arg_area, reg_save_area}
        X86Reg xr = REG(r);
        int vaf = is_fp ? 4 : 0; // offset of fp_offs or gr_offs
        int limit = is_fp ? 160 : 40; // max offset before overflow
        int step = is_fp ? 16 : 8; // increment
        if (ty->kind == TY_LDOUBLE) {
            // long double (80-bit extended in a 16-byte slot) is classified
            // as MEMORY by the x86-64 SysV ABI and is never passed in SSE
            // registers, even for variadic calls — read it straight from the
            // overflow area, 16-byte aligned, and advance by 16. The slot
            // holds an x87 80-bit extended value; rcc's long double is
            // truncated to a 64-bit double everywhere else, so convert it
            // in place (the slot is dead after this read) and hand back a
            // pointer to the resulting double.
            x86_mov_rm(cg_sec, 8, X86_RCX, x86_mem(xr, 8)); // movq 8(r), %rcx
            x86_add_ri(cg_sec, 8, X86_RCX, 15); // addq $15, %rcx
            x86_and_ri(cg_sec, 8, X86_RCX, -16); // andq $-16, %rcx
            x86_lea(cg_sec, 8, X86_RDX, x86_mem(X86_RCX, 16)); // leaq 16(%rcx), %rdx
            x86_mov_mr(cg_sec, 8, x86_mem(xr, 8), X86_RDX); // movq %rdx, 8(r)
            x86_fldt_m(cg_sec, x86_mem(X86_RCX, 0)); // fldt (%rcx)
            x86_fstpl_m(cg_sec, x86_mem(X86_RCX, 0)); // fstpl (%rcx)
            {
                size_t _jmp = asm_jmp_label(cg_sec);
                asm_fixup_add(cg_sec, _jmp, format(".L.va_done.%d", rcc_label_count), 0);
            }
        } else if (is_fp) {
            x86_cmp_mi(cg_sec, 4, x86_mem(xr, 4), limit); // cmpl $160, 4(r)
            {
                size_t _cj = asm_jcc_label(cg_sec, X86_A);
                asm_fixup_add(cg_sec, _cj, format(".L.va_overflow.%d", rcc_label_count), 1);
            }
            x86_mov_rm(cg_sec, 4, X86_RCX, x86_mem(xr, 4)); // movl 4(r), %ecx
            x86_add_rm(cg_sec, 8, X86_RCX, x86_mem(xr, 16)); // addq 16(r), %rcx
            x86_add_mi(cg_sec, 4, x86_mem(xr, 4), step); // addl $16, 4(r)
            {
                size_t _jmp = asm_jmp_label(cg_sec);
                asm_fixup_add(cg_sec, _jmp, format(".L.va_done.%d", rcc_label_count), 0);
            }
        } else if (is_ptr_struct) {
            x86_cmp_mi(cg_sec, 4, x86_mem(xr, 0), limit); // cmpl $40, (r)
            {
                size_t _cj = asm_jcc_label(cg_sec, X86_A);
                asm_fixup_add(cg_sec, _cj, format(".L.va_overflow.%d", rcc_label_count), 1);
            }
            x86_mov_rm(cg_sec, 4, X86_RCX, x86_mem(xr, 0)); // movl (r), %ecx
            x86_add_rm(cg_sec, 8, X86_RCX, x86_mem(xr, 16)); // addq 16(r), %rcx
            x86_mov_rm(cg_sec, 8, X86_RCX, x86_mem(X86_RCX, 0)); // movq (%rcx), %rcx
            x86_add_mi(cg_sec, 4, x86_mem(xr, 0), step); // addl $8, (r)
            {
                size_t _jmp = asm_jmp_label(cg_sec);
                asm_fixup_add(cg_sec, _jmp, format(".L.va_done.%d", rcc_label_count), 0);
            }
        } else if (ty->kind == TY_INT128) {
            // int128 needs 2 consecutive GP reg slots (16 bytes)
            x86_cmp_mi(cg_sec, 4, x86_mem(xr, 0), 32); // cmpl $32, (r)
            {
                size_t _cj = asm_jcc_label(cg_sec, X86_A);
                asm_fixup_add(cg_sec, _cj, format(".L.va_overflow.%d", rcc_label_count), 1);
            }
            x86_mov_rm(cg_sec, 4, X86_RCX, x86_mem(xr, 0)); // movl (r), %ecx
            x86_add_rm(cg_sec, 8, X86_RCX, x86_mem(xr, 16)); // addq 16(r), %rcx
            x86_add_mi(cg_sec, 4, x86_mem(xr, 0), 16); // addl $16, (r)
            {
                size_t _jmp = asm_jmp_label(cg_sec);
                asm_fixup_add(cg_sec, _jmp, format(".L.va_done.%d", rcc_label_count), 0);
            }
        } else {
            x86_cmp_mi(cg_sec, 4, x86_mem(xr, 0), limit); // cmpl $40, (r)
            {
                size_t _cj = asm_jcc_label(cg_sec, X86_A);
                asm_fixup_add(cg_sec, _cj, format(".L.va_overflow.%d", rcc_label_count), 1);
            }
            x86_mov_rm(cg_sec, 4, X86_RCX, x86_mem(xr, 0)); // movl (r), %ecx
            x86_add_rm(cg_sec, 8, X86_RCX, x86_mem(xr, 16)); // addq 16(r), %rcx
            x86_add_mi(cg_sec, 4, x86_mem(xr, 0), step); // addl $8, (r)
            {
                size_t _jmp = asm_jmp_label(cg_sec);
                asm_fixup_add(cg_sec, _jmp, format(".L.va_done.%d", rcc_label_count), 0);
            }
        }
        cg_def_label(format(".L.va_overflow.%d", rcc_label_count));
        x86_mov_rm(cg_sec, 8, X86_RCX, x86_mem(xr, 8)); // movq 8(r), %rcx
        if (ty->kind == TY_INT128) {
            // int128 overflow: 16-byte aligned, advance by 16
            x86_add_ri(cg_sec, 8, X86_RCX, 15); // addq $15, %rcx
            x86_and_ri(cg_sec, 8, X86_RCX, -16); // andq $-16, %rcx
            x86_lea(cg_sec, 8, X86_RDX, x86_mem(X86_RCX, 16)); // leaq 16(%rcx), %rdx
            x86_mov_mr(cg_sec, 8, x86_mem(xr, 8), X86_RDX); // movq %rdx, 8(r)
        } else {
            { // leaq 8(%rcx), %rdx
                X86Mem ml = {X86_RCX, X86_NOREG, 1, 8};
                x86_lea(cg_sec, 8, X86_RDX, ml);
            }
            x86_mov_mr(cg_sec, 8, x86_mem(xr, 8), X86_RDX); // movq %rdx, 8(r)
            if (is_ptr_struct)
                x86_mov_rm(cg_sec, 8, X86_RCX, x86_mem(X86_RCX, 0)); // movq (%rcx), %rcx
        }
        cg_def_label(format(".L.va_done.%d", rcc_label_count));
        x86_mov_rr(cg_sec, 8, REG(r), X86_RCX); // movq %rcx, rr (va_arg result)
        (void)vaf;
#endif
        rcc_label_count++;
        return r;
#endif
    }

    case ND_ATOMIC_LOAD: {
        VReg r_addr = gen(node->lhs);
        int sz = node->ty->size;
        VReg r = alloc_reg();
        int ord = node->atomic_ord;
        if (is_flonum(node->ty)) {
            // A flonum VALUE living in a GP register is represented as
            // its DOUBLE bit pattern (see ND_DEREF's identical
            // is_flonum branch), not the same-size raw bytes a bare
            // integer load would produce -- every later consumer
            // (fmov/movq to an xmm register) expects that convention.
            // A plain byte-for-byte load of a `float` would leave its
            // 4-byte pattern zero-extended into the low 32 bits, which
            // gets misread as a genuine (garbage) DOUBLE bit pattern.
#ifdef ARCH_ARM64
            if (sz == 4) {
                asm_ldr_fp(cg_sec, 0, r_addr, 4); // ldr s0, [r_addr]
                asm_fcvt(cg_sec, 1, 0, 0, 0); // fcvt d0, s0
            } else {
                asm_ldr_fp(cg_sec, 0, r_addr, 8); // ldr d0, [r_addr]
            }
            asm_fmov_f2i(cg_sec, r, 0, 1); // fmov x{r}, d0
            // ldr/ldur are already atomic for a naturally-aligned
            // load; a plain barrier (rather than hand-rolling a raw-GP
            // ldar + bit-reinterpret-to-FP sequence) is sufficient to
            // provide the ACQUIRE ordering an integer ldar would add.
            bool use_acquire = (ord == MEMORDER_ACQUIRE || ord == MEMORDER_ACQ_REL || ord == MEMORDER_SEQ_CST || ord == MEMORDER_CONSUME);
            if (use_acquire)
                asm_dmb(cg_sec); // dmb ish
#else
            if (sz == 4) {
                asm_ldr_fp(cg_sec, 0, r_addr, 4); // movss (r_addr), %xmm0
                asm_cvtss2sd(cg_sec); // cvtss2sd %xmm0, %xmm0
            } else {
                asm_ldr_fp(cg_sec, 0, r_addr, 8); // movsd (r_addr), %xmm0
            }
            asm_movq_xmm_r(cg_sec, r, X86_XMM0); // movq %xmm0, r
            if (ord == MEMORDER_SEQ_CST)
                asm_mfence(cg_sec); // mfence
#endif
            free_reg(r_addr);
            return r;
        }
#ifdef ARCH_ARM64
        bool use_acquire = (ord == MEMORDER_ACQUIRE || ord == MEMORDER_ACQ_REL || ord == MEMORDER_SEQ_CST || ord == MEMORDER_CONSUME);
        if (use_acquire) {
            if (sz == 1) asm_ldarb(cg_sec, r, r_addr); // ldarb r, [r_addr]
            else if (sz == 2)
                asm_ldarh(cg_sec, r, r_addr); // ldarh r, [r_addr]
            else
                asm_ldar(cg_sec, r, r_addr, sz); // ldar r, [r_addr]
        } else
            emit_load(node->ty, r, r_addr, 0);
#else
        if (sz < 4) {
            if (sz == 1) {
                if (use_unsigned(node->ty))
                    asm_movzx_mem_reg(cg_sec, r, r_addr, 4, 1); // movzbl (%r_addr), rr
                else
                    asm_movsx_mem_reg(cg_sec, r, r_addr, 4, 1); // movsbl (%r_addr), rr
            } else {
                if (use_unsigned(node->ty))
                    asm_movzx_mem_reg(cg_sec, r, r_addr, 4, 2); // movzwl (%r_addr), rr
                else
                    asm_movsx_mem_reg(cg_sec, r, r_addr, 4, 2); // movswl (%r_addr), rr
            }
        } else if (sz == 4) {
            asm_mov_mem_reg(cg_sec, r, r_addr, 4); // movl (%r_addr), rr
        } else {
            asm_mov_mem_reg(cg_sec, r, r_addr, 8); // movq (%r_addr), rr
        }
        if (ord == MEMORDER_SEQ_CST)
            asm_mfence(cg_sec); // mfence
#endif
        free_reg(r_addr);
#ifdef ARCH_ARM64
        if (use_acquire && sz < 8 && !use_unsigned(node->ty))
            sign_extend_to(r, sz, 8);
#endif
        return r;
    }
    case ND_ATOMIC_STORE: {
        VReg r_addr = gen(node->lhs);
        VReg r_val = gen(node->rhs);
        int sz = node->lhs->ty && node->lhs->ty->base ? node->lhs->ty->base->size : 4;
        if (sz < 1 || sz > 8) sz = 4;
        int ord = node->atomic_ord;
#ifdef ARCH_ARM64
        bool use_release = (ord == MEMORDER_RELEASE || ord == MEMORDER_ACQ_REL || ord == MEMORDER_SEQ_CST);
        if (use_release) {
            if (sz == 1) asm_stlrb(cg_sec, r_val, r_addr); // stlrb r_val, [r_addr]
            else if (sz == 2)
                asm_stlrh(cg_sec, r_val, r_addr); // stlrh r_val, [r_addr]
            else
                asm_stlr(cg_sec, r_val, r_addr, sz); // stlr r_val, [r_addr]
        } else
            emit_store(node->lhs->ty->base ? node->lhs->ty->base : ty_int, r_val, r_addr, 0);
        if (ord == MEMORDER_SEQ_CST)
            asm_dmb(cg_sec); // mfence
#else
        asm_mov_reg_mem(cg_sec, r_val, r_addr, sz); // mov r_val, (r_addr)
        if (ord == MEMORDER_SEQ_CST)
            asm_mfence(cg_sec); // mfence
#endif
        free_reg(r_val);
        free_reg(r_addr);
        return -1;
    }
    case ND_ATOMIC_EXCHANGE: {
        VReg r_addr = gen(node->lhs);
        VReg r_val = gen(node->rhs);
        int sz = node->ty->size;
        VReg r_result = alloc_reg();
#ifdef ARCH_ARM64
        int lbl = rcc_label_count++;
        cg_def_label(format(".L.atom_xchg.%d", lbl));
        asm_ldxr(cg_sec, r_result, r_addr, sz); // ldxr[b/h] r_result, [r_addr]
        asm_stxr(cg_sec, r_val, r_addr, sz); // stxr[b/h] w9, r_val, [r_addr]
        {
            size_t _cj = cg_sec->len;
            arm64_cbnz(cg_sec, 0, ARM64_X9, 0); // cbnz w9, .L.atom_xchg.lbl
            asm_record(ASM_JCC, _cj, 1, ARM64_X9, -1, -1, 4, 0, 0, NULL, ARM64_NE, -1, false);
            asm_fixup_add(cg_sec, _cj, format(".L.atom_xchg.%d", lbl), 1);
        }
        if (node->atomic_ord == MEMORDER_SEQ_CST || node->atomic_ord == MEMORDER_ACQ_REL)
            asm_dmb(cg_sec); // dmb ish
#else
        asm_xchg_mem(cg_sec, r_addr, r_val, sz); // xchg (r_addr), r_val
        if (sz < 4) {
            asm_mov_reg_reg(cg_sec, r_result, r_val, 8); // mov rr_val -> rr_result
            if (use_unsigned(node->ty))
                zero_extend_to(r_result, sz, 4);
            else
                sign_extend_to(r_result, sz, 4);
        } else {
            asm_mov_reg_reg(cg_sec, r_result, r_val, 8); // mov rr_val -> rr_result
        }
#endif
        free_reg(r_val);
        free_reg(r_addr);
#ifdef ARCH_ARM64
        if (sz < 8 && !use_unsigned(node->ty))
            sign_extend_to(r_result, sz, 8);
#endif
        return r_result;
    }
    case ND_ATOMIC_CAS: {
        VReg r_addr = gen(node->lhs);
        VReg r_expectedaddr = gen(node->body);
        VReg r_desired = gen(node->rhs);
        int sz = node->lhs->ty && node->lhs->ty->base ? node->lhs->ty->base->size : 4;
        VReg r_result = alloc_reg();
#ifdef ARCH_ARM64
        VReg r_expected = alloc_reg();
        Type *elem_ty = node->lhs->ty && node->lhs->ty->base ? node->lhs->ty->base : ty_int;
        emit_load(elem_ty, r_expected, r_expectedaddr, 0);
        VReg r_old = alloc_reg();
        int lbl = rcc_label_count++;
        cg_def_label(format(".L.atom_cas.%d", lbl));
        asm_ldxr(cg_sec, r_old, r_addr, sz); // ldxr[b/h] r_old, [r_addr]
        asm_cmp_reg_reg(cg_sec, r_old, r_expected, sz > 4 ? 8 : 4); // cmp r_old, r_expected
        {
            size_t _cj = asm_jcc_label(cg_sec, ARM64_NE);
            asm_fixup_add(cg_sec, _cj, format(".L.atom_cas_fail.%d", lbl), 1);
        }
        asm_stxr(cg_sec, r_desired, r_addr, sz); // stxr[b/h] w9, r_desired, [r_addr]
        if (node->atomic_weak) {
            size_t _cj = cg_sec->len;
            arm64_cbnz(cg_sec, 0, ARM64_X9, 0); // cbnz w9, .L.atom_cas_fail.lbl
            asm_record(ASM_JCC, _cj, 1, ARM64_X9, -1, -1, 4, 0, 0, NULL, ARM64_NE, -1, false);
            asm_fixup_add(cg_sec, _cj, format(".L.atom_cas_fail.%d", lbl), 1);
        } else {
            size_t _cj = cg_sec->len;
            arm64_cbnz(cg_sec, 0, ARM64_X9, 0); // cbnz w9, .L.atom_cas.lbl
            asm_record(ASM_JCC, _cj, 1, ARM64_X9, -1, -1, 4, 0, 0, NULL, ARM64_NE, -1, false);
            asm_fixup_add(cg_sec, _cj, format(".L.atom_cas.%d", lbl), 1);
        }
        if (node->atomic_cas_return_old)
            asm_mov_reg_reg(cg_sec, r_result, r_old, 8); // __sync_val_compare_and_swap: return the value we just compared against (success path)
        else
            asm_mov_imm(cg_sec, r_result, 8, 1); // mov $1, rr_result
        {
            size_t _jmp = asm_jmp_label(cg_sec);
            asm_fixup_add(cg_sec, _jmp, format(".L.atom_cas_done.%d", lbl), 0);
        }
        cg_def_label(format(".L.atom_cas_fail.%d", lbl));
        if (node->atomic_cas_return_old)
            asm_mov_reg_reg(cg_sec, r_result, r_old, 8); // __sync_val_compare_and_swap: return the actual current value (failure path)
        else
            asm_movq_zero(cg_sec, r_result); // xor rr_result, rr_result
        emit_store(elem_ty, r_old, r_expectedaddr, 0);
        cg_def_label(format(".L.atom_cas_done.%d", lbl));
        free_reg(r_old);
        free_reg(r_expected);
#else
        VReg r_expected = alloc_reg();
        // Load expected value from r_expectedaddr into RAX
        {
            X86Mem mex = {REG(r_expectedaddr), X86_NOREG, 1, 0};
            if (sz == 1) x86_movzx_rm(cg_sec, 4, 1, X86_RAX, mex); // movzbl (r_expectedaddr), %eax
            else if (sz == 2)
                x86_movzx_rm(cg_sec, 4, 2, X86_RAX, mex); // movzwl (r_expectedaddr), %eax
            else if (sz == 4)
                x86_mov_rm(cg_sec, 4, X86_RAX, mex); // movl (r_expectedaddr), %eax
            else
                x86_mov_rm(cg_sec, 8, X86_RAX, mex); // movq (r_expectedaddr), %rax
        }
        asm_lock_cmpxchg_mem(cg_sec, r_addr, r_desired, sz); // lock cmpxchg (r_addr), r_desired
        if (node->atomic_cas_return_old) {
            // __sync_val_compare_and_swap returns the actual OLD *ptr
            // value: cmpxchg leaves it in RAX either way (unchanged on
            // success since it already matched; updated to the real
            // current value on failure), unlike __sync_bool_/
            // __atomic_compare_exchange's success/fail sete boolean.
            asm_mov_retval(cg_sec, r_result, 8); // mov %rax, r_result
        } else {
            asm_sete(cg_sec, r_result); // sete r_result8
            asm_movzx(cg_sec, r_result, r_result, 4, 1); // movzx rr_result -> rr_result
        }
        x86_mov_mr(cg_sec, sz, x86_mem(REG(r_expectedaddr), 0), X86_RAX);
        free_reg(r_expected);
#endif
        free_reg(r_desired);
        free_reg(r_expectedaddr);
        free_reg(r_addr);
        return r_result;
    }
    case ND_ATOMIC_FENCE: {
        int ord = node->atomic_ord;
        if (!node->atomic_signal_fence) {
            if (ord == MEMORDER_SEQ_CST || ord == MEMORDER_ACQ_REL) {
#ifdef ARCH_ARM64
                asm_dmb(cg_sec); // dmb ish
#else
                x86_mfence(cg_sec); // mfence
#endif
            } else if (ord == MEMORDER_ACQUIRE || ord == MEMORDER_CONSUME) {
#ifdef ARCH_ARM64
                asm_dmb(cg_sec); // mfence
#endif
            } else if (ord == MEMORDER_RELEASE) {
#ifdef ARCH_ARM64
                asm_dmb(cg_sec); // dmb ishld
#endif
            }
        }
        return -1;
    }
    case ND_ATOMIC_FETCH_OP: {
        VReg r_addr = gen(node->lhs);
        VReg r_val = gen(node->rhs);
        int sz = node->ty->size;
        // The value operand keeps its own (possibly narrower) static type
        // from parsing (e.g. `__sync_fetch_and_add(ptr, -1)` where `-1` is
        // a plain `int` but `*ptr` is `intptr_t`/8 bytes) -- gen() above
        // only materializes it at ITS OWN width, so a 4-byte value in an
        // 8-byte atomic op leaves the register's upper 32 bits either
        // stale or (per x86-64's implicit-zero-extend-on-32-bit-write
        // rule) zeroed, not sign-extended. Every op below (xadd, or/xor/
        // and/nand's cmpxchg-loop combine) operates on the full `sz`-wide
        // register, so extend to match here -- once, before any op-
        // specific codegen -- following the same source-signedness rule
        // used for every other implicit narrow-to-wide conversion.
        if (node->rhs->ty && node->rhs->ty->size > 0 && node->rhs->ty->size < sz) {
            if (node->rhs->ty->is_unsigned)
                zero_extend_to(r_val, node->rhs->ty->size, sz);
            else
                sign_extend_to(r_val, node->rhs->ty->size, sz);
        }
        int op = node->atomic_fetch_op;
        bool is_store = node->atomic_is_store;
#ifdef ARCH_ARM64
        int old_dummy = alloc_reg();
        int old_slot = spill_offset(old_dummy);
        free_reg(old_dummy);
        VReg r_tmp = alloc_reg();
        int sf = (sz == 8) ? 1 : 0;
        // Move r_val to physical w9/x9 for the operation
        asm_mov_x9_from_vreg(cg_sec, r_val, sf == 1 ? 8 : 4); // mov w9/x9, r_val
        free_reg(r_val);
        int lbl = rcc_label_count++;
        cg_def_label(format(".L.atom_fop.%d", lbl));
        asm_ldxr(cg_sec, r_tmp, r_addr, sz); // ldxr[b/h] r_tmp, [r_addr]
        asm_stur_fp(cg_sec, r_tmp, old_slot); // str x(r_tmp), [x29, #-old_slot]
        switch (op) {
        case 0: { // add r_tmp, r_tmp, x9
            size_t _off = cg_sec->len;
            arm64_add_reg(cg_sec, sf, REG(r_tmp), REG(r_tmp), ARM64_X9, ARM64_LSL, 0);
            asm_record(ASM_ADD_RR, _off, 1, REG(r_tmp), ARM64_X9, -1, sz, 0, 0, NULL, 0, -1, false);
            break;
        }
        case 1: { // sub r_tmp, r_tmp, x9
            size_t _off = cg_sec->len;
            arm64_sub_reg(cg_sec, sf, REG(r_tmp), REG(r_tmp), ARM64_X9, ARM64_LSL, 0);
            asm_record(ASM_SUB_RR, _off, 1, REG(r_tmp), ARM64_X9, -1, sz, 0, 0, NULL, 0, -1, false);
            break;
        }
        case 2: { // orr r_tmp, r_tmp, x9
            size_t _off = cg_sec->len;
            arm64_orr_reg(cg_sec, sf, REG(r_tmp), REG(r_tmp), ARM64_X9, ARM64_LSL, 0);
            asm_record(ASM_OR_RR, _off, 1, REG(r_tmp), ARM64_X9, -1, sz, 0, 0, NULL, 0, -1, false);
            break;
        }
        case 3: { // eor r_tmp, r_tmp, x9
            size_t _off = cg_sec->len;
            arm64_eor_reg(cg_sec, sf, REG(r_tmp), REG(r_tmp), ARM64_X9, ARM64_LSL, 0);
            asm_record(ASM_XOR_RR, _off, 1, REG(r_tmp), ARM64_X9, -1, sz, 0, 0, NULL, 0, -1, false);
            break;
        }
        case 4: { // and r_tmp, r_tmp, x9
            size_t _off = cg_sec->len;
            arm64_and_reg(cg_sec, sf, REG(r_tmp), REG(r_tmp), ARM64_X9, ARM64_LSL, 0);
            asm_record(ASM_AND_RR, _off, 1, REG(r_tmp), ARM64_X9, -1, sz, 0, 0, NULL, 0, -1, false);
            break;
        }
        case 5: { // and r_tmp, r_tmp, x9; mvn r_tmp, r_tmp
            size_t _off = cg_sec->len;
            arm64_and_reg(cg_sec, sf, REG(r_tmp), REG(r_tmp), ARM64_X9, ARM64_LSL, 0);
            asm_record(ASM_AND_RR, _off, 1, REG(r_tmp), ARM64_X9, -1, sz, 0, 0, NULL, 0, -1, false);
            asm_not(cg_sec, r_tmp, sz);
            break;
        }
        }
        // stxr w8, r_tmp, [r_addr]
        asm_stxr_8(cg_sec, r_tmp, r_addr, sz); // stxr[b/h] w8, r_tmp, [r_addr]
        // cbnz w8, .L.atom_fop.lbl
        {
            size_t _cj = cg_sec->len;
            arm64_cbnz(cg_sec, 0, ARM64_X8, 0); // cbnz w8, .L.atom_fop.lbl
            asm_record(ASM_JCC, _cj, 1, ARM64_X8, -1, -1, 4, 0, 0, NULL, ARM64_NE, -1, false);
            asm_fixup_add(cg_sec, _cj, format(".L.atom_fop.%d", lbl), 1);
        }
        if (node->atomic_ord == MEMORDER_SEQ_CST)
            asm_dmb(cg_sec); // stxrh w8, %s, [%s]
        if (!is_store)
            asm_ldur_fp(cg_sec, r_tmp, old_slot); // ldr x(r_tmp), [x29, #-old_slot]
        free_reg(r_addr);
        if (sz < 8 && !use_unsigned(node->ty))
            sign_extend_to(r_tmp, sz, 8);
        return r_tmp;
#else
        VReg r_old = alloc_reg();
        if (op == 0 || op == 1) {
            if (op == 1)
                asm_neg(cg_sec, r_val, sz); // neg rr_val
            // Save val to stack before xadd clobbers it
            asm_mov_rbp_spill(cg_sec, r_val, sz, spill_logand); // mov r_val, -spill_logand(%rbp)
            asm_mov_reg_reg(cg_sec, r_old, r_val, sz > 4 ? 8 : 4); // mov rr_val -> rr_old
            free_reg(r_val);
            if (r_old == r_addr && (spilled_regs & (1 << r_addr))) {
                asm_mov_rbp_reg(cg_sec, r_addr, 8, spill_offset(r_addr)); // mov [rbp-8], rr_addr
            }
            asm_lock_xadd_mem(cg_sec, r_addr, r_old, sz); // lock xadd (r_addr), r_old
            free_reg(r_addr);
            if (node->atomic_ord == MEMORDER_SEQ_CST)
                asm_mfence(cg_sec); // mfence
            if (is_store) {
                // add_fetch/sub_fetch: return new value = old + val
                asm_add_spill_reg(cg_sec, r_old, sz, spill_logand); // add -spill_logand(%rbp), r_old
            }
            if (sz < 4) {
                if (!use_unsigned(node->ty))
                    sign_extend_to(r_old, sz, 4);
                else
                    zero_extend_to(r_old, sz, 4);
            }
            return r_old;
        } else {
            // Save r_val to stack before it might be spilled
            asm_mov_rbp_spill(cg_sec, r_val, sz, spill_logand); // mov r_val, -spill_logand(%rbp)
            free_reg(r_val);
            VReg r_new = alloc_reg();
            int lbl2 = rcc_label_count++;
            cg_def_label(format(".L.atom_fop.%d", lbl2));
            if (r_new == r_addr && (spilled_regs & (1 << r_addr))) {
                asm_mov_rbp_reg(cg_sec, r_addr, 8, spill_offset(r_addr)); // mov [rbp-8], rr_addr
            } else if (r_old == r_addr && (spilled_regs & (1 << r_addr))) {
                asm_mov_rbp_reg(cg_sec, r_addr, 8, spill_offset(r_addr)); // mov [rbp-8], rr_addr
            }
            asm_mov_mem_reg(cg_sec, r_old, r_addr, sz); // mov (r_addr), r_old
            // Save old value before computing new (r_old may == r_new)
            asm_mov_rbp_spill(cg_sec, r_old, sz, spill_atomic_old); // mov r_old, -spill_atomic_old(%rbp)
            asm_mov_reg_reg(cg_sec, r_new, r_old, sz > 4 ? 8 : 4); // mov r_old, r_new
            char sc = size_suffix(sz);
            (void)sc;
            switch (op) {
            case 2: asm_or_rbp_reg(cg_sec, r_new, sz, spill_logand); break;
            case 3: asm_xor_rbp_reg(cg_sec, r_new, sz, spill_logand); break;
            case 4: asm_and_rbp_reg(cg_sec, r_new, sz, spill_logand); break;
            case 5:
                asm_and_rbp_reg(cg_sec, r_new, sz, spill_logand); // and -spill_logand(%rbp), r_new
                asm_not(cg_sec, r_new, sz); // not rr_new
                break;
            }
            // lock cmpxchg (r_addr), r_new  (rax=r_old loaded from spill_atomic_old)
            asm_mov_rbp(cg_sec, X86_RAX, sz, spill_atomic_old); // mov -spill_atomic_old(%rbp), %rax
            {
                size_t _cj = asm_lock_cmpxchg_rax(cg_sec, r_addr, r_new, sz);
                asm_jcc_label(cg_sec, X86_NE); // emit JNE first so fixup can patch it
                asm_fixup_add(cg_sec, _cj, format(".L.atom_fop.%d", lbl2), 1);
            }
            if (node->atomic_ord == MEMORDER_SEQ_CST)
                asm_mfence(cg_sec); // mfence
            free_reg(r_addr);
            if (is_store) {
                free_reg(r_old);
                if (sz < 4 && !use_unsigned(node->ty))
                    sign_extend_to(r_new, sz, 4);
                else if (sz < 4)
                    zero_extend_to(r_new, sz, 4);
                return r_new;
            } else {
                free_reg(r_new);
                // r_old has the old value (loaded before cmpxchg)
                asm_mov_spill_rbp(cg_sec, r_old, sz, spill_atomic_old); // reload old from spill
                if (sz < 4 && !use_unsigned(node->ty))
                    sign_extend_to(r_old, sz, 4);
                else if (sz < 4)
                    zero_extend_to(r_old, sz, 4);
                return r_old;
            }
        }
#endif
        return -1;
    }

    default:
        break;
    }
    // Complex binary arithmetic: add/sub with complex operands
    if ((node->kind == ND_ADD || node->kind == ND_SUB || node->kind == ND_MUL || node->kind == ND_DIV) && is_complex(node->ty)) {
        int complex_sz = node->ty->size;
        int base_sz = node->ty->base->size;
        bool lhs_cx = is_complex(node->lhs->ty);
        bool rhs_cx = is_complex(node->rhs->ty);
        // Non-complex operands must be converted to the complex base type and
        // materialized as a {real, imag} pair, so never reuse their plain
        // scalar address (it is too small and holds the wrong representation).
        int addr_lhs = lhs_cx ? gen_addr(node->lhs) : -1;
        int addr_rhs = rhs_cx ? gen_addr(node->rhs) : -1;
        int need_free_lhs = 0, need_free_rhs = 0;
        if (addr_lhs < 0) {
            need_free_lhs = 1;
            addr_lhs = alloc_reg();
#ifdef ARCH_ARM64
            arm64_sub_imm(cg_sec, 1, ARM64_SP, ARM64_SP, (complex_sz + 15) & ~15, 0); // sub sp, sp, #N
            asm_mov_reg_sp(cg_sec, REG(addr_lhs)); // mov x{addr_lhs}, sp
#else
            asm_sub_rsp_imm(cg_sec, (complex_sz + 15) & ~15); // subq $N, %rsp
            x86_mov_rr(cg_sec, 8, REG(addr_lhs), X86_RSP); // movq %rsp, addr_lhs
#endif
            int v = gen(node->lhs);
            if (lhs_cx) {
                // gen() returned address — copy complex data
#ifdef ARCH_ARM64
                asm_ldr_x16_reg_uoff(cg_sec, v, 0); // ldr x16, [v]
                asm_str_x16_reg_uoff(cg_sec, addr_lhs, 0); // str x16, [addr_lhs]
                if (complex_sz > 8) {
                    asm_ldr_x16_reg_uoff(cg_sec, v, base_sz); // ldr x16, [v, #base_sz]
                    asm_str_x16_reg_uoff(cg_sec, addr_lhs, base_sz); // str x16, [addr_lhs, #base_sz]
                }
#else
                x86_mov_rm(cg_sec, 8, X86_RAX, x86_mem(REG(v), 0)); // movq (v), %rax
                x86_mov_mr(cg_sec, 8, x86_mem(REG(addr_lhs), 0), X86_RAX); // movq %rax, (addr_lhs)
                if (complex_sz > 8) {
                    x86_mov_rm(cg_sec, 8, X86_RAX, x86_mem(REG(v), base_sz)); // movq base_sz(v), %rax
                    x86_mov_mr(cg_sec, 8, x86_mem(REG(addr_lhs), base_sz), X86_RAX); // movq %rax, base_sz(addr_lhs)
                }
#endif
            } else {
                emit_scalar_to_complex(v, node->lhs->ty, node->ty->base, addr_lhs);
            }
            free_reg(v);
        }
        if (addr_rhs < 0) {
            need_free_rhs = 1;
            addr_rhs = alloc_reg();
#ifdef ARCH_ARM64
            arm64_sub_imm(cg_sec, 1, ARM64_SP, ARM64_SP, (complex_sz + 15) & ~15, 0); // sub sp, sp, #N
            asm_mov_reg_sp(cg_sec, REG(addr_rhs)); // mov x{addr_rhs}, sp
#else
            asm_sub_rsp_imm(cg_sec, (complex_sz + 15) & ~15); // subq $N, %rsp
            x86_mov_rr(cg_sec, 8, REG(addr_rhs), X86_RSP); // movq %rsp, addr_rhs
#endif
            int v = gen(node->rhs);
            if (rhs_cx) {
#ifdef ARCH_ARM64
                asm_ldr_x16_reg_uoff(cg_sec, v, 0); // ldr x16, [v]
                asm_str_x16_reg_uoff(cg_sec, addr_rhs, 0); // str x16, [addr_rhs]
                if (complex_sz > 8) {
                    asm_ldr_x16_reg_uoff(cg_sec, v, base_sz); // ldr x16, [v, #base_sz]
                    asm_str_x16_reg_uoff(cg_sec, addr_rhs, base_sz); // str x16, [addr_rhs, #base_sz]
                }
#else
                x86_mov_rm(cg_sec, 8, X86_RAX, x86_mem(REG(v), 0)); // movq (v), %rax
                x86_mov_mr(cg_sec, 8, x86_mem(REG(addr_rhs), 0), X86_RAX); // movq %rax, (addr_rhs)
                if (complex_sz > 8) {
                    x86_mov_rm(cg_sec, 8, X86_RAX, x86_mem(REG(v), base_sz)); // movq base_sz(v), %rax
                    x86_mov_mr(cg_sec, 8, x86_mem(REG(addr_rhs), base_sz), X86_RAX); // movq %rax, base_sz(addr_rhs)
                }
#endif
            } else {
                emit_scalar_to_complex(v, node->rhs->ty, node->ty->base, addr_rhs);
            }
            free_reg(v);
        }
        // Result allocated from fn_struct_ret_total (stable across calls/returns)
        int alloc = (complex_sz + 7) & ~7;
        fn_struct_ret_off += alloc;
        if (fn_struct_ret_off > fn_struct_ret_total)
            fn_struct_ret_total = fn_struct_ret_off;
        int result_off = current_fn_stack_size + fn_struct_ret_off;
        int result = alloc_reg();
#ifdef ARCH_ARM64
        asm_sub_reg_fp_imm(cg_sec, result, result_off); // sub result, x29, #result_off
#else
        asm_lea_rbp_reg(cg_sec, result, 8, result_off);
#endif
        if (is_flonum(node->ty->base)) {
#ifdef ARCH_ARM64
            int ftype = base_sz == 4 ? 0 : 1;
            int sf = base_sz == 4 ? 0 : 1;
            asm_ldr_fp(cg_sec, 0, addr_lhs, base_sz);
            asm_ldr_fp_off(cg_sec, 1, addr_lhs, base_sz, (uint32_t)base_sz);
            asm_ldr_fp(cg_sec, 2, addr_rhs, base_sz);
            asm_ldr_fp_off(cg_sec, 3, addr_rhs, base_sz, (uint32_t)base_sz);
            if (node->kind == ND_ADD || node->kind == ND_SUB) {
                if (node->kind == ND_ADD) {
                    arm64_fadd(cg_sec, ftype, 0, 0, 2);
                    arm64_fadd(cg_sec, ftype, 1, 1, 3);
                } else {
                    arm64_fsub(cg_sec, ftype, 0, 0, 2);
                    arm64_fsub(cg_sec, ftype, 1, 1, 3);
                }
            } else if (node->kind == ND_MUL) {
                // (a+bi)*(c+di) = (ac-bd) + (ad+bc)i
                int t = alloc_reg();
                arm64_fmov_f2i(cg_sec, sf, REG(t), 0);
                arm64_fmov_i2f(cg_sec, sf, 4, REG(t));
                free_reg(t);
                t = alloc_reg();
                arm64_fmov_f2i(cg_sec, sf, REG(t), 1);
                arm64_fmov_i2f(cg_sec, sf, 5, REG(t));
                free_reg(t);
                arm64_fmul(cg_sec, ftype, 4, 4, 2);
                arm64_fmul(cg_sec, ftype, 5, 5, 3);
                arm64_fsub(cg_sec, ftype, 4, 4, 5);
                arm64_fmul(cg_sec, ftype, 0, 0, 3);
                arm64_fmul(cg_sec, ftype, 1, 1, 2);
                arm64_fadd(cg_sec, ftype, 1, 0, 1);
                t = alloc_reg();
                arm64_fmov_f2i(cg_sec, sf, REG(t), 4);
                arm64_fmov_i2f(cg_sec, sf, 0, REG(t));
                free_reg(t);
            } else {
                // Complex float division via libgcc: d0/d1 = lhs, d2/d3 = rhs.
                int result_spill = spill_offset(result);
                asm_stur_fp(cg_sec, result, result_spill); // str x{result}, [x29, #-spill_offset]
                emit_direct_call(base_sz == 4 ? "__divsc3" : "__divdc3", false); // bl __divsc3/__divdc3
                asm_ldur_fp(cg_sec, result, result_spill); // ldr x{result}, [x29, #-spill_offset]
                asm_str_fp(cg_sec, 0, result, base_sz);
                arm64_str_fp(cg_sec, base_sz == 4 ? 2 : 3, 1, REG(result), (uint32_t)base_sz);
                goto cx_arith_done;
            }
            if (node->kind != ND_DIV) {
                asm_str_fp(cg_sec, 0, result, base_sz);
                arm64_str_fp(cg_sec, base_sz == 4 ? 2 : 3, 1, REG(result), (uint32_t)base_sz);
            }
#else
            int sfx = base_sz; // 4 -> ss, 8 -> sd
            X86Mem m_lhs0 = x86_mem(REG(addr_lhs), 0);
            X86Mem m_lhs1 = x86_mem(REG(addr_lhs), base_sz);
            X86Mem m_rhs0 = x86_mem(REG(addr_rhs), 0);
            X86Mem m_rhs1 = x86_mem(REG(addr_rhs), base_sz);
            X86Mem m_res0 = x86_mem(REG(result), 0);
            X86Mem m_res1 = x86_mem(REG(result), base_sz);
            asm_mov_fp_rm(cg_sec, sfx, X86_XMM0, m_lhs0); // mov[s|s]s/d (addr_lhs), %xmm0
            asm_mov_fp_rm(cg_sec, sfx, X86_XMM1, m_lhs1); // mov[s|s]s/d base_sz(addr_lhs), %xmm1
            asm_mov_fp_rm(cg_sec, sfx, X86_XMM2, m_rhs0); // mov[s|s]s/d (addr_rhs), %xmm2
            asm_mov_fp_rm(cg_sec, sfx, X86_XMM3, m_rhs1); // mov[s|s]s/d base_sz(addr_rhs), %xmm3
            if (node->kind == ND_ADD || node->kind == ND_SUB) {
                if (node->kind == ND_ADD) {
                    asm_add_fp(cg_sec, sfx, X86_XMM0, X86_XMM2); // add[s|s]s/d %xmm2, %xmm0
                    asm_add_fp(cg_sec, sfx, X86_XMM1, X86_XMM3); // add[s|s]s/d %xmm3, %xmm1
                } else {
                    asm_sub_fp(cg_sec, sfx, X86_XMM0, X86_XMM2); // sub[s|s]s/d %xmm2, %xmm0
                    asm_sub_fp(cg_sec, sfx, X86_XMM1, X86_XMM3); // sub[s|s]s/d %xmm3, %xmm1
                }
            } else if (node->kind == ND_MUL) {
                // (a+bi)*(c+di) = (ac-bd) + (ad+bc)i
                // xmm0=a.r, xmm1=a.i, xmm2=b.r, xmm3=b.i
                asm_movapd_rr(cg_sec, X86_XMM4, X86_XMM0); // movapd %xmm0, %xmm4
                asm_movapd_rr(cg_sec, X86_XMM5, X86_XMM1); // movapd %xmm1, %xmm5
                asm_mul_fp(cg_sec, sfx, X86_XMM4, X86_XMM2); // mul[s|s]s/d %xmm2, %xmm4  -- a.r * b.r
                asm_mul_fp(cg_sec, sfx, X86_XMM5, X86_XMM3); // mul[s|s]s/d %xmm3, %xmm5  -- a.i * b.i
                asm_sub_fp(cg_sec, sfx, X86_XMM4, X86_XMM5); // sub[s|s]s/d %xmm5, %xmm4  -- ac - bd = real
                asm_mul_fp(cg_sec, sfx, X86_XMM0, X86_XMM3); // mul[s|s]s/d %xmm3, %xmm0  -- a.r * b.i
                asm_mul_fp(cg_sec, sfx, X86_XMM1, X86_XMM2); // mul[s|s]s/d %xmm2, %xmm1  -- a.i * b.r
                asm_add_fp(cg_sec, sfx, X86_XMM1, X86_XMM0); // add[s|s]s/d %xmm0, %xmm1  -- ad + bc = imag
                asm_movapd_rr(cg_sec, X86_XMM0, X86_XMM4); // movapd %xmm4, %xmm0
            } else {
                // (a+bi)/(c+di) = ((ac+bd)/(cc+dd)) + ((bc-ad)/(cc+dd))i
                // xmm0=a.r, xmm1=a.i, xmm2=b.r, xmm3=b.i
                asm_movapd_rr(cg_sec, X86_XMM6, X86_XMM2); // movapd %xmm2, %xmm6  -- save b.r
                asm_movapd_rr(cg_sec, X86_XMM7, X86_XMM3); // movapd %xmm3, %xmm7  -- save b.i
                asm_movapd_rr(cg_sec, X86_XMM4, X86_XMM0); // movapd %xmm0, %xmm4  -- save a.r
                asm_movapd_rr(cg_sec, X86_XMM5, X86_XMM1); // movapd %xmm1, %xmm5  -- save a.i
                // denom = b.r² + b.i²
                asm_mul_fp(cg_sec, sfx, X86_XMM2, X86_XMM2); // mul[s|s]s/d %xmm2, %xmm2  -- b.r²
                asm_mul_fp(cg_sec, sfx, X86_XMM3, X86_XMM3); // mul[s|s]s/d %xmm3, %xmm3  -- b.i²
                asm_add_fp(cg_sec, sfx, X86_XMM2, X86_XMM3); // add[s|s]s/d %xmm3, %xmm2  -- denom in xmm2
                // real = (a.r*b.r + a.i*b.i) / denom
                asm_mul_fp(cg_sec, sfx, X86_XMM4, X86_XMM6); // mul[s|s]s/d %xmm6, %xmm4  -- a.r * b.r
                asm_mul_fp(cg_sec, sfx, X86_XMM5, X86_XMM7); // mul[s|s]s/d %xmm7, %xmm5  -- a.i * b.i
                asm_add_fp(cg_sec, sfx, X86_XMM4, X86_XMM5); // add[s|s]s/d %xmm5, %xmm4  -- ac+bd
                asm_div_fp(cg_sec, sfx, X86_XMM4, X86_XMM2); // div[s|s]s/d %xmm2, %xmm4  -- real = (ac+bd)/denom
                // imag = (a.i*b.r - a.r*b.i) / denom
                asm_mul_fp(cg_sec, sfx, X86_XMM1, X86_XMM6); // mul[s|s]s/d %xmm6, %xmm1  -- a.i * b.r
                asm_mul_fp(cg_sec, sfx, X86_XMM0, X86_XMM7); // mul[s|s]s/d %xmm7, %xmm0  -- a.r * b.i
                asm_sub_fp(cg_sec, sfx, X86_XMM1, X86_XMM0); // sub[s|s]s/d %xmm0, %xmm1  -- bc-ad
                asm_div_fp(cg_sec, sfx, X86_XMM1, X86_XMM2); // div[s|s]s/d %xmm2, %xmm1  -- imag = (bc-ad)/denom
                asm_movapd_rr(cg_sec, X86_XMM0, X86_XMM4); // movapd %xmm4, %xmm0
            }
            asm_mov_fp_mr(cg_sec, sfx, m_res0, X86_XMM0); // mov[s|s]s/d %xmm0, (result)
            asm_mov_fp_mr(cg_sec, sfx, m_res1, X86_XMM1); // mov[s|s]s/d %xmm1, base_sz(result)
#endif
        } else {
#ifdef ARCH_ARM64
            arm64_ldr_uoff(cg_sec, 3, ARM64_X16, REG(addr_lhs), 0);
            if (complex_sz <= 8) {
                arm64_ldr_uoff(cg_sec, 3, ARM64_X17, REG(addr_rhs), 0);
                switch (node->kind) {
                case ND_ADD:
                    arm64_add_reg(cg_sec, 0, ARM64_X16, ARM64_X16, ARM64_X17, ARM64_LSL, 0);
                    arm64_ldrsw_uoff(cg_sec, ARM64_X17, REG(addr_lhs), (uint32_t)(base_sz / 4));
                    arm64_ldrsw_uoff(cg_sec, ARM64_X18, REG(addr_rhs), (uint32_t)(base_sz / 4));
                    arm64_add_reg(cg_sec, 0, ARM64_X18, ARM64_X17, ARM64_X18, ARM64_LSL, 0);
                    arm64_ubfx(cg_sec, 1, ARM64_X16, ARM64_X16, 0, 32);
                    arm64_orr_reg(cg_sec, 1, ARM64_X16, ARM64_X16, ARM64_X18, ARM64_LSL, 32);
                    break;
                case ND_SUB:
                    arm64_sub_reg(cg_sec, 0, ARM64_X16, ARM64_X16, ARM64_X17, ARM64_LSL, 0);
                    arm64_ldrsw_uoff(cg_sec, ARM64_X17, REG(addr_lhs), (uint32_t)(base_sz / 4));
                    arm64_ldrsw_uoff(cg_sec, ARM64_X18, REG(addr_rhs), (uint32_t)(base_sz / 4));
                    arm64_sub_reg(cg_sec, 0, ARM64_X18, ARM64_X17, ARM64_X18, ARM64_LSL, 0);
                    arm64_ubfx(cg_sec, 1, ARM64_X16, ARM64_X16, 0, 32);
                    arm64_orr_reg(cg_sec, 1, ARM64_X16, ARM64_X16, ARM64_X18, ARM64_LSL, 32);
                    break;
                case ND_MUL: {
                    // (a+bi)(c+di) = (ac-bd) + (bc+ad)i — integer arithmetic
                    arm64_ldr_uoff(cg_sec, 2, ARM64_X16, REG(addr_lhs), 0);
                    arm64_ldr_uoff(cg_sec, 2, ARM64_X18, REG(addr_rhs), 0);
                    arm64_mul(cg_sec, 0, ARM64_X16, ARM64_X16, ARM64_X18);
                    arm64_ldr_uoff(cg_sec, 2, ARM64_X17, REG(addr_lhs), (uint32_t)(base_sz / 4));
                    arm64_ldr_uoff(cg_sec, 2, ARM64_X18, REG(addr_rhs), (uint32_t)(base_sz / 4));
                    arm64_mul(cg_sec, 0, ARM64_X17, ARM64_X17, ARM64_X18);
                    arm64_sub_reg(cg_sec, 0, ARM64_X16, ARM64_X16, ARM64_X17, ARM64_LSL, 0);
                    arm64_ldr_uoff(cg_sec, 2, ARM64_X17, REG(addr_lhs), (uint32_t)(base_sz / 4));
                    arm64_ldr_uoff(cg_sec, 2, ARM64_X18, REG(addr_rhs), 0);
                    arm64_mul(cg_sec, 0, ARM64_X17, ARM64_X17, ARM64_X18);
                    arm64_ldr_uoff(cg_sec, 2, ARM64_X18, REG(addr_rhs), (uint32_t)(base_sz / 4));
                    arm64_ldr_uoff(cg_sec, 2, ARM64_X9, REG(addr_lhs), 0);
                    arm64_mul(cg_sec, 0, ARM64_X9, ARM64_X9, ARM64_X18);
                    arm64_add_reg(cg_sec, 0, ARM64_X17, ARM64_X17, ARM64_X9, ARM64_LSL, 0);
                    arm64_str_uoff(cg_sec, 2, ARM64_X16, REG(result), 0);
                    arm64_str_uoff(cg_sec, 2, ARM64_X17, REG(result), (uint32_t)(base_sz / 4));
                    goto cx_arith_done;
                }
                case ND_DIV: {
                    // (a+bi)/(c+di) = ((ac+bd)/(c²+d²)) + ((bc-ad)/(c²+d²))i
                    const char *dv = node->ty->base->is_unsigned ? "udiv" : "sdiv";
                    (void)dv;
                    arm64_ldr_uoff(cg_sec, 2, ARM64_X16, REG(addr_rhs), 0);
                    arm64_ldr_uoff(cg_sec, 2, ARM64_X17, REG(addr_rhs), (uint32_t)(base_sz / 4));
                    arm64_mul(cg_sec, 0, ARM64_X16, ARM64_X16, ARM64_X16);
                    arm64_mul(cg_sec, 0, ARM64_X17, ARM64_X17, ARM64_X17);
                    arm64_add_reg(cg_sec, 0, ARM64_X9, ARM64_X16, ARM64_X17, ARM64_LSL, 0);
                    arm64_ldr_uoff(cg_sec, 2, ARM64_X16, REG(addr_lhs), 0);
                    arm64_ldr_uoff(cg_sec, 2, ARM64_X17, REG(addr_rhs), 0);
                    arm64_mul(cg_sec, 0, ARM64_X16, ARM64_X16, ARM64_X17);
                    arm64_ldr_uoff(cg_sec, 2, ARM64_X17, REG(addr_lhs), (uint32_t)(base_sz / 4));
                    arm64_ldr_uoff(cg_sec, 2, ARM64_X18, REG(addr_rhs), (uint32_t)(base_sz / 4));
                    arm64_mul(cg_sec, 0, ARM64_X17, ARM64_X17, ARM64_X18);
                    arm64_add_reg(cg_sec, 0, ARM64_X16, ARM64_X16, ARM64_X17, ARM64_LSL, 0);
                    if (node->ty->base->is_unsigned)
                        arm64_udiv(cg_sec, 0, ARM64_X16, ARM64_X16, ARM64_X9);
                    else
                        arm64_sdiv(cg_sec, 0, ARM64_X16, ARM64_X16, ARM64_X9);
                    arm64_str_uoff(cg_sec, 2, ARM64_X16, REG(result), 0);
                    arm64_ldr_uoff(cg_sec, 2, ARM64_X16, REG(addr_lhs), (uint32_t)(base_sz / 4));
                    arm64_ldr_uoff(cg_sec, 2, ARM64_X17, REG(addr_rhs), 0);
                    arm64_mul(cg_sec, 0, ARM64_X16, ARM64_X16, ARM64_X17);
                    arm64_ldr_uoff(cg_sec, 2, ARM64_X17, REG(addr_lhs), 0);
                    arm64_ldr_uoff(cg_sec, 2, ARM64_X18, REG(addr_rhs), (uint32_t)(base_sz / 4));
                    arm64_mul(cg_sec, 0, ARM64_X17, ARM64_X17, ARM64_X18);
                    arm64_sub_reg(cg_sec, 0, ARM64_X16, ARM64_X16, ARM64_X17, ARM64_LSL, 0);
                    if (node->ty->base->is_unsigned)
                        arm64_udiv(cg_sec, 0, ARM64_X16, ARM64_X16, ARM64_X9);
                    else
                        arm64_sdiv(cg_sec, 0, ARM64_X16, ARM64_X16, ARM64_X9);
                    arm64_str_uoff(cg_sec, 2, ARM64_X16, REG(result), (uint32_t)(base_sz / 4));
                    goto cx_arith_done;
                }
                default:
                    __builtin_unreachable();
                }
                arm64_str_uoff(cg_sec, 3, ARM64_X16, REG(result), 0);
            } else {
                arm64_ldr_uoff(cg_sec, 3, ARM64_X17, REG(addr_rhs), 0);
                switch (node->kind) {
                case ND_ADD:
                    arm64_adds_reg(cg_sec, 1, ARM64_X16, ARM64_X16, ARM64_X17, ARM64_LSL, 0);
                    arm64_ldr_uoff(cg_sec, 3, ARM64_X17, REG(addr_lhs), (uint32_t)(base_sz / 8));
                    arm64_ldr_uoff(cg_sec, 3, ARM64_X18, REG(addr_rhs), (uint32_t)(base_sz / 8));
                    arm64_adc(cg_sec, 1, ARM64_X17, ARM64_X17, ARM64_X18);
                    arm64_str_uoff(cg_sec, 3, ARM64_X16, REG(result), 0);
                    arm64_str_uoff(cg_sec, 3, ARM64_X17, REG(result), (uint32_t)(base_sz / 8));
                    goto cx_arith_done;
                case ND_SUB:
                    arm64_subs_reg(cg_sec, 1, ARM64_X16, ARM64_X16, ARM64_X17, ARM64_LSL, 0);
                    arm64_ldr_uoff(cg_sec, 3, ARM64_X17, REG(addr_lhs), (uint32_t)(base_sz / 8));
                    arm64_ldr_uoff(cg_sec, 3, ARM64_X18, REG(addr_rhs), (uint32_t)(base_sz / 8));
                    arm64_sbc(cg_sec, 1, ARM64_X17, ARM64_X17, ARM64_X18);
                    arm64_str_uoff(cg_sec, 3, ARM64_X16, REG(result), 0);
                    arm64_str_uoff(cg_sec, 3, ARM64_X17, REG(result), (uint32_t)(base_sz / 8));
                    goto cx_arith_done;
                case ND_MUL: {
                    // (a+bi)(c+di) = (ac-bd) + (bc+ad)i — 64-bit integer arithmetic
                    arm64_ldr_uoff(cg_sec, 3, ARM64_X16, REG(addr_lhs), 0);
                    arm64_ldr_uoff(cg_sec, 3, ARM64_X18, REG(addr_rhs), 0);
                    arm64_mul(cg_sec, 1, ARM64_X16, ARM64_X16, ARM64_X18);
                    arm64_ldr_uoff(cg_sec, 3, ARM64_X17, REG(addr_lhs), (uint32_t)(base_sz / 8));
                    arm64_ldr_uoff(cg_sec, 3, ARM64_X18, REG(addr_rhs), (uint32_t)(base_sz / 8));
                    arm64_mul(cg_sec, 1, ARM64_X17, ARM64_X17, ARM64_X18);
                    arm64_sub_reg(cg_sec, 1, ARM64_X16, ARM64_X16, ARM64_X17, ARM64_LSL, 0);
                    arm64_ldr_uoff(cg_sec, 3, ARM64_X17, REG(addr_lhs), (uint32_t)(base_sz / 8));
                    arm64_ldr_uoff(cg_sec, 3, ARM64_X18, REG(addr_rhs), 0);
                    arm64_mul(cg_sec, 1, ARM64_X17, ARM64_X17, ARM64_X18);
                    arm64_ldr_uoff(cg_sec, 3, ARM64_X18, REG(addr_rhs), (uint32_t)(base_sz / 8));
                    arm64_ldr_uoff(cg_sec, 3, ARM64_X9, REG(addr_lhs), 0);
                    arm64_mul(cg_sec, 1, ARM64_X9, ARM64_X9, ARM64_X18);
                    arm64_add_reg(cg_sec, 1, ARM64_X17, ARM64_X17, ARM64_X9, ARM64_LSL, 0);
                    arm64_str_uoff(cg_sec, 3, ARM64_X16, REG(result), 0);
                    arm64_str_uoff(cg_sec, 3, ARM64_X17, REG(result), (uint32_t)(base_sz / 8));
                    goto cx_arith_done;
                }
                case ND_DIV: {
                    const char *dv = node->ty->base->is_unsigned ? "udiv" : "sdiv";
                    (void)dv;
                    arm64_ldr_uoff(cg_sec, 3, ARM64_X16, REG(addr_rhs), 0);
                    arm64_ldr_uoff(cg_sec, 3, ARM64_X17, REG(addr_rhs), (uint32_t)(base_sz / 8));
                    arm64_mul(cg_sec, 1, ARM64_X16, ARM64_X16, ARM64_X16);
                    arm64_mul(cg_sec, 1, ARM64_X17, ARM64_X17, ARM64_X17);
                    arm64_add_reg(cg_sec, 1, ARM64_X9, ARM64_X16, ARM64_X17, ARM64_LSL, 0);
                    arm64_ldr_uoff(cg_sec, 3, ARM64_X16, REG(addr_lhs), 0);
                    arm64_ldr_uoff(cg_sec, 3, ARM64_X17, REG(addr_rhs), 0);
                    arm64_mul(cg_sec, 1, ARM64_X16, ARM64_X16, ARM64_X17);
                    arm64_ldr_uoff(cg_sec, 3, ARM64_X17, REG(addr_lhs), (uint32_t)(base_sz / 8));
                    arm64_ldr_uoff(cg_sec, 3, ARM64_X18, REG(addr_rhs), (uint32_t)(base_sz / 8));
                    arm64_mul(cg_sec, 1, ARM64_X17, ARM64_X17, ARM64_X18);
                    arm64_add_reg(cg_sec, 1, ARM64_X16, ARM64_X16, ARM64_X17, ARM64_LSL, 0);
                    if (node->ty->base->is_unsigned)
                        arm64_udiv(cg_sec, 1, ARM64_X16, ARM64_X16, ARM64_X9);
                    else
                        arm64_sdiv(cg_sec, 1, ARM64_X16, ARM64_X16, ARM64_X9);
                    arm64_str_uoff(cg_sec, 3, ARM64_X16, REG(result), 0);
                    arm64_ldr_uoff(cg_sec, 3, ARM64_X16, REG(addr_lhs), (uint32_t)(base_sz / 8));
                    arm64_ldr_uoff(cg_sec, 3, ARM64_X17, REG(addr_rhs), 0);
                    arm64_mul(cg_sec, 1, ARM64_X16, ARM64_X16, ARM64_X17);
                    arm64_ldr_uoff(cg_sec, 3, ARM64_X17, REG(addr_lhs), 0);
                    arm64_ldr_uoff(cg_sec, 3, ARM64_X18, REG(addr_rhs), (uint32_t)(base_sz / 8));
                    arm64_mul(cg_sec, 1, ARM64_X17, ARM64_X17, ARM64_X18);
                    arm64_sub_reg(cg_sec, 1, ARM64_X16, ARM64_X16, ARM64_X17, ARM64_LSL, 0);
                    if (node->ty->base->is_unsigned)
                        arm64_udiv(cg_sec, 1, ARM64_X16, ARM64_X16, ARM64_X9);
                    else
                        arm64_sdiv(cg_sec, 1, ARM64_X16, ARM64_X16, ARM64_X9);
                    arm64_str_uoff(cg_sec, 3, ARM64_X16, REG(result), (uint32_t)(base_sz / 8));
                    goto cx_arith_done;
                }
                default:
                    __builtin_unreachable();
                }
            }
#else
            // Operate on each base_sz-byte component (real/imag) in a
            // W-byte register (W=4 for char/short/int bases, 8 for long
            // bases), sign- or zero-extending narrower components on load
            // and truncating on store. Scratch registers (rax, rcx, rdx,
            // r8, r9, rdi) are all outside the alloc_reg pool ({%r10, %r11,
            // %rbx, %r12-%r15, %rsi}), so they can never collide with
            // addr_lhs/addr_rhs/result or any live register from an
            // enclosing expression.
            bool base_unsigned = node->ty->base->is_unsigned;
            int W = base_sz <= 4 ? 4 : 8;
            const char *rax_w = W == 4 ? "%eax" : "%rax";
            const char *rdx_w = W == 4 ? "%edx" : "%rdx";
            const char *rcx_w = W == 4 ? "%ecx" : "%rcx";
            const char *r8_w = W == 4 ? "%r8d" : "%r8";
            const char *r9_w = W == 4 ? "%r9d" : "%r9";
            const char *rdi_w = W == 4 ? "%edi" : "%rdi";
#define CX_LOAD(memreg, off, dstw) do { \
    if (base_sz == W) \
        printf("  mov %d(%s), %s\n", off, memreg, dstw); \
    else if (base_sz == 1) \
        printf("  %s %d(%s), %s\n", base_unsigned ? "movzbl" : "movsbl", off, memreg, dstw); \
    else \
        printf("  %s %d(%s), %s\n", base_unsigned ? "movzwl" : "movswl", off, memreg, dstw); \
} while (0)
            CX_LOAD(reg64[addr_lhs], 0, rax_w); // a = lhs real
            CX_LOAD(reg64[addr_lhs], base_sz, rdx_w); // b = lhs imag
            CX_LOAD(reg64[addr_rhs], 0, rcx_w); // c = rhs real
            CX_LOAD(reg64[addr_rhs], base_sz, r8_w); // d = rhs imag
            const char *real_reg, *imag_reg;
            switch (node->kind) {
            case ND_ADD:
                x86_add_rr(cg_sec, W, X86_RAX, X86_RCX); // add %ecx/%rcx, %eax/%rax (real = a+c)
                x86_add_rr(cg_sec, W, X86_RDX, X86_R8); // add %r8d/%r8, %edx/%rdx (imag = b+d)
                real_reg = rax_w;
                imag_reg = rdx_w;
                break;
            case ND_SUB:
                x86_sub_rr(cg_sec, W, X86_RAX, X86_RCX); // sub %ecx/%rcx, %eax/%rax (real = a-c)
                x86_sub_rr(cg_sec, W, X86_RDX, X86_R8); // sub %r8d/%r8, %edx/%rdx (imag = b-d)
                real_reg = rax_w;
                imag_reg = rdx_w;
                break;
            case ND_MUL:
                // (a+bi)*(c+di) = (ac-bd) + (ad+bc)i
                printf("  mov %s, %s\n", rax_w, r9_w);
                printf("  mov %s, %s\n", rax_w, rdi_w);
                printf("  imul %s, %s\n", rcx_w, r9_w); // r9 = a*c
                printf("  imul %s, %s\n", r8_w, rdi_w); // rdi = b*d
                printf("  sub %s, %s\n", rdi_w, r9_w); // r9 = ac-bd (real)
                printf("  imul %s, %s\n", r8_w, rax_w); // rax = a*d
                printf("  imul %s, %s\n", rcx_w, rdx_w); // rdx = b*c
                printf("  add %s, %s\n", rdx_w, rax_w); // rax = ad+bc (imag)
                printf("  mov %s, %s\n", r9_w, rdx_w); // rdx = real result
                real_reg = rdx_w;
                imag_reg = rax_w;
                break;
            case ND_DIV:
                // (a+bi)/(c+di) = ((ac+bd)/(cc+dd)) + ((bc-ad)/(cc+dd))i
                printf("  mov %s, %s\n", rax_w, r9_w);
                printf("  mov %s, %s\n", rdx_w, rdi_w);
                printf("  imul %s, %s\n", rcx_w, r9_w); // r9 = a*c
                printf("  imul %s, %s\n", r8_w, rdi_w); // rdi = b*d
                printf("  add %s, %s\n", rdi_w, r9_w); // r9 = ac+bd (num_real)
                printf("  mov %s, %s\n", rdx_w, rdi_w); // rdi = b
                printf("  imul %s, %s\n", rcx_w, rdi_w); // rdi = b*c
                printf("  imul %s, %s\n", r8_w, rax_w); // rax = a*d
                printf("  sub %s, %s\n", rax_w, rdi_w); // rdi = bc-ad (num_imag)
                printf("  mov %s, %s\n", rcx_w, rax_w); // rax = c
                printf("  imul %s, %s\n", rcx_w, rax_w); // rax = c*c
                printf("  mov %s, %s\n", r8_w, rdx_w); // rdx = d
                printf("  imul %s, %s\n", r8_w, rdx_w); // rdx = d*d
                printf("  add %s, %s\n", rdx_w, rax_w); // rax = cc+dd (denom)
                printf("  mov %s, %s\n", rax_w, rcx_w); // rcx = denom
                printf("  mov %s, %s\n", r9_w, rax_w); // rax = num_real
                printf("  %s\n", base_unsigned ? "xor %edx, %edx" : (W == 8 ? "cqo" : "cdq"));
                printf("  %s %s\n", base_unsigned ? "div" : "idiv", rcx_w); // rax = real
                printf("  push %%rax\n"); // save real result
                printf("  mov %s, %s\n", rdi_w, rax_w); // rax = num_imag
                printf("  %s\n", base_unsigned ? "xor %edx, %edx" : (W == 8 ? "cqo" : "cdq"));
                printf("  %s %s\n", base_unsigned ? "div" : "idiv", rcx_w); // rax = imag
                printf("  pop %%rdx\n"); // rdx = real result
                real_reg = rdx_w;
                imag_reg = rax_w;
                break;
            default:
                __builtin_unreachable();
            }
            if (base_sz == W) {
                printf("  mov %s, (%s)\n", real_reg, reg64[result]);
                printf("  mov %s, %d(%s)\n", imag_reg, base_sz, reg64[result]);
            } else {
                printf("  mov %s, (%s)\n", x86_subreg(real_reg, base_sz), reg64[result]);
                printf("  mov %s, %d(%s)\n", x86_subreg(imag_reg, base_sz), base_sz, reg64[result]);
            }
#undef CX_LOAD
#endif
        }
#ifdef ARCH_ARM64
    cx_arith_done:
#endif
        free_reg(addr_lhs);
        free_reg(addr_rhs);
#ifdef ARCH_ARM64
        if (need_free_lhs) arm64_add_imm(cg_sec, 1, ARM64_SP, ARM64_SP, (complex_sz + 15) & ~15, 0); // add sp, sp, #complex_sz
        if (need_free_rhs) arm64_add_imm(cg_sec, 1, ARM64_SP, ARM64_SP, (complex_sz + 15) & ~15, 0); // add sp, sp, #complex_sz
#else
        if (need_free_lhs) asm_add_rsp_imm(cg_sec, (complex_sz + 15) & ~15); // addq $N, %rsp
        if (need_free_rhs) asm_add_rsp_imm(cg_sec, (complex_sz + 15) & ~15); // addq $N, %rsp
#endif
        return result;
    }

    if (node->kind == ND_REAL || node->kind == ND_IMAG) {
        // gen_addr adds the offset for ND_IMAG, so the returned address
        // already points to the real or imag part of the complex value.
        // This must be handled before the unconditional gen(node->lhs)
        // below, which would otherwise evaluate node->lhs (the whole
        // complex value) into a register that is never used or freed.
        int r = gen_addr(node);
        if (r < 0) {
            // Complex sub-expression is not an lvalue (e.g., __real__(a*b)).
            // Evaluate it; gen() for complex arithmetic returns the result address.
            r = gen(node->lhs);
            // For ND_IMAG, add the base type size offset to get the imag part
            if (node->kind == ND_IMAG && r >= 0) {
                int offset = node->lhs->ty->base->size;
#ifdef ARCH_ARM64
                if (offset <= 4095)
                    asm_add_imm(cg_sec, r, 8, offset); // add r, r, #offset
                else {
                    VReg ti = alloc_reg();
                    emit_mov_imm64(REG(ti), (uint64_t)offset); // mov ti, #offset
                    asm_add_reg_reg(cg_sec, r, ti, 8); // add r, r, ti
                    free_reg(ti);
                }
#else
                asm_add_imm(cg_sec, r, 8, offset); // add $offset, r
#endif
            }
        }
        if (r >= 0) {
            if (is_flonum(node->ty)) {
#ifdef ARCH_ARM64
                if (node->ty->size == 4) {
                    asm_ldr_fp(cg_sec, ARM64_S0, r, 4); // ldr s0, [x{r}]
                    asm_fcvt(cg_sec, 1, 0, ARM64_D0, ARM64_S0); // fcvt d0, s0
                } else {
                    asm_ldr_fp(cg_sec, ARM64_D0, r, 8); // ldr d0, [x{r}]
                }
                asm_fmov_f2i(cg_sec, r, 0, 1); // fmov x{r}, d0
#else
                if (node->ty->size == 4) {
                    x86_movss_rm(cg_sec, X86_XMM0, x86_mem(REG(r), 0)); // movss (r), %xmm0
                    x86_cvtss2sd(cg_sec, X86_XMM0, X86_XMM0); // cvtss2sd %xmm0, %xmm0
                } else {
                    x86_movsd_rm(cg_sec, X86_XMM0, x86_mem(REG(r), 0)); // movsd (r), %xmm0
                }
                asm_movq_xmm_r(cg_sec, r, X86_XMM0); // movq %xmm0, r
#endif
            } else {
                emit_load(node->ty, r, r, 0);
            }
            return r;
        }
    }

    // Complex comparisons: compare both real and imag parts
    if ((node->kind == ND_EQ || node->kind == ND_NE) &&
        node->lhs->ty && is_complex(node->lhs->ty)) {
        int addr_lhs = gen_addr(node->lhs);
        int addr_rhs = gen_addr(node->rhs);
        int need_free_lhs = 0, need_free_rhs = 0;
        int complex_sz = node->lhs->ty->size;
        int base_sz = node->lhs->ty->base->size;
        int sz = complex_sz;
        if (addr_lhs < 0) {
            addr_lhs = alloc_reg();
#ifdef ARCH_ARM64
            arm64_sub_imm(cg_sec, 1, ARM64_SP, ARM64_SP, (complex_sz + 15) & ~15, 0); // sub sp, sp, #complex_sz
            asm_mov_reg_sp(cg_sec, REG(addr_lhs)); // mov addr_lhs, sp
#else
            asm_sub_rsp_imm(cg_sec, (complex_sz + 15) & ~15); // subq $N, %rsp
            x86_mov_rr(cg_sec, 8, REG(addr_lhs), X86_RSP); // movq %rsp, addr_lhs
#endif
            int v = gen(node->lhs);
            if (is_complex(node->lhs->ty)) {
                // gen() returned the address of the complex payload — copy it
#ifdef ARCH_ARM64
                arm64_ldr_uoff(cg_sec, 3, ARM64_X16, REG(v), 0); // ldr x16, [v]
                arm64_str_uoff(cg_sec, 3, ARM64_X16, REG(addr_lhs), 0); // str x16, [addr_lhs]
                if (complex_sz > 8) {
                    arm64_ldr_uoff(cg_sec, 3, ARM64_X16, REG(v), (uint32_t)(base_sz / 8)); // ldr x16, [v, #base_sz]
                    arm64_str_uoff(cg_sec, 3, ARM64_X16, REG(addr_lhs), (uint32_t)(base_sz / 8)); // str x16, [addr_lhs, #base_sz]
                }
#else
                x86_mov_rm(cg_sec, 8, X86_RAX, x86_mem(REG(v), 0)); // movq (v), %rax
                x86_mov_mr(cg_sec, 8, x86_mem(REG(addr_lhs), 0), X86_RAX); // movq %rax, (addr_lhs)
                if (complex_sz > 8) {
                    x86_mov_rm(cg_sec, 8, X86_RAX, x86_mem(REG(v), base_sz)); // movq base_sz(v), %rax
                    x86_mov_mr(cg_sec, 8, x86_mem(REG(addr_lhs), base_sz), X86_RAX); // movq %rax, base_sz(addr_lhs)
                }
#endif
            } else {
#ifdef ARCH_ARM64
                asm_str_reg_off(cg_sec, v, addr_lhs, 8, 0);
                if (complex_sz > 8)
                    arm64_str_uoff(cg_sec, 3, ARM64_XZR, REG(addr_lhs), (uint32_t)(base_sz / 8)); // str xzr, [addr_lhs, #base_sz]
#else
                asm_mov_reg_mem(cg_sec, v, addr_lhs, 8); // movq v, (addr_lhs)
                if (complex_sz > 8)
                    x86_mov_mi(cg_sec, 8, x86_mem(REG(addr_lhs), base_sz), 0); // movq $0, base_sz(addr_lhs)
#endif
            }
            free_reg(v);
            need_free_lhs = 1;
        }
        if (addr_rhs < 0 || (is_complex(node->rhs->ty) && node->rhs->ty->size < complex_sz)) {
            addr_rhs = alloc_reg();
#ifdef ARCH_ARM64
            arm64_sub_imm(cg_sec, 1, ARM64_SP, ARM64_SP, (complex_sz + 15) & ~15, 0); // sub sp, sp, #complex_sz
            asm_mov_reg_sp(cg_sec, REG(addr_rhs)); // mov addr_rhs, sp
#else
            asm_sub_rsp_imm(cg_sec, (complex_sz + 15) & ~15); // subq $N, %rsp
            x86_mov_rr(cg_sec, 8, REG(addr_rhs), X86_RSP); // movq %rsp, addr_rhs
#endif
            int v = gen(node->rhs);
            if (is_complex(node->rhs->ty)) {
                // gen() returned the address of the complex payload — copy it
#ifdef ARCH_ARM64
                int rhs_sz = node->rhs->ty->size;
                int rhs_base = node->rhs->ty->base ? node->rhs->ty->base->size : 8;
                arm64_ldr_uoff(cg_sec, 3, ARM64_X16, REG(v), 0); // ldr x16, [v]
                arm64_str_uoff(cg_sec, 3, ARM64_X16, REG(addr_rhs), 0); // str x16, [addr_rhs]
                if (rhs_sz > rhs_base) {
                    arm64_ldr_uoff(cg_sec, 3, ARM64_X16, REG(v), (uint32_t)(rhs_base / 8)); // ldr x16, [v, #rhs_base]
                    arm64_str_uoff(cg_sec, 3, ARM64_X16, REG(addr_rhs), (uint32_t)(rhs_base / 8)); // str x16, [addr_rhs, #rhs_base]
                }
#else
                x86_mov_rm(cg_sec, 8, X86_RAX, x86_mem(REG(v), 0)); // movq (v), %rax
                x86_mov_mr(cg_sec, 8, x86_mem(REG(addr_rhs), 0), X86_RAX); // movq %rax, (addr_rhs)
                if (complex_sz > 8) {
                    x86_mov_rm(cg_sec, 8, X86_RAX, x86_mem(REG(v), base_sz)); // movq base_sz(v), %rax
                    x86_mov_mr(cg_sec, 8, x86_mem(REG(addr_rhs), base_sz), X86_RAX); // movq %rax, base_sz(addr_rhs)
                }
#endif
            } else {
#ifdef ARCH_ARM64
                asm_str_reg_off(cg_sec, v, addr_rhs, 8, 0);
                if (complex_sz > 8)
                    arm64_str_uoff(cg_sec, 3, ARM64_XZR, REG(addr_rhs), (uint32_t)(base_sz / 8)); // str xzr, [addr_rhs, #base_sz]
#else
                asm_mov_reg_mem(cg_sec, v, addr_rhs, 8); // movq v, (addr_rhs)
                if (complex_sz > 8)
                    x86_mov_mi(cg_sec, 8, x86_mem(REG(addr_rhs), base_sz), 0); // movq $0, base_sz(addr_rhs)
#endif
            }
            free_reg(v);
            need_free_rhs = 1;
        }
        int result = alloc_reg();
#ifdef ARCH_ARM64
        if (sz <= 4) {
            asm_movq_zero(cg_sec, result);
            if (sz == 2) {
                arm64_ldrh_uoff(cg_sec, ARM64_X16, REG(addr_lhs), 0); // ldrh w16, [addr_lhs]
                arm64_ldrh_uoff(cg_sec, ARM64_X17, REG(addr_rhs), 0); // ldrh w17, [addr_rhs]
            } else {
                arm64_ldr_uoff(cg_sec, 2, ARM64_X16, REG(addr_lhs), 0); // ldr w16, [addr_lhs]
                arm64_ldr_uoff(cg_sec, 2, ARM64_X17, REG(addr_rhs), 0); // ldr w17, [addr_rhs]
            }
            arm64_eor_reg(cg_sec, 0, ARM64_X16, ARM64_X16, ARM64_X17, ARM64_LSL, 0); // eor w16, w16, w17
            arm64_subs_imm(cg_sec, 0, ARM64_XZR, ARM64_X16, 0, 0); // cmp w16, #0
        } else {
            asm_ldr_reg_off(cg_sec, result, addr_lhs, 8, 0);
            arm64_ldr_uoff(cg_sec, 3, ARM64_X16, REG(addr_rhs), 0); // ldr x16, [addr_rhs]
            arm64_eor_reg(cg_sec, 1, REG(result), REG(result), ARM64_X16, ARM64_LSL, 0); // eor result, result, x16
            if (sz > 8) {
                arm64_ldr_uoff(cg_sec, 3, ARM64_X16, REG(addr_lhs), (uint32_t)(base_sz / 8)); // ldr x16, [addr_lhs, #base_sz]
                arm64_ldr_uoff(cg_sec, 3, ARM64_X17, REG(addr_rhs), (uint32_t)(base_sz / 8)); // ldr x17, [addr_rhs, #base_sz]
                arm64_eor_reg(cg_sec, 1, ARM64_X16, ARM64_X16, ARM64_X17, ARM64_LSL, 0); // eor x16, x16, x17
                arm64_orr_reg(cg_sec, 1, REG(result), REG(result), ARM64_X16, ARM64_LSL, 0); // orr result, result, x16
            }
            asm_cmp_zero(cg_sec, result, 8);
        }
        if (node->kind == ND_EQ)
            asm_cset(cg_sec, result, ARM64_EQ);
        else
            asm_cset(cg_sec, result, ARM64_NE);
#else
        if (sz <= 4) {
            asm_movl_zero(cg_sec, result);
            x86_mov_rm(cg_sec, 1, X86_RAX, x86_mem(REG(addr_lhs), 0)); // movb (addr_lhs), %al
            x86_xor_rm(cg_sec, 1, X86_RAX, x86_mem(REG(addr_rhs), 0)); // xorb (addr_rhs), %al
            asm_movzx_phys(cg_sec, result, X86_RAX, 4, 1); // movzbl %al, result
        } else {
            x86_mov_rm(cg_sec, 8, REG(result), x86_mem(REG(addr_lhs), 0)); // movq (addr_lhs), result
            x86_xor_rm(cg_sec, 8, REG(result), x86_mem(REG(addr_rhs), 0)); // xorq (addr_rhs), result
            if (sz > 8) {
                x86_mov_rm(cg_sec, 8, X86_RAX, x86_mem(REG(addr_lhs), base_sz)); // movq base_sz(addr_lhs), %rax
                x86_xor_rm(cg_sec, 8, X86_RAX, x86_mem(REG(addr_rhs), base_sz)); // xorq base_sz(addr_rhs), %rax
                x86_or_rr(cg_sec, 8, REG(result), X86_RAX); // orq %rax, result
            }
        }
        if (node->kind == ND_EQ)
            asm_setcc(cg_sec, X86_RAX, X86_E);
        else
            asm_setcc(cg_sec, X86_RAX, X86_NE);
        asm_movzx_phys(cg_sec, result, X86_RAX, 4, 1);
#endif
        free_reg(addr_lhs);
        free_reg(addr_rhs);
#ifdef ARCH_ARM64
        if (need_free_lhs) arm64_add_imm(cg_sec, 1, ARM64_SP, ARM64_SP, (complex_sz + 15) & ~15, 0); // add sp, sp, #complex_sz
        if (need_free_rhs) arm64_add_imm(cg_sec, 1, ARM64_SP, ARM64_SP, (complex_sz + 15) & ~15, 0); // add sp, sp, #complex_sz
#else
        if (need_free_lhs) asm_add_rsp_imm(cg_sec, (complex_sz + 15) & ~15);
        if (need_free_rhs) asm_add_rsp_imm(cg_sec, (complex_sz + 15) & ~15);
#endif
        return result;
    }

    // Commute a constant-lhs equality (C == x / C != x) so the constant lands
    // on the rhs and uses the immediate compare path.  Otherwise gen(lhs)
    // materialises the constant into a register that is then spilled while the
    // rhs address is computed; if the rhs reuses that register the compare
    // degenerates to a self-compare (and the spill slot can be clobbered).
    // EQ/NE are commutative so this is a pure win; LT/LE cannot be flipped
    // without a GT/GE opcode (the parser already normalises those away).
    if ((node->kind == ND_EQ || node->kind == ND_NE) && node->lhs->kind == ND_NUM && node->rhs->kind != ND_NUM) {
        Node *swap_tmp = node->lhs;
        node->lhs = node->rhs;
        node->rhs = swap_tmp;
    }
    VReg r_lhs = gen(node->lhs);


    // Float binary operations (must come before integer ops)
    if (is_flonum(node->ty) && (node->kind == ND_ADD || node->kind == ND_SUB || node->kind == ND_MUL || node->kind == ND_DIV)) {
        VReg r_rhs = gen(node->rhs);
#ifdef ARCH_ARM64
        bool rhs_in_x16 = false;
        if (r_lhs == r_rhs && (spilled_regs & (1 << r_lhs))) {
            // alloc_reg() reused r_lhs for rhs and spilled the original lhs.
            // Preserve rhs in x16, then reload lhs before feeding d0/d1.
            asm_mov_phy_reg(cg_sec, ARM64_X16, r_rhs, 1); // mov x16, x{r_rhs}
            asm_ldur_fp(cg_sec, r_lhs, spill_offset(r_lhs)); // ldr x{r_lhs}, [x29, #-spill_offset]
            spilled_regs &= ~(1 << r_lhs);
            rhs_in_x16 = true;
        }
        asm_fmov_i2f(cg_sec, 0, r_lhs, 1); // fmov d0, x{r_lhs}
        if (rhs_in_x16)
            arm64_fmov_i2f(cg_sec, 1, ARM64_D1, ARM64_X16); // fmov d1, x16
        else
            asm_fmov_i2f(cg_sec, 1, r_rhs, 1); // fmov d1, x{r_rhs}
        if (node->kind == ND_ADD) asm_fadd(cg_sec, 1); // fadd d0, d0, d1
        else if (node->kind == ND_SUB)
            asm_fsub(cg_sec, 1); // fsub d0, d0, d1
        else if (node->kind == ND_MUL)
            asm_fmul(cg_sec, 1); // fmul d0, d0, d1
        else if (node->kind == ND_DIV)
            asm_fdiv(cg_sec, 1); // fdiv d0, d0, d1
        if (node->ty->kind == TY_FLOAT) {
            asm_fcvt(cg_sec, 0, 1, 0, 0); // fcvt s0, d0 (opc=0=double->single)
            asm_fcvt(cg_sec, 1, 0, 0, 0); // asm_fcvt(1, 0, 0, 0)
        }
        asm_fmov_f2i(cg_sec, r_lhs, 0, 1); // asm_fmov_f2i(r_lhs, 0, 1)
        if (!rhs_in_x16 && r_rhs != r_lhs)
            free_reg(r_rhs);
#else
        asm_movq_r_xmm(cg_sec, X86_XMM0, r_lhs); // fcvt s0, d0
        asm_movq_r_xmm(cg_sec, X86_XMM1, r_rhs); // fcvt d0, s0
        free_reg(r_rhs);
        __attribute__((unused)) char *inst = "";
        if (node->kind == ND_ADD) inst = "addsd";
        else if (node->kind == ND_SUB)
            inst = "subsd";
        else if (node->kind == ND_MUL)
            inst = "mulsd";
        else if (node->kind == ND_DIV)
            inst = "divsd";
        if (node->kind == ND_ADD) asm_addsd(cg_sec); // %s %%xmm1, %%xmm0
        else if (node->kind == ND_SUB)
            asm_subsd(cg_sec); // cvtsd2ss %%xmm0, %%xmm0
        else if (node->kind == ND_MUL)
            asm_mulsd(cg_sec); // cvtss2sd %%xmm0, %%xmm0
        else if (node->kind == ND_DIV)
            asm_divsd(cg_sec); // movq %%xmm0, %s
        if (node->ty->kind == TY_FLOAT) {
            asm_cvtsd2ss(cg_sec); // %s %%xmm1, %%xmm0
            asm_cvtss2sd(cg_sec); // cvtsd2ss %%xmm0, %%xmm0
        }
        asm_movq_xmm_r(cg_sec, r_lhs, X86_XMM0); // cvtss2sd %%xmm0, %%xmm0
#endif
        return r_lhs;
    }

    // Float comparisons
    if ((node->kind == ND_EQ || node->kind == ND_NE || node->kind == ND_LT || node->kind == ND_LE) &&
        node->lhs->ty && node->rhs->ty && (is_flonum(node->lhs->ty) || is_flonum(node->rhs->ty))) {
        VReg r_rhs = gen(node->rhs);
#ifdef ARCH_ARM64
        bool rhs_in_x16 = false;
        if (r_lhs == r_rhs && (spilled_regs & (1 << r_lhs))) {
            // Preserve rhs in x16, then reload the spilled lhs for comparison.
            asm_mov_phy_reg(cg_sec, ARM64_X16, r_rhs, 1); // mov x16, x{r_rhs}
            asm_ldur_fp(cg_sec, r_lhs, spill_offset(r_lhs)); // ldr x{r_lhs}, [x29, #-spill_offset]
            spilled_regs &= ~(1 << r_lhs);
            rhs_in_x16 = true;
        }
        asm_fmov_i2f(cg_sec, 0, r_lhs, 1); // fmov d0, x{r_lhs}
        if (rhs_in_x16)
            arm64_fmov_i2f(cg_sec, 1, ARM64_D1, ARM64_X16); // fmov d1, x16
        else
            asm_fmov_i2f(cg_sec, 1, r_rhs, 1); // fmov d1, x{r_rhs}
        asm_fcmp(cg_sec, 1); // fcmp d0, d1
        if (node->kind == ND_EQ) asm_cset(cg_sec, r_lhs, ARM64_EQ); // cset rr_lhs
        else if (node->kind == ND_NE)
            asm_cset(cg_sec, r_lhs, ARM64_NE); // fcmp d0, d1
        else if (node->kind == ND_LT)
            asm_cset(cg_sec, r_lhs, ARM64_MI); // cset rr_lhs
        else if (node->kind == ND_LE)
            asm_cset(cg_sec, r_lhs, ARM64_LS); // cset rr_lhs
#else
        asm_movq_r_xmm(cg_sec, X86_XMM0, r_lhs); // movq %s, %%xmm0
        asm_movq_r_xmm(cg_sec, X86_XMM1, r_rhs); // movq %s, %%xmm1
        asm_ucomisd(cg_sec); // ucomisd %%xmm1, %%xmm0
        if (node->kind == ND_EQ) {
            asm_setcc(cg_sec, X86_RAX, X86_E); // sete %%al
            asm_setcc(cg_sec, X86_RCX, X86_NP); // setnp %%cl
            x86_and_rr(cg_sec, 1, X86_RAX, X86_RCX); // andb %%cl, %%al
        } else if (node->kind == ND_NE) {
            asm_setcc(cg_sec, X86_RAX, X86_NE); // setne %%al
            asm_setcc(cg_sec, X86_RCX, X86_P); // setp %%cl
            x86_or_rr(cg_sec, 1, X86_RAX, X86_RCX); // orb %%cl, %%al
        } else if (node->kind == ND_LT) {
            // ucomisd sets CF=1 for both "lhs < rhs" AND unordered (either
            // operand NaN), so a bare `setb` reports NaN comparisons as
            // true. IEEE 754/C require every ordering comparison against
            // NaN to be false, so gate on "ordered" (PF=0) same as EQ/NE
            // above. `>` is parsed as a swapped `<` (see parser.c), so
            // this single fix also corrects `a > b`.
            asm_setcc(cg_sec, X86_RAX, X86_B); // setb %%al
            asm_setcc(cg_sec, X86_RCX, X86_NP); // setnp %%cl
            x86_and_rr(cg_sec, 1, X86_RAX, X86_RCX); // andb %%cl, %%al
        } else if (node->kind == ND_LE) {
            // Same unordered pitfall as LT: BE (CF=1 or ZF=1) is also true
            // when unordered. `>=` is a swapped `<=` (see parser.c).
            asm_setcc(cg_sec, X86_RAX, X86_BE); // setbe %%al
            asm_setcc(cg_sec, X86_RCX, X86_NP); // setnp %%cl
            x86_and_rr(cg_sec, 1, X86_RAX, X86_RCX); // andb %%cl, %%al
        }
        asm_movzx_phys(cg_sec, r_lhs, X86_RAX, 4, 1); // movzbl %%al, %s
#endif
        if (r_rhs != r_lhs)
            free_reg(r_rhs);
        return r_lhs;
    }


    // Fused Division/Modulo Optimization
    if (node->kind == ND_DIV || node->kind == ND_MOD) {
        int sz = op_size(node->ty);
        bool is_unsigned = use_unsigned(node->ty);
        VReg r_rhs = gen(node->rhs);
#ifdef ARCH_ARM64
        if (node->kind == ND_DIV) {
            if (is_unsigned)
                asm_udiv_reg_reg(cg_sec, r_lhs, r_rhs, sz); // udiv r_lhs, r_lhs, r_rhs
            else
                asm_sdiv_reg_reg(cg_sec, r_lhs, r_rhs, sz); // sdiv r_lhs, r_lhs, r_rhs
        } else {
            VReg tmp = alloc_reg();
            asm_mov_reg_reg(cg_sec, tmp, r_lhs, sz); // mov tmp, r_lhs
            if (is_unsigned)
                asm_udiv_reg_reg(cg_sec, tmp, r_rhs, sz); // udiv tmp, tmp, r_rhs
            else
                asm_sdiv_reg_reg(cg_sec, tmp, r_rhs, sz); // sdiv tmp, tmp, r_rhs
            asm_mul_reg_reg(cg_sec, tmp, r_rhs, sz); // mul tmp, tmp, rhs
            asm_sub_reg_reg(cg_sec, r_lhs, tmp, sz); // sub r_lhs, r_lhs, tmp
            free_reg(tmp);
        }
#else
        x86_mov_rr(cg_sec, sz, X86_RAX, REG(r_lhs)); // movq r_lhs, %rax
        if (is_unsigned) {
            x86_xor_rr(cg_sec, 4, X86_RDX, X86_RDX); // xorl %edx, %edx
        } else {
            if (sz == 8) asm_cqo(cg_sec); // cqo
            else
                asm_cdq(cg_sec); // cdq
        }
        if (is_unsigned)
            asm_div(cg_sec, r_rhs, sz); // div r_rhs
        else
            asm_idiv(cg_sec, r_rhs, sz); // idiv r_rhs
        if (node->kind == ND_DIV)
            x86_mov_rr(cg_sec, sz, REG(r_lhs), X86_RAX); // movq %rax, r_lhs
        else
            x86_mov_rr(cg_sec, sz, REG(r_lhs), X86_RDX); // movq %rdx, r_lhs
#endif
        free_reg(r_rhs);
        return r_lhs;
    }

    // Binary operators with potential immediate/memory optimization for RHS
    if (node->kind == ND_SHL || node->kind == ND_SHR) {
        int sz = op_size(node->ty);
#ifdef ARCH_ARM64
        int sf = (sz == 8) ? 1 : 0;
        if (node->rhs->kind == ND_NUM) {
            int s = (int)node->rhs->val;
            if (s >= sz * 8) {
                asm_movq_zero(cg_sec, r_lhs); // xor rr_lhs,rr_lhs (set zero)
            } else {
                if (node->kind == ND_SHL)
                    asm_shl_imm(cg_sec, r_lhs, sf ? 8 : 4, (uint8_t)(s)); // shl r_lhs, #s
                else if (use_unsigned(node->ty))
                    asm_shr_imm(cg_sec, r_lhs, sf ? 8 : 4, (uint8_t)(s)); // shr r_lhs, #s
                else
                    asm_sar_imm(cg_sec, r_lhs, sf ? 8 : 4, (uint8_t)(s)); // sar r_lhs, #s
            }
        } else {
            VReg r_rhs = gen(node->rhs);
            if (node->kind == ND_SHL)
                asm_shl_cl(cg_sec, r_lhs, sf ? 8 : 4, r_rhs); // lsl r_lhs, r_lhs, r_rhs
            else if (use_unsigned(node->ty))
                asm_shr_cl(cg_sec, r_lhs, sf ? 8 : 4, r_rhs); // lsr r_lhs, r_lhs, r_rhs
            else
                asm_sar_cl(cg_sec, r_lhs, sf ? 8 : 4, r_rhs); // asr r_lhs, r_lhs, r_rhs
            free_reg(r_rhs);
        }
#else
        if (node->rhs->kind == ND_NUM) {
            int imm = (int)node->rhs->val;
            if (node->kind == ND_SHL)
                asm_shl_imm(cg_sec, r_lhs, sz, (uint8_t)(imm)); // shl $(uint8_t)(imm), rr_lhs
            else if (use_unsigned(node->ty))
                asm_shr_imm(cg_sec, r_lhs, sz, (uint8_t)(imm)); // shr $(uint8_t)(imm), rr_lhs
            else
                asm_sar_imm(cg_sec, r_lhs, sz, (uint8_t)(imm)); // sar $(uint8_t)(imm), rr_lhs
        } else {
            VReg r_rhs = gen(node->rhs);
            // Save the shift count into CL before restoring a spilled lhs.
            // If r_rhs landed on the same VReg as r_lhs (register pressure
            // spilled r_lhs during gen(rhs) and alloc_reg returned it), the
            // physical register now holds the rhs (shift count), not the
            // lhs (value to shift).  We must capture the count in CL first,
            // then restore the lhs from its spill slot.
            x86_mov_rr(cg_sec, 4, X86_RCX, REG(r_rhs)); // movl %s, %ecx
            bool spilled_lhs = (r_lhs == r_rhs && (spilled_regs & (1 << r_lhs)));
            if (spilled_lhs) {
                asm_mov_rbp_reg(cg_sec, r_lhs, 8, spill_offset(r_lhs));
                spilled_regs &= ~(1 << r_lhs);
            }
            if (node->kind == ND_SHL)
                asm_shl_cl(cg_sec, r_lhs, sz, r_rhs); // shl cl, rr_lhs
            else if (use_unsigned(node->ty))
                asm_shr_cl(cg_sec, r_lhs, sz, r_rhs);
            else
                asm_sar_cl(cg_sec, r_lhs, sz, r_rhs);
            free_reg(r_rhs);
        }
#endif
        // Extended bitfield shift: mask result to bitfield width.
        // GCC truncates shifts of a >32-bit unsigned bitfield to its width.
        {
            int bw = 0;
            if (node->lhs && node->lhs->kind == ND_MEMBER && node->lhs->member &&
                node->lhs->member->bit_width > 32 && node->lhs->member->bit_width < 64 &&
                node->lhs->member->ty && node->lhs->member->ty->size >= 8 &&
                node->lhs->member->ty->is_unsigned)
                bw = node->lhs->member->bit_width;
            if (bw > 0) {
                unsigned long long mask = (1ULL << bw) - 1;
#ifdef ARCH_ARM64
                emit_mov_imm64(ARM64_X17, mask);
                asm_and_reg_phy(cg_sec, r_lhs, ARM64_X17, 8); // and r_lhs, r_lhs, x17
#else
                asm_movabs_phy(cg_sec, X86_RAX, (uint64_t)mask); // movabsq $mask, %rax
                asm_and_rax(cg_sec, r_lhs, 8); // andq %rax, r_lhs
#endif
            }
        }
        return r_lhs;
    }

    if (node->kind == ND_ADD || node->kind == ND_SUB || node->kind == ND_MUL ||
        node->kind == ND_BITAND || node->kind == ND_BITXOR || node->kind == ND_BITOR ||
        node->kind == ND_EQ || node->kind == ND_NE || node->kind == ND_LT || node->kind == ND_LE) {
        char *inst = "";
        if (node->kind == ND_ADD) inst = "add";
        else if (node->kind == ND_SUB)
            inst = "sub";
        else if (node->kind == ND_MUL)
            inst =
#ifdef ARCH_ARM64
                "mul";
#else
                "imul";
#endif
        else if (node->kind == ND_BITAND)
            inst = "and";
        else if (node->kind == ND_BITXOR)
            inst =
#ifdef ARCH_ARM64
                "eor";
#else
                "xor";
#endif
        else if (node->kind == ND_BITOR)
            inst =
#ifdef ARCH_ARM64
                "orr";
#else
                "or";
#endif
        else
            inst = "cmp";

        int sz = (node->kind == ND_EQ || node->kind == ND_NE || node->kind == ND_LT || node->kind == ND_LE)
            ? op_size(node->lhs->ty)
            : op_size(node->ty);
        if (sz < op_size(node->rhs->ty))
            sz = op_size(node->rhs->ty);

        // Fold small integer constants, including -C from ND_NEG(ND_NUM(C))
        int64_t rhs_imm_val;
        bool rhs_use_imm = false;
        if (node->rhs->kind == ND_NUM) {
            rhs_imm_val = node->rhs->val;
            rhs_use_imm = true;
        } else if (node->rhs->kind == ND_CAST && node->rhs->lhs &&
                   node->rhs->lhs->kind == ND_NUM) {
            // Cast of a constant (e.g. (U8)1ULL): peel the cast and use the
            // underlying immediate.  Otherwise gen(rhs) allocates a VReg that
            // may collide with r_lhs under register pressure in huge functions
            // (>2000 labels), overwriting the lhs value with a mov-imm and
            // degenerating the ALU reg-reg op (or/xor/and) into a no-op.
            rhs_imm_val = node->rhs->lhs->val;
            rhs_use_imm = true;
        } else if (node->rhs->kind == ND_NEG && node->rhs->lhs &&
                   node->rhs->lhs->kind == ND_NUM) {
        }
        if (rhs_use_imm && rhs_imm_val == (int32_t)rhs_imm_val) {
            // Skip identity operations: add 0, sub 0, mul 1, and ~0, or 0, xor 0
            int imm = (int)rhs_imm_val;
            if ((node->kind == ND_MUL && imm == 1) ||
                ((node->kind == ND_ADD || node->kind == ND_SUB || node->kind == ND_BITOR || node->kind == ND_BITXOR) && imm == 0)) {
                // no-op, just return r_lhs
#ifdef ARCH_ARM64
            } else if (node->kind == ND_MUL && imm > 0 && (imm & (imm - 1)) == 0) {
                // Strength reduction: multiply by power of 2 → shift
                int shift = 0;
                int tmp = imm;
                while (tmp > 1) {
                    shift++;
                    tmp >>= 1;
                }
                asm_shl_imm(cg_sec, r_lhs, sz, (uint8_t)shift); // lsl r_lhs, r_lhs, #shift
            } else if (node->kind == ND_MUL) {
                // ARM64 mul doesn't take immediates; load into a scratch register
                VReg tmp = alloc_reg();
                emit_mov_imm(reg(tmp, sz), imm);
                asm_mul_reg_reg(cg_sec, r_lhs, tmp, sz); // mul r_lhs, r_lhs, tmp
                free_reg(tmp);
            } else if (!strcmp(inst, "cmp")) {
                if (imm >= 0 && imm <= 4095)
                    asm_cmp_imm(cg_sec, r_lhs, sz, imm); // cmp r_lhs, #imm
                else if (imm < 0 && imm >= -4095)
                    asm_cmn_imm(cg_sec, r_lhs, sz == 8 ? 1 : 0, -imm); // cmn r_lhs, #-imm
                else {
                    emit_mov_imm64(ARM64_X16, (uint64_t)(int64_t)imm);
                    asm_cmp_reg_phy(cg_sec, r_lhs, ARM64_X16, sz); // cmp r_lhs, x16
                }
            } else if (node->kind != ND_BITAND && node->kind != ND_BITOR && node->kind != ND_BITXOR &&
                       imm >= 0 && imm <= 4095) {
                if (!strcmp(inst, "add")) asm_add_imm(cg_sec, r_lhs, sz, imm); // add r_lhs, r_lhs, #imm
                else if (!strcmp(inst, "sub"))
                    asm_sub_imm(cg_sec, r_lhs, sz, imm); // sub r_lhs, r_lhs, #imm
            } else {
                VReg tmp = alloc_reg();
                emit_mov_imm(reg(tmp, sz), imm);
                if (!strcmp(inst, "add")) asm_add_reg_reg(cg_sec, r_lhs, tmp, sz); // add r_lhs, r_lhs, tmp
                else if (!strcmp(inst, "sub"))
                    asm_sub_reg_reg(cg_sec, r_lhs, tmp, sz); // sub r_lhs, r_lhs, tmp
                else if (!strcmp(inst, "and"))
                    asm_and_reg_reg(cg_sec, r_lhs, tmp, sz); // and r_lhs, r_lhs, tmp
                else if (!strcmp(inst, "orr"))
                    asm_or_reg_reg(cg_sec, r_lhs, tmp, sz); // orr r_lhs, r_lhs, tmp
                else if (!strcmp(inst, "eor"))
                    asm_eor_reg_reg(cg_sec, r_lhs, tmp, sz); // eor r_lhs, r_lhs, tmp
                free_reg(tmp);
            }
#else
            } else if (node->kind == ND_MUL && imm > 0 && (imm & (imm - 1)) == 0) {
                // Strength reduction: multiply by power of 2 → shift
                int shift = 0;
                int tmp = imm;
                while (tmp > 1) {
                    shift++;
                    tmp >>= 1;
                }
                asm_shl_imm(cg_sec, r_lhs, sz, (uint8_t)shift); // shl r_lhs, #shift
            } else {
                if (!strcmp(inst, "add")) asm_add_imm(cg_sec, r_lhs, sz, imm); // add r_lhs, #imm
                else if (!strcmp(inst, "sub"))
                    asm_sub_imm(cg_sec, r_lhs, sz, imm); // sub r_lhs, #imm
                else if (!strcmp(inst, "imul")) {
                    // FIXME arm
                    asm_imul_imm(cg_sec, r_lhs, r_lhs, sz, (int32_t)rhs_imm_val); // imul $val, r_lhs, r_lhs
                } else if (!strcmp(inst, "and"))
                    asm_and_imm(cg_sec, r_lhs, sz, imm); // and r_lhs, #imm
                else if (!strcmp(inst, "or"))
                    asm_or_imm(cg_sec, r_lhs, sz, imm); // or r_lhs, #imm
                else if (!strcmp(inst, "xor"))
                    asm_xor_imm(cg_sec, r_lhs, sz, imm); // xor r_lhs, #imm
                else if (!strcmp(inst, "cmp"))
                    asm_cmp_imm(cg_sec, r_lhs, sz, imm); // cmp r_lhs, #imm
            }
#endif
        } else {
            VReg r_rhs = gen(node->rhs);
#ifdef ARCH_ARM64
            // Sign-extend rhs to 64 bits when operation is 64-bit but rhs was
            // computed as 32-bit signed. ARM64: use sxtw.
            if (sz == 8 && op_size(node->rhs->ty) == 4 && !use_unsigned(node->rhs->ty))
                asm_movsx(cg_sec, r_rhs, r_rhs, 8, 4); // movsx8->r_rhs rr_rhs, rr_rhs
            if (!strcmp(inst, "cmp")) {
                asm_cmp_reg_reg(cg_sec, r_lhs, r_rhs, sz); // cmp rr_rhs, rr_lhs
            } else {
                if (!strcmp(inst, "add")) {
                    if (r_lhs == r_rhs && (spilled_regs & (1 << r_lhs))) {
                        asm_ldur_fp_phy(cg_sec, ARM64_X16, spill_offset(r_lhs));
                        spilled_regs &= ~(1 << r_lhs);
                        arm64_add_reg(cg_sec, sz == 8 ? 1 : 0, REG(r_lhs), REG(r_lhs), ARM64_X16, ARM64_LSL, 0);
                    } else {
                        asm_add_reg_reg(cg_sec, r_lhs, r_rhs, sz);
                    }
                } else if (!strcmp(inst, "sub"))
                    asm_sub_reg_reg(cg_sec, r_lhs, r_rhs, sz); // sub rr_lhs, rr_rhs
                else if (!strcmp(inst, "mul"))
                    asm_mul_reg_reg(cg_sec, r_lhs, r_rhs, sz); // imul rr_lhs, rr_rhs
                else if (!strcmp(inst, "and"))
                    asm_and_reg_reg(cg_sec, r_lhs, r_rhs, sz); // and rr_lhs, rr_rhs
                else if (!strcmp(inst, "eor"))
                    asm_eor_reg_reg(cg_sec, r_lhs, r_rhs, sz); // xor rr_lhs, rr_rhs
                else if (!strcmp(inst, "orr"))
                    asm_or_reg_reg(cg_sec, r_lhs, r_rhs, sz); // or rr_lhs, rr_rhs
            }
            // Pointer subtraction: divide byte difference by element size
            if (node->kind == ND_SUB && node->lhs->ty->base && node->rhs->ty->base) {
                int elem_sz = node->lhs->ty->base->size;
                if (elem_sz > 1) {
                    if ((elem_sz & (elem_sz - 1)) == 0) {
                        int shift = 0, tmp = elem_sz;
                        while (tmp > 1) {
                            shift++;
                            tmp >>= 1;
                        }
                        asm_sar_imm(cg_sec, r_lhs, sz, (uint8_t)shift); // asr r_lhs, r_lhs, #shift
                    } else {
                        VReg tmp = alloc_reg();
                        asm_mov_imm(cg_sec, tmp, sz, (int64_t)elem_sz); // mov tmp, #elem_sz
                        asm_sdiv_reg_reg(cg_sec, r_lhs, tmp, sz); // sdiv r_lhs, r_lhs, tmp
                        free_reg(tmp);
                    }
                }
            }
#else
            // Sign-extend rhs to 64 bits when operation is 64-bit but rhs was
            // computed as 32-bit signed (e.g. pointer + int, long + int).
            // 32-bit writes zero-extend in x86-64, so without this a negative
            // int would be treated as a large positive offset.
            if (sz == 8 && op_size(node->rhs->ty) == 4 && !use_unsigned(node->rhs->ty))
                asm_movsx(cg_sec, r_rhs, r_rhs, 8, 4);
            if (!strcmp(inst, "add")) {
                if (r_lhs == r_rhs && (spilled_regs & (1 << r_lhs))) {
                    asm_add_spill_reg(cg_sec, r_lhs, sz, spill_offset(r_lhs));
                    spilled_regs &= ~(1 << r_lhs);
                } else {
                    asm_add_reg_reg(cg_sec, r_lhs, r_rhs, sz);
                }
            } else if (!strcmp(inst, "sub")) {
                if (r_lhs == r_rhs && (spilled_regs & (1 << r_lhs))) {
                    // reg holds rhs, spill holds lhs: reg = rhs - lhs, then negate
                    // to yield lhs - rhs.
                    asm_sub_spill_reg(cg_sec, r_lhs, sz, spill_offset(r_lhs));
                    asm_neg(cg_sec, r_lhs, sz);
                    spilled_regs &= ~(1 << r_lhs);
                } else {
                    asm_sub_reg_reg(cg_sec, r_lhs, r_rhs, sz);
                }
            } else if (!strcmp(inst, "imul")) {
                if (r_lhs == r_rhs && (spilled_regs & (1 << r_lhs))) {
                    // reg holds rhs, spill holds lhs. imul is commutative:
                    // load lhs into RAX scratch and multiply reg by it.
                    x86_mov_rm(cg_sec, sz, X86_RAX,
                               x86_mem(X86_RBP, -spill_offset(r_lhs))); // mov -off(%rbp), %rax
                    x86_imul_rr(cg_sec, sz, REG(r_lhs), X86_RAX); // imul %rax, reg
                    spilled_regs &= ~(1 << r_lhs);
                } else {
                    asm_mul_reg_reg(cg_sec, r_lhs, r_rhs, sz);
                }
            } else if (!strcmp(inst, "and")) {
                if (r_lhs == r_rhs && (spilled_regs & (1 << r_lhs))) {
                    asm_and_spill_reg(cg_sec, r_lhs, sz, spill_offset(r_lhs));
                    spilled_regs &= ~(1 << r_lhs);
                } else {
                    asm_and_reg_reg(cg_sec, r_lhs, r_rhs, sz);
                }
            } else if (!strcmp(inst, "xor")) {
                if (r_lhs == r_rhs && (spilled_regs & (1 << r_lhs))) {
                    asm_xor_spill_reg(cg_sec, r_lhs, sz, spill_offset(r_lhs));
                    spilled_regs &= ~(1 << r_lhs);
                } else {
                    asm_eor_reg_reg(cg_sec, r_lhs, r_rhs, sz);
                }
            } else if (!strcmp(inst, "or")) {
                if (r_lhs == r_rhs && (spilled_regs & (1 << r_lhs))) {
                    asm_or_spill_reg(cg_sec, r_lhs, sz, spill_offset(r_lhs));
                    spilled_regs &= ~(1 << r_lhs);
                } else {
                    asm_or_reg_reg(cg_sec, r_lhs, r_rhs, sz);
                }
            } else if (!strcmp(inst, "cmp")) {
                if (r_lhs == r_rhs && (spilled_regs & (1 << r_lhs))) {
                    asm_cmp_spill_reg(cg_sec, r_lhs, sz, spill_offset(r_lhs));
                    spilled_regs &= ~(1 << r_lhs);
                } else {
                    asm_cmp_reg_reg(cg_sec, r_lhs, r_rhs, sz);
                }
            }
            // Pointer subtraction: divide byte difference by element size
            if (node->kind == ND_SUB && node->lhs->ty->base && node->rhs->ty->base) {
                int elem_sz = node->lhs->ty->base->size;
                if (elem_sz > 1) {
                    if ((elem_sz & (elem_sz - 1)) == 0) {
                        // Power of 2: use arithmetic shift right
                        int shift = 0;
                        int tmp = elem_sz;
                        while (tmp > 1) {
                            shift++;
                            tmp >>= 1;
                        }
                        asm_sar_imm(cg_sec, r_lhs, sz, (uint8_t)(shift)); // sar $(uint8_t)(shift), rr_lhs
                    } else {
                        // Non-power of 2: use idiv
                        x86_mov_rr(cg_sec, sz, X86_RAX, REG(r_lhs)); // movq r_lhs, %rax
                        if (sz == 8) asm_cqo(cg_sec); // cqo
                        else
                            asm_cdq(cg_sec); // cdq
                        x86_mov_ri(cg_sec, sz, X86_RCX, elem_sz); // movq $elem_sz, %rcx
                        x86_idiv_r(cg_sec, sz, X86_RCX); // idiv %rcx
                        x86_mov_rr(cg_sec, sz, REG(r_lhs), X86_RAX); // movq %rax, r_lhs
                    }
                }
            }
#endif
            free_reg(r_rhs);
        }

        if (node->kind == ND_EQ || node->kind == ND_NE || node->kind == ND_LT || node->kind == ND_LE) {
#ifdef ARCH_ARM64
            Arm64Cond cset_cond = ARM64_EQ;
            if (node->kind == ND_EQ) cset_cond = ARM64_EQ;
            else if (node->kind == ND_NE)
                cset_cond = ARM64_NE;
            else if (node->kind == ND_LT)
                cset_cond = use_unsigned_cmp(node) ? ARM64_LO : ARM64_LT;
            else if (node->kind == ND_LE)
                cset_cond = use_unsigned_cmp(node) ? ARM64_LS : ARM64_LE;
            asm_cset(cg_sec, r_lhs, cset_cond); // cset w{r_lhs}, cond
#else
            X86Cond cond = X86_E;
            if (node->kind == ND_EQ) cond = X86_E;
            else if (node->kind == ND_NE)
                cond = X86_NE;
            else if (node->kind == ND_LT)
                cond = use_unsigned_cmp(node) ? X86_B : X86_L;
            else if (node->kind == ND_LE)
                cond = use_unsigned_cmp(node) ? X86_BE : X86_LE;
            asm_setcc(cg_sec, REG(r_lhs), cond); // setcc rr_lhs
            asm_movzx(cg_sec, r_lhs, r_lhs, 4, 1); // movzx4->r_lhs rr_lhs, rr_lhs
#endif
        }
        // Extended bitfield arithmetic masking (GCC extension for unsigned long long : N).
        // When both operands involve extended (>= 8-byte underlying type) bitfield reads,
        // mask the result to the max bitfield width, matching GCC's behavior.
        if (node->kind == ND_MUL || node->kind == ND_ADD || node->kind == ND_SUB) {
            int bw = 0;
            Node *ops[2] = {node->lhs, node->rhs};
            for (int oi = 0; oi < 2; oi++) {
                Node *op = ops[oi];
                if (op && op->kind == ND_MEMBER && op->member &&
                    op->member->bit_width > 32 && op->member->bit_width < 64 &&
                    op->member->ty && op->member->ty->size >= 8 &&
                    op->member->ty->is_unsigned) {
                    if (op->member->bit_width > bw)
                        bw = op->member->bit_width;
                }
            }
            if (bw > 0) {
                unsigned long long mask = (1ULL << bw) - 1;
#ifdef ARCH_ARM64
                emit_mov_imm64(ARM64_X16, mask);
                asm_and_reg_phy(cg_sec, r_lhs, ARM64_X16, 8); // and rr_lhs, r16
#else
                asm_movabs_phy(cg_sec, X86_RAX, (uint64_t)(mask)); // movabs $(uint64_t)(mask), rX86_RAX
                asm_and_rax(cg_sec, r_lhs, 8); // andq %rax, rr_lhs
#endif
            }
        }

        return r_lhs;
    }

    error("invalid expression %d", node->kind);
    return -1;
}


// Fast peephole parsing helpers (avoid sscanf in hot loop)

static const char *node_kind_name(NodeKind k) {
    switch (k) {
    case ND_ADD: return "ADD";
    case ND_SUB: return "SUB";
    case ND_MUL: return "MUL";
    case ND_DIV: return "DIV";
    case ND_MOD: return "MOD";
    case ND_SHL: return "SHL";
    case ND_SHR: return "SHR";
    case ND_BITAND: return "BITAND";
    case ND_BITXOR: return "BITXOR";
    case ND_BITOR: return "BITOR";
    case ND_EQ: return "EQ";
    case ND_NE: return "NE";
    case ND_LT: return "LT";
    case ND_LE: return "LE";
    case ND_ASSIGN: return "ASSIGN";
    case ND_POST_INC: return "POST_INC";
    case ND_POST_DEC: return "POST_DEC";
    case ND_ADDR: return "ADDR";
    case ND_DEREF: return "DEREF";
    case ND_CAST: return "CAST";
    case ND_BITNOT: return "BITNOT";
    case ND_FUNCALL: return "FUNCALL";
    case ND_LVAR: return "LVAR";
    case ND_NUM: return "NUM";
    case ND_RETURN: return "RETURN";
    case ND_IF: return "IF";
    case ND_FOR: return "FOR";
    case ND_DO: return "DO";
    case ND_SWITCH: return "SWITCH";
    case ND_CASE: return "CASE";
    case ND_BREAK: return "BREAK";
    case ND_CONTINUE: return "CONTINUE";
    case ND_GOTO: return "GOTO";
    case ND_GOTO_IND: return "GOTO_IND";
    case ND_LABEL: return "LABEL";
    case ND_LABEL_VAL: return "LABEL_VAL";
    case ND_STMT_EXPR: return "STMT_EXPR";
    case ND_BLOCK: return "BLOCK";
    case ND_EXPR_STMT: return "EXPR_STMT";
    case ND_NULL: return "NULL";
    case ND_STR: return "STR";
    case ND_MEMBER: return "MEMBER";
    case ND_LOGAND: return "LOGAND";
    case ND_LOGOR: return "LOGOR";
    case ND_COND: return "COND";
    case ND_COMMA: return "COMMA";
    case ND_SIZEOF: return "SIZEOF";
    case ND_FNUM: return "FNUM";
    case ND_NEG: return "NEG";
    case ND_NOT: return "NOT";
    case ND_ZERO_INIT: return "ZERO_INIT";
    case ND_ASM: return "ASM";
    case ND_VA_START: return "VA_START";
    case ND_VA_COPY: return "VA_COPY";
    case ND_VA_ARG: return "VA_ARG";
    case ND_VA_ARG_PACK: return "VA_ARG_PACK";
    case ND_ALLOCA: return "ALLOCA";
    case ND_ALLOCA_ZINIT: return "ALLOCA_ZINIT";
    case ND_CHAIN: return "CHAIN";
    case ND_ATOMIC_LOAD: return "ATOMIC_LOAD";
    case ND_ATOMIC_STORE: return "ATOMIC_STORE";
    case ND_ATOMIC_EXCHANGE: return "ATOMIC_EXCHANGE";
    case ND_ATOMIC_CAS: return "ATOMIC_CAS";
    case ND_ATOMIC_FENCE: return "ATOMIC_FENCE";
    case ND_ATOMIC_FETCH_OP: return "ATOMIC_FETCH_OP";
    default: return "UNKNOWN";
    }
}

static void dump_node(FILE *f, Node *node, int depth) {
    if (!node) return;
    for (int i = 0; i < depth; i++) fputc(' ', f);
    fprintf(f, "%s", node_kind_name(node->kind));
    if (node->kind == ND_NUM) fprintf(f, " %lld", (long long)node->val);
    if (node->kind == ND_LVAR && node->var) fprintf(f, " '%s'", node->var->name ? node->var->name : "(anon)");
    if (node->kind == ND_FUNCALL && node->lhs && node->lhs->kind == ND_LVAR && node->lhs->var)
        fprintf(f, " %s", node->lhs->var->name);
    fprintf(f, "\n");
    if (node->kind == ND_BLOCK)
        for (Node *n = node->body; n; n = n->next) dump_node(f, n, depth + 2);
    if (node->lhs) dump_node(f, node->lhs, depth + 2);
    if (node->rhs) dump_node(f, node->rhs, depth + 2);
    if (node->then) dump_node(f, node->then, depth + 2);
    if (node->els) dump_node(f, node->els, depth + 2);
    if (node->cond) dump_node(f, node->cond, depth + 2);
    if (node->body) dump_node(f, node->body, depth + 2);
    if (node->init) dump_node(f, node->init, depth + 2);
    if (node->inc) dump_node(f, node->inc, depth + 2);
    if (node->kind == ND_CASE && node->lhs) dump_node(f, node->lhs, depth + 2);
}

void dump_ast(Program *prog) {
    fprintf(stderr, "=== AST dump ===\n");
    for (TLItem *item = prog->items; item; item = item->next) {
        if (item->kind == TL_FUNC && item->fn->body) {
            fprintf(stderr, "function %s:\n", item->fn->name);
            dump_node(stderr, item->fn->body, 2);
        }
    }
    fprintf(stderr, "=== end AST dump ===\n");
}

// Deferred label-address-DIFFERENCE patch (GCC's `&&label_a - &&label_b`
// computed-goto jump-table static-initializer idiom — see
// parser.c's extract_label_diff()/append_label_diff_reloc()). Neither
// label's .text offset is known when its owning static's data is emitted
// (the early globals-emission pass below runs strictly before any
// function body, hence before any cg_def_label() call), so the byte patch
// is queued here and resolved right after `owner_fn`'s body has been fully
// codegenned (see the resolution snippet at the end of the function-body
// loop) — NOT after the whole loop: cg_label_ht is reset per-function
// (Pass 2 needs a clean slate), so it only ever holds one function's
// labels at a time. There is no ELF/Mach-O relocation kind for a symbol
// difference, so this is always a same-object byte patch, never
// objfile_add_reloc().
typedef struct LabelDiffPatch LabelDiffPatch;
struct LabelDiffPatch {
    LabelDiffPatch *next;
    int section;
    size_t patch_off;
    char *label_hi;
    char *label_lo;
    char *owner_fn; // function whose codegen defines both labels (LVar.decl_fn_name)
    int size; // patch width in bytes (1/2/4/8)
};
static LabelDiffPatch *pending_label_diffs;

static void queue_label_diff_patch(int section, size_t patch_off, char *label_hi, char *label_lo,
                                   char *owner_fn, int size) {
    LabelDiffPatch *p = arena_alloc(sizeof(LabelDiffPatch));
    p->section = section;
    p->patch_off = patch_off;
    p->label_hi = label_hi;
    p->label_lo = label_lo;
    p->owner_fn = owner_fn;
    p->size = size;
    p->next = pending_label_diffs;
    pending_label_diffs = p;
}

struct ObjFile *codegen(Program *prog) {
    init_local_builtins();
    // Reset label and fixup hashtables for new compilation unit
    cg_label_ht_reset();
    asm_fixup_ht_reset();
    pending_label_diffs = NULL;


    // Initialize binary ObjFile for asm_* emission
    static ObjFile _obj;
    cg_obj = &_obj;
    memset(cg_obj, 0, sizeof(*cg_obj));
    objfile_init(cg_obj);
    cg_set_section(SEC_TEXT);
    cg_dry_run = false;
    all_items = prog->items;
    memset(func_htab, 0, sizeof(func_htab));
    for (TLItem *item = all_items; item; item = item->next)
        if (item->kind == TL_FUNC) {
            uint32_t h = func_hash_name(item->fn->name) % FUNC_HASH_SIZE;
            item->hash_next = func_htab[h];
            func_htab[h] = item;
        }
    all_strs = prog->strs;
    alloca_needed = false;
    last_debug_file = 0;
    last_debug_line = 0;
    // Assembly header
#ifndef ARCH_ARM64
    /* AT&T syntax is now the default */
#endif
#ifndef _WIN32
#if !defined(__APPLE__)
    /* .note.GNU-stack: default in ELF, no bytes emitted */
#endif
#endif

    // Emit data section for strings
    if (prog->globals || prog->strs || float_lits) {
        (void)0 /* section directive */;
        // Track emitted symbols to avoid duplicates (e.g. __asm__("same_name"))
        char **emitted_syms = NULL;
        int emitted_count = 0;
        for (LVar *var = prog->globals; var; var = var->next) {
            // GCC global register variable (`register T x asm("reg");`): no
            // storage, no symbol — see LVar.is_global_reg in rcc.h and
            // gen_global_reg_read() above. Emitting it as an ordinary
            // uninitialized global here is exactly the bug that produced
            // "multiple definition of `rsp`" linking the x86-64 vDSO: every
            // TU that includes the declaring header got its own colliding
            // non-static .bss tentative definition of the register's name.
            if (var->is_global_reg)
                continue;
            if (var->is_extern && !var->alias_target && !var->asm_name)
                continue;
            char *label = var->asm_name ? var->asm_name : var->name;
            if (var->is_function && !var->alias_target && !var->asm_name)
                continue;
            // Handle function aliases (__attribute__((alias)) or __asm__ renaming)
            if (var->is_function && !var->alias_target && var->asm_name) {
                // Skip if asm_name resolves to same symbol as C name (circular)
                if (strcmp(sym_name(var->name), var_sym_label(var)) == 0)
                    continue;
                // __asm__("target") on a function: alias the C name to the asm_name
                // The function body is emitted under asm_name; we need the C name to point to it.
                // Create a symbol alias by emitting both names at the same section offset.
                // The target symbol will be added when the function body is emitted below.
                continue;
            }
#ifdef _WIN32
            // Windows/MinGW uses emulated TLS via __emutls_get_address.
            if (var->is_tls) {
                if (!var->is_extern)
                    cg_emit_emutls_data(var);
                continue;
            }
#endif
            if (var->alias_target) {
                // __attribute__((alias("target")))
                // Defer alias resolution until after all symbols are emitted
                // Add as UNDEF for now; will be resolved later
                objfile_add_sym(cg_obj, asm_sym_name(sym_name(var->name)),
                                SEC_UNDEF, 0, 0, SB_GLOBAL, ST_NOTYPE);
                continue;
            }
            // If a global with this asm_name already emitted, skip (alias target)
            bool sym_already_emitted = false;
            const char *canon = asm_sym_name(var_sym_label(var));
            for (int i = 0; i < emitted_count; i++) {
                if (strcmp(emitted_syms[i], canon) == 0) {
                    sym_already_emitted = true;
                    break;
                }
            }
            if (sym_already_emitted)
                continue;
            if (var->asm_name) {
                // Check if asm_name matches another global that provides data
                // (with or without a leading underscore on the C name).
                LVar *existing = NULL;
                for (LVar *g = prog->globals; g; g = g->next) {
                    if (g != var && !g->is_extern && !g->is_function &&
                        (g->has_init || g->init_data)) {
                        if (strcmp(sym_name(g->name), var->asm_name) == 0 ||
                            (var->asm_name[0] == '_' &&
                             strcmp(sym_name(g->name), var->asm_name + 1) == 0)) {
                            existing = g;
                            break;
                        }
                    }
                }
                if (existing) {
                    // This is an alias via __asm__ — defer alias resolution
                    // until after all globals have been emitted.
                    continue;
                }
            }
            char **new_syms = realloc(emitted_syms, (emitted_count + 1) * sizeof(char *));
            if (!new_syms) {
                fprintf(stderr, "realloc failed\n");
                exit(1);
            }
            emitted_syms = new_syms;
            emitted_syms[emitted_count++] = (char *)canon;
            bool reserved = !var->asm_name && is_asm_reserved(var->name);
            char *safe_label = reserved ? format(".L_rcc_%s", var->name) : label;
            int is_bss = (!var->init_data && !var->relocs && !var->has_init) && !var->is_tls && !var->section_name;
            // __attribute__((section("name"))): place this global's data
            // in its own custom section instead of .data, so the linker's
            // __start_<name>/__stop_<name> boundary-symbol synthesis has
            // something to bracket. Always routed through the "has data"
            // path above (never BSS) even for a zero-initialized section()
            // global -- a named section commonly holds a single marker
            // object deliberately kept out of ordinary .bss/.data, and real
            // GNU ld's own PROGBITS-vs-NOBITS choice for it isn't something
            // this narrower feature needs to replicate.
            int data_sec = var->section_name
                ? objfile_find_or_add_section(cg_obj, var->section_name,
                                              SHF_ALLOC | (ty_const(var->ty) ? 0 : SHF_WRITE), 0)
                : (var->is_tls ? SEC_TDATA : SEC_DATA);
            // A custom section defaults to sh_addralign=1 (no constraint);
            // raise it to this global's own required alignment so the
            // linker places the section itself (and thus every object in
            // it, GC-tagged-pointer schemes especially) at an address
            // whose low bits stay zero -- see obj.h's ExtraSection::align.
            if (var->section_name)
                objfile_section_align(cg_obj, data_sec, var->ty->align > 1 ? (uint32_t)var->ty->align : 1);
            const char *sym_name_str = asm_sym_name(sym_name(safe_label)); // .balign %d
            if (is_bss) {
                size_t align = var->ty->align > 1 ? var->ty->align : 1;
                size_t rem = cg_obj->bss_size % align;
                if (rem) cg_obj->bss_size += align - rem;
                size_t off = cg_obj->bss_size;
                // A tentative (no-initializer) global whose declaration
                // carries __attribute__((weak)) needs STB_WEAK, not
                // STB_GLOBAL -- otherwise identical definitions across
                // multiple translation units (the exact idiom
                // `int x __attribute__((weak));` in a shared header,
                // e.g. Plan9/Go-toolchain's AUTOLIB() macro) collide as
                // real "multiple definition" link errors instead of the
                // silently-merged single definition weak linkage exists
                // for. Matches cg_weak_label()'s SB_WEAK choice for
                // functions; __attribute__((weak)) on a *static*
                // (internal-linkage) variable has no effect (GCC treats
                // it as a no-op too), so is_static still wins.
                SymBind bss_bind = var->is_static ? SB_LOCAL : (var->is_weak ? SB_WEAK : SB_GLOBAL);
                objfile_add_sym(cg_obj, sym_name_str, var->is_tls ? SEC_TDATA : SEC_BSS, off, var->ty->size, bss_bind, var->is_tls ? ST_TLS : ST_OBJECT);
                cg_label_ht_add(sym_name_str, off);
                if (reserved)
                    objfile_add_sym(cg_obj, asm_sym_name(sym_name(label)), var->is_tls ? SEC_TDATA : SEC_BSS, off, var->ty->size, bss_bind, var->is_tls ? ST_TLS : ST_OBJECT); // .globl %s
                cg_obj->bss_size += var->ty->size;
            } else {
                cg_set_section(data_sec);
                secbuf_align(cg_sec, var->ty->align > 1 ? var->ty->align : 1);
#ifdef __APPLE__
                // Apple ARM64: ensure at least 8-byte alignment for data symbols
                // (needed for proper relocation handling, fixes 119_random_stuff)
                secbuf_align(cg_sec, 8);
#endif
                size_t off = cg_sec->len;
#if defined(__APPLE__) && defined(ARCH_ARM64)
                if (var->is_tls) {
                    char *init_name = format("%s$tlv$init", label);
                    objfile_add_sym(cg_obj, init_name, SEC_TDATA, off, var->ty->size,
                                    SB_LOCAL, ST_OBJECT);
                    int tlv_boot = objfile_find_sym(cg_obj, "__tlv_bootstrap");
                    if (tlv_boot < 0)
                        tlv_boot = objfile_add_sym(cg_obj, "__tlv_bootstrap", SEC_UNDEF, 0, 0, SB_GLOBAL, ST_FUNC);
                    cg_set_section(SEC_THREAD_VARS);
                    secbuf_align(cg_sec, 8);
                    size_t desc_off = cg_sec->len;
                    secbuf_emit64le(cg_sec, 0);
                    objfile_add_reloc(cg_obj, SEC_THREAD_VARS, desc_off, tlv_boot, R_AARCH64_ABS64, 0);
                    secbuf_emit64le(cg_sec, 0);
                    int init_sidx = objfile_find_sym(cg_obj, init_name);
                    secbuf_emit64le(cg_sec, 0);
                    objfile_add_reloc(cg_obj, SEC_THREAD_VARS, desc_off + 16, init_sidx, R_AARCH64_ABS64, 0);
                    if (!var->is_static)
                        objfile_add_sym(cg_obj, sym_name_str, SEC_THREAD_VARS, desc_off, 24,
                                        SB_GLOBAL, ST_TLS);
                    else
                        objfile_add_sym(cg_obj, sym_name_str, SEC_THREAD_VARS, desc_off, 24,
                                        SB_LOCAL, ST_TLS);
                    cg_label_ht_add(sym_name_str, desc_off);
                    if (reserved)
                        objfile_add_sym(cg_obj, asm_sym_name(sym_name(label)), SEC_THREAD_VARS,
                                        desc_off, 24, var->is_static ? SB_LOCAL : SB_GLOBAL, ST_TLS);
                    cg_set_section(SEC_TDATA);
                } else
#endif
                {
                    // Same STB_WEAK rationale as the .bss branch above.
                    SymBind data_bind = var->is_static ? SB_LOCAL : (var->is_weak ? SB_WEAK : SB_GLOBAL);
                    objfile_add_sym(cg_obj, sym_name_str, data_sec, off, var->ty->size, data_bind, var->is_tls ? ST_TLS : ST_OBJECT);
                    cg_label_ht_add(sym_name_str, off);
                    if (reserved)
                        objfile_add_sym(cg_obj, asm_sym_name(sym_name(label)), data_sec, off, var->ty->size, data_bind, var->is_tls ? ST_TLS : ST_OBJECT); // %s:
                }
                if (var->is_tls && !var->has_init && !var->init_data && !var->relocs) {
                    for (int _zi = 0; _zi < var->ty->size; _zi++) secbuf_emit8(cg_sec, 0);
                }
                if (var->init_data || var->relocs) {
                    int pos = 0;
                    for (Reloc *rel = var->relocs; rel; rel = rel->next) {
                        for (; pos < rel->offset; pos++)
                            secbuf_emit8(cg_sec, (uint8_t)var->init_data[pos]); // .set %s, %s
                        size_t rel_off = cg_sec->len;
                        if (rel->label2) {
                            // Label-address DIFFERENCE: neither label's .text
                            // offset is known yet (see queue_label_diff_patch's
                            // doc comment above) — reserve placeholder bytes
                            // and defer the real patch to the resolution pass
                            // after this owning function's body has been
                            // generated (see the resolution snippet at the
                            // end of the function-body loop below).
                            for (int zi = 0; zi < rel->size; zi++)
                                secbuf_emit8(cg_sec, 0);
                            queue_label_diff_patch(data_sec, rel_off,
                                                   rel->label, rel->label2, var->decl_fn_name, rel->size);
                            pos += rel->size;
                            continue;
                        }
                        secbuf_emit64le(cg_sec, (uint64_t)rel->addend); // .quad addend (reloc addend embedded in data)
                        // A reloc label that exactly matches a declared
                        // symbol's __asm__ name is already the final linker
                        // name (the parser stored var->asm_name verbatim);
                        // re-applying sym_name() would double the platform
                        // prefix -- e.g. macOS __DARWIN_ALIAS'd `close`
                        // (declared `int close(int) __asm("_close")`) taken
                        // as a function pointer in a static table would emit
                        // "__close" and fault at load.
                        const char *rel_label = rel->label;
                        bool rel_is_asm = false;
                        for (LVar *g = prog->globals; g; g = g->next)
                            if (g->asm_name && strcmp(g->asm_name, rel->label) == 0) {
                                rel_is_asm = true;
                                break;
                            }
                        if (!rel_is_asm) rel_label = sym_name(rel->label);
                        int sidx;
                        // .L. labels are local text labels; use SB_LOCAL SEC_TEXT for section-relative reloc
                        bool is_local_label = (rel_label[0] == '.' && rel_label[1] == 'L' && rel_label[2] == '.');
                        sidx = objfile_find_sym(cg_obj, rel_label);
                        if (sidx < 0)
                            sidx = objfile_add_sym(cg_obj, rel_label,
                                                   is_local_label ? SEC_TEXT : SEC_UNDEF, 0, 0,
                                                   is_local_label ? SB_LOCAL : SB_GLOBAL, ST_NOTYPE);
#ifdef ARCH_ARM64
                        objfile_add_reloc(cg_obj, data_sec, rel_off, sidx, R_AARCH64_ABS64, (int64_t)rel->addend);
#else
                        objfile_add_reloc(cg_obj, data_sec, rel_off, sidx, R_X86_64_64, (int64_t)rel->addend);
#endif
                        pos += 8;
                    }
                    for (; pos < var->init_size; pos++)
                        secbuf_emit8(cg_sec, (uint8_t)var->init_data[pos]); // .quad %s%+d
                    if (var->ty->size > var->init_size) {
                        size_t pad = var->ty->size - var->init_size;
                        secbuf_reserve(cg_sec, pad);
                        memset(cg_sec->data + cg_sec->len, 0, pad);
                        cg_sec->len += pad;
                    }
                } else if (var->has_init) {
                    if (var->ty->size == 1)
                        secbuf_emit8(cg_sec, (uint8_t)var->init_val); // .quad %s
                    else if (var->ty->size == 2)
                        secbuf_emit16le(cg_sec, (uint16_t)var->init_val); // .quad %s%+d
                    else if (var->ty->size == 4)
                        secbuf_emit32le(cg_sec, (uint32_t)var->init_val); // .quad %s
                    else
                        secbuf_emit64le(cg_sec, (uint64_t)var->init_val); // .byte %u
                }
            }
        }
        // Resolve __asm__("name") aliases that refer to another global.
        // The matching is done with/without a leading underscore so that
        // e.g. `extern struct xx7 y __asm__("_z7")` aliases the C global `z7`
        // on targets where C symbols are prefixed with an underscore.
        for (LVar *var = prog->globals; var; var = var->next) {
            if (!var->is_extern || !var->asm_name)
                continue;
            LVar *existing = NULL;
            for (LVar *g = prog->globals; g; g = g->next) {
                if (g != var && !g->is_extern && !g->is_function &&
                    (g->has_init || g->init_data)) {
                    if (strcmp(sym_name(g->name), var->asm_name) == 0 ||
                        (var->asm_name[0] == '_' &&
                         strcmp(sym_name(g->name), var->asm_name + 1) == 0)) {
                        existing = g;
                        break;
                    }
                }
            }
            if (existing) {
                const char *target_name = asm_sym_name(sym_name(existing->name));
                const char *alias_name = asm_sym_name(var->asm_name);
                int tidx = objfile_find_sym(cg_obj, target_name);
                if (tidx >= 0 && cg_obj->syms[tidx].section != SEC_UNDEF) {
                    int aidx = objfile_find_sym(cg_obj, alias_name);
                    if (aidx < 0) {
                        objfile_add_sym(cg_obj, alias_name,
                                        cg_obj->syms[tidx].section,
                                        cg_obj->syms[tidx].offset,
                                        cg_obj->syms[tidx].size,
                                        cg_obj->syms[tidx].bind,
                                        cg_obj->syms[tidx].type);
                    } else if (cg_obj->syms[aidx].section == SEC_UNDEF) {
                        cg_obj->syms[aidx].section = cg_obj->syms[tidx].section;
                        cg_obj->syms[aidx].offset = cg_obj->syms[tidx].offset;
                        cg_obj->syms[aidx].size = cg_obj->syms[tidx].size;
                        cg_obj->syms[aidx].bind = cg_obj->syms[tidx].bind;
                        cg_obj->syms[aidx].type = cg_obj->syms[tidx].type;
                    }
                }
            }
        }
        free(emitted_syms);
        cg_set_section(SEC_RODATA);
        for (StrLit *s = prog->strs; s; s = s->next) {
            // Wide string literals (wchar_t[]/char16_t[]/char32_t[]) must
            // start on a boundary matching their element size: glibc's
            // wcslen()/wcscmp()/etc. are commonly vectorized (SSE2/AVX2),
            // reading 16+ bytes at a time under the assumption that any
            // wchar_t object -- an ordinary string literal included --
            // satisfies _Alignof(wchar_t) (4 on Linux/macOS, 2 on
            // Windows). Consecutive .rodata entries were packed back-to-
            // back with no padding, so a wide literal immediately
            // following a narrow one (or another wide literal whose byte
            // length isn't a multiple of elem_size) could land at any
            // byte offset. A misaligned literal fed to a vectorized
            // wcslen() silently returned the wrong length (confirmed: a
            // 12-wchar_t literal at a 1-mod-4 address read back as 11),
            // corrupting every one of libarchive's archive_mstring
            // wide-character setters built on it.
            if (s->prefix != 0 && s->prefix != '8')
                secbuf_align(cg_sec, s->elem_size);
            cg_def_label_sec(format(".LC%d", s->id), SEC_RODATA); // .zero %d
            if (s->prefix != 0 && s->prefix != '8') {
                // Wide string: decode UTF-8 and emit wide characters
                char *p = s->str;
                while (*p) {
                    char *next;
                    uint32_t c = decode_utf8(&next, p);
                    if (next == p) {
                        p++;
                        continue;
                    } // invalid UTF-8, skip
                    p = next;
                    if (s->elem_size == 2) {
                        // UTF-16: Windows wchar_t is UCS-2 (2 bytes).
                        // Code points above U+FFFF need a surrogate pair.
                        if (c > 0xFFFF) {
                            uint32_t sc = c - 0x10000;
                            secbuf_emit16le(cg_sec, (uint16_t)(0xD800 + (sc >> 10))); // .2byte %u
                            secbuf_emit16le(cg_sec, (uint16_t)(0xDC00 + (sc & 0x3FF))); // .2byte %u
                        } else {
                            secbuf_emit16le(cg_sec, (uint16_t)c); // .2byte %u
                        }
                    } else {
                        secbuf_emit32le(cg_sec, c); // .4byte %u
                    }
                }
                if (s->elem_size == 2)
                    secbuf_emit16le(cg_sec, 0); // .2byte 0
                else
                    secbuf_emit32le(cg_sec, 0); // .4byte 0
            } else {
                secbuf_emitbuf(cg_sec, s->str, s->len); // .byte %u
                secbuf_emit8(cg_sec, 0); // .byte 0
            }
        }
        cg_set_section(SEC_TEXT);
    }

    for (TLItem *item = prog->items; item; item = item->next) {
        if (item->kind == TL_ASM) {
            // File-scope asm(...) — no enclosing function, so there's no
            // later C-level goto label it could ever forward-reference
            // (on_forward is only relevant inside a function body).
            // Previously this had its own tiny hand-rolled parser that
            // only understood a single "label: insn" line with a
            // hardcoded handful of instructions (ret/nop/int3), silently
            // doing nothing for anything else — real uses like the
            // kernel's raw __define_initcall asm() (.section/label/
            // .long .../.previous, no C-callable single instruction in
            // sight) fell straight through that "for other instructions,
            // assembler needed" gap. Route through the same full
            // assembler function bodies already use instead.
            assemble_inline(cg_obj, item->asm_str, NULL, NULL);
            continue;
        }
        Function *fn = item->fn;
        bool fn_emitting_local_copy = false;
        // GNU89 extern inline (gnu_inline attribute): the body is an inline
        // definition only; the external definition lives elsewhere (glibc
        // _FORTIFY_SOURCE wrappers). GCC emits no symbol and relies on
        // inlining. rcc has no guaranteed inliner (vector-returning
        // functions, -O0), so compile the body as a LOCAL copy per TU:
        // intra-TU calls bind to it, no global symbol is exported (GCC's
        // actual promise), and multi-TU links never collide. The -O2
        // inliner replaces the calls when it can (the header functions are
        // single-return bodies), after which the local copy is dead.
        //
        // EXCEPTION: a fortify-style wrapper whose body calls its OWN name
        // (the fallback to the libc function, e.g. glibc's
        // `... ? abort() : memcpy(__dest, __src, __len)`). A local copy of
        // it would shadow that fallback call inside the same TU, recursing
        // until the stack overflows (the fallback MUST bind to the real
        // libc symbol). Skip the copy for those: call sites then bind to
        // libc, which is what the wrapper's fallback needs anyway.
        bool gnu_ext_inline = fn->is_inline && fn->is_extern && fn->is_gnu_inline;
        if (gnu_ext_inline) {
            if (fn_self_references(fn))
                continue; // fortify-style: bind call sites to libc
#ifdef _WIN32
            // mingw's SIMD headers use runtime `const int` operands in
            // several wrappers (_mm_round_pd(..., int M), _mm_blend_ps(
            // ..., int mask)); their local copies can't compile (the
            // intrinsics need a constant immediate) and, since the
            // unused-copy elimination is disabled on Windows, would be
            // compiled anyway. The always-inline at -O1 handles the real
            // call sites; skip the copies here entirely.
            continue;
#else
            fn_emitting_local_copy = true;
#endif
        }
        current_fn = fn->name;
        cg_dbg_fn = fn->name;
        current_fn_def = fn;
        current_fn_stack_size = fn->stack_size;
        memset(spill_slot, 0, sizeof(spill_slot));
        memset(spill_depth, 0, sizeof(spill_depth));
        next_spill_slot = 8;
#ifndef ARCH_ARM64
        init_spill_slots();
#endif
        spill_reserved_base = next_spill_slot; // 8 (ARM64) / 24 (x86-64)
        fn_struct_ret_off = 0;
        fn_struct_ret_total = 0;
        fn_trampoline_off = 0;
        fn_trampoline_total = 0;

        // Pass 1: Generate dummy function body to discover register usage
        SecBuf _dummy;
        secbuf_init(&_dummy);
        SecBuf *saved_sec = cg_sec;
        // codeql[cpp/stack-address-escape]: restored to saved_sec (line
        // ~12445 below) before this loop iteration ends, well before
        // _dummy (a local of codegen() itself, not this loop body) goes
        // out of scope.
        cg_sec = &_dummy;
        cg_dry_run = true;
        used_regs = 0;
        ever_used_regs = 0;
        spilled_regs = 0;
        spill_count = 0;
        memset(reg_owner, 0, sizeof(reg_owner));
        ctrl_depth = 0;
        fn_uses_alloca = false;
        last_debug_file = 0;
        last_debug_line = 0;

        // Save params to locals (emitted to body buffer, will be after prologue)
        // ARM64: handled in the Pass 2 prologue instead
#ifndef ARCH_ARM64
        int param_xmm_index = 0;
        int stack_param_index = 0;
        int param_index = fn->ty->return_ty &&
#ifdef _WIN32
                (((fn->ty->return_ty->kind == TY_STRUCT || fn->ty->return_ty->kind == TY_UNION) && fn->ty->return_ty->size > 8) || (is_complex(fn->ty->return_ty) && fn->ty->return_ty->size > 8))
#else
                (((fn->ty->return_ty->kind == TY_STRUCT || fn->ty->return_ty->kind == TY_UNION) && !struct_returns_in_gp_regs(fn->ty->return_ty)) || (is_complex(fn->ty->return_ty) && fn->ty->return_ty->size > 16))
#endif
            ? 1
            : 0;
#ifdef _WIN32
        int max_param_regs = 4;
        X86Reg cg_x86_paramreg[] = {X86_RCX, X86_RDX, X86_R8, X86_R9};
#else
        X86Reg cg_x86_paramreg[] = {X86_RDI, X86_RSI, X86_RDX, X86_RCX, X86_R8, X86_R9};
        int max_param_regs = 6;
        int max_param_xmm = 8;
#endif
        // Physical param register tables for prologue
        for (LVar *var = fn->params; var; var = var->param_next) {
#ifdef _WIN32
            if (is_flonum(var->ty)) {
                if (param_index < max_param_regs) {
                    if (var->ty->size == 4) {
                        // ABI: the caller (per the call-site fix above) now
                        // places a genuine narrow float in the low bits of
                        // the incoming xmm register; store it directly, no
                        // narrowing needed.
                        asm_movss_mr_rbp(cg_sec, param_index, var->offset); // movss xmm_i, -(off)(%rbp)
                    } else {
                        asm_movsd_xmm_rbp(cg_sec, param_index, var->offset); // movsd xmm_i, -(off)(%rbp)
                    }
                    param_index++;
                    param_xmm_index++;
                } else {
                    // Float on stack (arg position >= 4 in Win64)
                    int stack_off = 48 + stack_param_index * 8;
                    if (var->ty->size == 4) {
                        // ABI: the caller (per the call-site stack-arg fix
                        // above) now stores a genuine narrow float on the
                        // stack; load it directly, no narrowing needed.
                        asm_movss_rm_rbp(cg_sec, 0, stack_off); // movss stack_off(%rbp), xmm0
                        asm_movss_mr_rbp(cg_sec, 0, var->offset); // movss xmm0, -(off)(%rbp)
                    } else {
                        asm_movsd_rm_rbp(cg_sec, 0, stack_off); // movsd stack_off(%rbp), xmm0
                        asm_movsd_mr_rbp(cg_sec, 0, var->offset); // movsd xmm0, -(off)(%rbp)
                    }
                    stack_param_index++;
                }
            } else if (param_index < max_param_regs) {
                int psz = var->ty->size == 1 ? 1 : var->ty->size == 2 ? 2
                    : var->ty->size <= 4                              ? 4
                                                                      : 8;
                // Structs > 8 bytes (and 16-byte _Complex double) are passed by
                // pointer; copy the pointee to the local stack slot.
                if ((var->ty->kind == TY_STRUCT || var->ty->kind == TY_UNION || is_complex(var->ty)) && var->ty->size > 8) {
                    int c = ++rcc_label_count;
                    asm_mov_preg_r11(cg_sec, cg_x86_paramreg[param_index]); // movq preg, %r11
                    x86_mov_ri(cg_sec, 8, X86_R10, var->ty->size); // movq $size, %r10
                    cg_def_label(format(".L.pcopy.%d", c)); // .L.pcopy.%d:\n", c
                    x86_cmp_ri(cg_sec, 8, X86_R10, 0); // cmpq $0, %r10
                    size_t jze1 = asm_jcc_label(cg_sec, X86_E);
                    asm_fixup_add(cg_sec, jze1, format(".L.pcopy_end.%d", c), 0);
                    asm_movb_r11_r10_al(cg_sec, -1); // movb -1(%r11,%r10), %%al
                    asm_movb_al_rbp_r10(cg_sec, var->offset); // movb %%al, -(off)-1(%rbp,%r10) !!
                    x86_sub_ri(cg_sec, 8, X86_R10, 1); // subq $1, %r10
                    size_t jmp1 = asm_jmp_label(cg_sec);
                    asm_fixup_add(cg_sec, jmp1, format(".L.pcopy.%d", c), 0);
                    cg_def_label(format(".L.pcopy_end.%d", c));
                } else if (var->ty->kind == TY_INT128 || var->ty->kind == TY_DECIMAL128) {
                    // __int128/_Decimal128 passed by pointer on Windows x64.
                    // param_regs64[param_index] holds a pointer; dereference it.
                    x86_mov_rm(cg_sec, 8, X86_RAX, x86_mem(cg_x86_paramreg[param_index], 0)); // movq (preg), %rax
                    x86_mov_mr(cg_sec, 8, x86_mem(X86_RBP, -var->offset), X86_RAX); // movq %rax, -off(%rbp)
                    x86_mov_rm(cg_sec, 8, X86_RAX, x86_mem(cg_x86_paramreg[param_index], 8)); // movq 8(preg), %rax
                    x86_mov_mr(cg_sec, 8, x86_mem(X86_RBP, -(var->offset - 8)), X86_RAX); // movq %rax, -(off-8)(%rbp)
                } else if (var->ty->kind == TY_BITINT && var->ty->size > 8) {
                    // wide _BitInt (>8 bytes): passed by pointer; copy pointee.
                    // (65..128-bit bitints take the int128 pointer path above.)
                    int c = ++rcc_label_count;
                    asm_mov_preg_r11(cg_sec, cg_x86_paramreg[param_index]); // movq preg, %r11
                    x86_mov_ri(cg_sec, 8, X86_R10, var->ty->size); // movq $size, %r10
                    cg_def_label(format(".L.pcopy.%d", c));
                    x86_cmp_ri(cg_sec, 8, X86_R10, 0); // cmpq $0, %r10
                    size_t jz = asm_jcc_label(cg_sec, X86_E);
                    asm_fixup_add(cg_sec, jz, format(".L.pcopy_end.%d", c), 0);
                    asm_movb_r11_r10_al(cg_sec, -1); // movb -1(%r11,%r10), %%al
                    asm_movb_al_rbp_r10(cg_sec, var->offset); // movb %%al, -(off)-1(%rbp,%r10)
                    x86_sub_ri(cg_sec, 8, X86_R10, 1); // subq $1, %r10
                    size_t jm = asm_jmp_label(cg_sec);
                    asm_fixup_add(cg_sec, jm, format(".L.pcopy.%d", c), 0);
                    cg_def_label(format(".L.pcopy_end.%d", c));
                } else {
                    asm_mov_rbp(cg_sec, cg_x86_paramreg[param_index], psz, var->offset); // mov preg, -(off)(%rbp)
                }
                param_index++;
            } else {
                // Stack argument on Windows (shadow space = 32 bytes)
                int stack_off = 48 + stack_param_index * 8;
                if (is_flonum(var->ty)) {
                    if (var->ty->size == 4) {
                        // ABI: the caller (per the call-site stack-arg fix
                        // above) now stores a genuine narrow float on the
                        // stack; load it directly, no narrowing needed.
                        asm_movss_rm_rbp(cg_sec, 0, stack_off); // movss stack_off(%rbp), xmm0
                        asm_movss_mr_rbp(cg_sec, 0, var->offset); // movss xmm0, -(off)(%rbp)
                    } else {
                        asm_movsd_rm_rbp(cg_sec, 0, stack_off); // movsd stack_off(%rbp), xmm0
                        asm_movsd_mr_rbp(cg_sec, 0, var->offset); // movsd xmm0, -(off)(%rbp)
                    }
                } else if (var->ty->kind == TY_INT128 || var->ty->kind == TY_DECIMAL128) {
                    // __int128/_Decimal128 on stack: slot holds a pointer.
                    x86_mov_rm(cg_sec, 8, X86_RAX, x86_mem(X86_RBP, stack_off)); // movq stack_off(%rbp), %rax
                    x86_mov_rm(cg_sec, 8, X86_RCX, x86_mem(X86_RAX, 0)); // movq (%rax), %rcx  (lo)
                    asm_mov_phyreg_rbp(cg_sec, X86_RCX, 8, var->offset);
                    x86_mov_rm(cg_sec, 8, X86_RCX, x86_mem(X86_RAX, 8)); // movq 8(%rax), %rcx  (hi)
                    asm_mov_phyreg_rbp(cg_sec, X86_RCX, 8, var->offset - 8);
                } else if ((var->ty->kind == TY_STRUCT || var->ty->kind == TY_UNION || is_complex(var->ty) || var->ty->kind == TY_BITINT) && var->ty->size > 8) {
                    // Structs > 8 bytes (and 16-byte _Complex double) are passed
                    // by pointer even on the stack; load the pointer and copy
                    // the pointee into the local slot.
                    int c = ++rcc_label_count;
                    x86_mov_rm(cg_sec, 8, X86_R11, x86_mem(X86_RBP, stack_off)); // movq stack_off(%rbp), %r11
                    x86_mov_ri(cg_sec, 8, X86_R10, var->ty->size); // movq $size, %r10
                    cg_def_label(format(".L.pcopy.%d", c));
                    x86_cmp_ri(cg_sec, 8, X86_R10, 0); // cmpq $0, %r10
                    size_t jze3 = asm_jcc_label(cg_sec, X86_E);
                    asm_fixup_add(cg_sec, jze3, format(".L.pcopy_end.%d", c), 0);
                    asm_movb_r11_r10_al(cg_sec, -1); // movb -1(%r11,%r10), %%al
                    asm_movb_al_rbp_r10(cg_sec, var->offset); // movb %%al, -(off)-1(%rbp,%r10)
                    x86_sub_ri(cg_sec, 8, X86_R10, 1); // subq $1, %r10
                    size_t jmp3 = asm_jmp_label(cg_sec);
                    asm_fixup_add(cg_sec, jmp3, format(".L.pcopy.%d", c), 0);
                    cg_def_label(format(".L.pcopy_end.%d", c));
                } else {
                    int psz = var->ty->size == 1 ? 1 : var->ty->size == 2 ? 2
                        : var->ty->size <= 4                              ? 4
                                                                          : 8;
                    // tmpreg = sized %rax
                    asm_mov_rbp_tmpreg(cg_sec, stack_off, psz); // mov stack_off(%rbp), tmpreg
                    asm_mov_tmpreg_rbp(cg_sec, var->offset, psz); // mov tempreg, -(off)(%rbp)
                }
                stack_param_index++;
            }
#else
            if (var->ty->kind == TY_LDOUBLE) {
                // SysV x86-64 ABI: `long double` (80-bit x87 extended)
                // is ALWAYS classified MEMORY -- passed on the stack in
                // a 16-byte-aligned, 2-eightbyte slot, regardless of how
                // many XMM registers are still free (unlike float/
                // double, it never occupies an XMM register at all).
                // Mirrors the call-site classification fix in
                // gen_funcall's argument-passing loop; without this,
                // the generic is_flonum branch below read an XMM
                // parameter register no caller ever populates for a
                // long double argument, reading garbage. `fldt` loads
                // the genuine 80-bit value the caller wrote to the
                // stack; `fstpl` narrows and pops it into the local
                // slot as a double, matching rcc's own internal
                // GP-register/memory representation for long double
                // values everywhere else (see the ND_ASSIGN long
                // double store path's "Store long double as 64-bit
                // double (truncated)").
                if (stack_param_index & 1)
                    stack_param_index++;
                int stack_off2 = 16 + stack_param_index * 8;
                X86Mem ld_src = {CG_X86_FP, X86_NOREG, 1, -stack_off2};
                X86Mem ld_dst = {CG_X86_FP, X86_NOREG, 1, -var->offset};
                x86_fldt_m(cg_sec, ld_src); // fldt off2(%rbp)
                x86_fstpl_m(cg_sec, ld_dst); // fstpl -(off)(%rbp)
                stack_param_index += 2;
            } else if (is_flonum(var->ty)) {
                if (param_xmm_index < max_param_xmm) {
                    if (var->ty->size == 4) {
                        // ABI: the caller (per the call-site fix above) now
                        // places a genuine narrow float in the low bits of
                        // the incoming xmm register; store it directly, no
                        // narrowing needed.
                        asm_movss_mr_rbp(cg_sec, param_xmm_index, var->offset); // movss xmm_i, -(off)(%rbp)
                    } else {
                        asm_movsd_xmm_rbp(cg_sec, param_xmm_index, var->offset); // movsd xmm_i, -(off)(%rbp)
                    }
                    param_xmm_index++;
                } else {
                    int stack_off2 = 16 + stack_param_index * 8;
                    if (var->ty->size == 4) {
                        asm_movss_rm_rbp(cg_sec, 0, stack_off2); // movss off2(%rbp), xmm0
                        asm_movss_mr_rbp(cg_sec, 0, var->offset); // movss xmm0, -(off2)(%rbp)
                    } else {
                        asm_movsd_rm_rbp(cg_sec, 0, stack_off2); // movsd off2(%rbp), xmm0
                        asm_movsd_mr_rbp(cg_sec, 0, var->offset); // movsd xmm0, -(off2)(%rbp)
                    }
                    stack_param_index++;
                }
            } else if (var->ty->kind == TY_INT128 || var->ty->kind == TY_DECIMAL128 ||
                       (var->ty->kind == TY_BITINT && var->ty->size == 16)) {
#ifdef _WIN32
                // Windows x64: __int128 is passed by pointer in a single GP register.
                // Dereference the pointer to copy the 16-byte value to the local slot.
                if (param_index < max_param_regs) {
                    x86_mov_rm(cg_sec, 8, X86_RAX, x86_mem(cg_x86_paramreg[param_index], 0)); // movq (preg), %rax  (lo)
                    asm_mov_phyreg_rbp(cg_sec, X86_RAX, 8, var->offset);
                    x86_mov_rm(cg_sec, 8, X86_RAX, x86_mem(cg_x86_paramreg[param_index], 8)); // movq 8(preg), %rax  (hi)
                    asm_mov_phyreg_rbp(cg_sec, X86_RAX, 8, var->offset - 8);
                    param_index++;
                } else {
                    // Stack: pointer is in the shadow-space-extended stack slot.
                    int stack_off = 48 + stack_param_index * 8;
                    x86_mov_rm(cg_sec, 8, X86_RAX, x86_mem(X86_RBP, stack_off)); // movq stack_off(%rbp), %rax
                    x86_mov_rm(cg_sec, 8, X86_RCX, x86_mem(X86_RAX, 0)); // movq (%rax), %rcx  (lo)
                    asm_mov_phyreg_rbp(cg_sec, X86_RCX, 8, var->offset);
                    x86_mov_rm(cg_sec, 8, X86_RCX, x86_mem(X86_RAX, 8)); // movq 8(%rax), %rcx  (hi)
                    asm_mov_phyreg_rbp(cg_sec, X86_RCX, 8, var->offset - 8);
                    stack_param_index++;
                }
#else
                // x86-64 SysV: int128 lo in param_regs64[N], hi in param_regs64[N+1]
                if (param_index + 1 < max_param_regs) {
                    x86_mov_mr(cg_sec, 8, x86_mem(X86_RBP, -var->offset), cg_x86_paramreg[param_index]); // movq param, -off(%rbp)
                    x86_mov_mr(cg_sec, 8, x86_mem(X86_RBP, -(var->offset - 8)), cg_x86_paramreg[param_index + 1]); // movq param+1, -(off-8)(%rbp)
                    param_index += 2;
                } else {
                    // On stack: two consecutive 8-byte slots at [rbp + 16 + k*8]
                    int stack_off = 16 + stack_param_index * 8;
                    x86_mov_rm(cg_sec, 8, X86_RAX, x86_mem(X86_RBP, stack_off)); // movq stack_off(%rbp), %rax
                    x86_mov_mr(cg_sec, 8, x86_mem(X86_RBP, -var->offset), X86_RAX); // movq %rax, -off(%rbp)
                    x86_mov_rm(cg_sec, 8, X86_RAX, x86_mem(X86_RBP, stack_off + 8)); // movq stack_off+8(%rbp), %rax
                    x86_mov_mr(cg_sec, 8, x86_mem(X86_RBP, -(var->offset - 8)), X86_RAX); // movq %rax, -(off-8)(%rbp)
                    stack_param_index += 2;
                }
#endif
            } else if (is_complex(var->ty)) {
                // Complex types: {real, imag} like struct of two base-type elements
                // _Complex float  (8 bytes):  one SSE eightbyte   → xmm0
                // _Complex double (16 bytes): two SSE eightbytes  → xmm0, xmm1
                // _Complex int    (8 bytes):  one INTEGER eightbyte → GP reg
                int base_sz = var->ty->base ? var->ty->base->size : 4;
#ifdef _WIN32
                // Win64: an 8-byte _Complex (float or int base) is passed by
                // value in a single GP register, like any other 8-byte
                // aggregate — XMM is only used for native float/double
                // scalars, not for _Complex.
                bool win_small_complex = var->ty->size <= 8;
#else
                bool win_small_complex = false;
#endif
                if (is_flonum(var->ty->base) && !win_small_complex) {
                    // Float complex: pass in xmm registers
                    if (var->ty->size <= 8) {
                        // _Complex float: 8 bytes in one xmm reg
                        if (param_xmm_index < max_param_xmm) {
                            asm_movsd_xmm_rbp(cg_sec, param_xmm_index, var->offset); // movsd xmm_i, -off(%rbp)
                            param_xmm_index++;
                        } else {
                            asm_movsd_rm_rbp(cg_sec, 0, 16 + stack_param_index * 8); // movsd stack_off(%rbp), %xmm0
                            asm_movsd_mr_rbp(cg_sec, 0, var->offset); // movsd %xmm0, -off(%rbp)
                            stack_param_index++;
                        }
                    } else {
                        // _Complex double: 16 bytes, two xmm regs
                        if (param_xmm_index + 1 < max_param_xmm) {
                            asm_movsd_xmm_rbp(cg_sec, param_xmm_index, var->offset); // movsd xmm_i, -off(%rbp)
                            asm_movsd_xmm_rbp(cg_sec, param_xmm_index + 1, var->offset - base_sz); // movsd xmm_i+1, -(off-base_sz)(%rbp)
                            param_xmm_index += 2;
                        } else {
                            int stack_off = 16 + stack_param_index * 8;
                            asm_movsd_rm_rbp(cg_sec, 0, stack_off); // movsd stack_off(%rbp), %xmm0
                            asm_movsd_mr_rbp(cg_sec, 0, var->offset); // movsd %xmm0, -off(%rbp)
                            asm_movsd_rm_rbp(cg_sec, 0, stack_off + 8); // movsd stack_off+8(%rbp), %xmm0
                            asm_movsd_mr_rbp(cg_sec, 0, var->offset - base_sz); // movsd %xmm0, -(off-base_sz)(%rbp)
                            stack_param_index += 2;
                        }
                    }
                } else {
                    // Integer complex: pass in GP registers
                    if (var->ty->size <= 8) {
                        // _Complex int: 8 bytes in one GP reg
                        if (param_index < max_param_regs) {
                            asm_mov_rbp(cg_sec, cg_x86_paramreg[param_index], 8, var->offset); // movq preg, -off(%rbp)
                            param_index++;
                        } else {
                            x86_mov_rm(cg_sec, 8, X86_RAX, x86_mem(X86_RBP, 16 + stack_param_index * 8)); // movq stack_off(%rbp), %rax
                            x86_mov_mr(cg_sec, 8, x86_mem(X86_RBP, -var->offset), X86_RAX); // movq %rax, -off(%rbp)
                            stack_param_index++;
                        }
                    } else {
                        // _Complex long long: 16 bytes in two GP regs
                        if (param_index + 1 < max_param_regs) {
                            asm_mov_rbp(cg_sec, cg_x86_paramreg[param_index], 8, var->offset); // movq preg, -off(%rbp)
                            asm_mov_rbp(cg_sec, cg_x86_paramreg[param_index + 1], 8, var->offset - base_sz); // movq preg+1, -(off-base_sz)(%rbp)
                            param_index += 2;
                        } else {
                            int stack_off = 16 + stack_param_index * 8;
                            x86_mov_rm(cg_sec, 8, X86_RAX, x86_mem(X86_RBP, stack_off)); // movq stack_off(%rbp), %rax
                            x86_mov_mr(cg_sec, 8, x86_mem(X86_RBP, -var->offset), X86_RAX); // movq %rax, -off(%rbp)
                            x86_mov_rm(cg_sec, 8, X86_RAX, x86_mem(X86_RBP, stack_off + 8)); // movq stack_off+8(%rbp), %rax
                            x86_mov_mr(cg_sec, 8, x86_mem(X86_RBP, -(var->offset - base_sz)), X86_RAX); // movq %rax, -(off-base_sz)(%rbp)
                            stack_param_index += 2;
                        }
                    }
                }
            } else if (param_index < max_param_regs) {
                int psz = var->ty->size == 1 ? 1 : var->ty->size == 2 ? 2
                    : var->ty->size <= 4                              ? 4
                                                                      : 8;
                if (((var->ty->kind == TY_STRUCT || var->ty->kind == TY_UNION) && var->ty->size > 8) ||
                    (var->ty->kind == TY_BITINT && var->ty->size > 16)) {
                    int c = ++rcc_label_count;
                    asm_mov_preg_r11(cg_sec, cg_x86_paramreg[param_index]); // movq preg, %r11
                    x86_mov_ri(cg_sec, 8, X86_R10, var->ty->size); // movq $size, %r10
                    cg_def_label(format(".L.pcopy.%d", c));
                    x86_cmp_ri(cg_sec, 8, X86_R10, 0); // cmpq $0, %r10
                    size_t jze2 = asm_jcc_label(cg_sec, X86_E);
                    asm_fixup_add(cg_sec, jze2, format(".L.pcopy_end.%d", c), 0);
                    asm_movb_r11_r10_al(cg_sec, -1); // movb -1(%r11,%r10), %%al
                    asm_movb_al_rbp_r10(cg_sec, var->offset); // movb %%al, -(off)-1(%rbp,%r10)
                    x86_sub_ri(cg_sec, 8, X86_R10, 1); // subq $1, %r10
                    size_t jmp2 = asm_jmp_label(cg_sec);
                    asm_fixup_add(cg_sec, jmp2, format(".L.pcopy.%d", c), 0);
                    cg_def_label(format(".L.pcopy_end.%d", c));
                } else {
                    asm_mov_rbp(cg_sec, cg_x86_paramreg[param_index], psz, var->offset); // mov preg, -(off)(%rbp)
                }
                param_index++;
            } else {
                if ((var->ty->kind == TY_STRUCT || var->ty->kind == TY_UNION) && var->ty->size > 8) {
                    int c = ++rcc_label_count;
                    int stack_off2 = 16 + stack_param_index * 8;
                    x86_mov_ri(cg_sec, 8, X86_R10, var->ty->size); // movq $size, %r10
                    cg_def_label(format(".L.param.%d", c));
                    x86_cmp_ri(cg_sec, 8, X86_R10, 0); // cmpq $0, %r10
                    size_t jze3 = asm_jcc_label(cg_sec, X86_E);
                    asm_fixup_add(cg_sec, jze3, format(".L.param_end.%d", c), 0);
                    asm_movb_rbp_r10_al(cg_sec, stack_off2); // movb stack_off-1(%rbp,%r10), %%al
                    asm_movb_al_rbp_r10(cg_sec, var->offset); // movb %%al, -(off)-1(%rbp,%r10)
                    x86_sub_ri(cg_sec, 8, X86_R10, 1); // subq $1, %r10
                    size_t jmp3 = asm_jmp_label(cg_sec);
                    asm_fixup_add(cg_sec, jmp3, format(".L.param.%d", c), 0);
                    cg_def_label(format(".L.param_end.%d", c));
                } else {
                    int stack_off2 = 16 + stack_param_index * 8;
                    int psz = var->ty->size == 1 ? 1 : var->ty->size == 2 ? 2
                        : var->ty->size <= 4                              ? 4
                                                                          : 8;
                    asm_mov_rbp_tmpreg(cg_sec, stack_off2, psz); // mov stack_off(%rbp), %rax
                    asm_mov_tmpreg_rbp(cg_sec, var->offset, psz); // mov %rax, -(off)(%rbp)
                }
                stack_param_index++;
            }
#endif
        }
#endif /* !ARCH_ARM64 */

#ifndef ARCH_ARM64
        // Save arg registers if function is variadic
        if (fn->is_variadic) {
            int gp_count = param_index;
            int va_fp = param_xmm_index;
#ifdef _WIN32
            // Windows x64 ABI: 4 GP regs (rcx,rdx,r8,r9), 4 XMM regs (xmm0-3)
            va_reg_save_ofs = current_fn_stack_size + 96;
            va_gp_start = gp_count * 8;
            va_fp_start = va_fp * 16 + 32;
            va_st_start = 48 + stack_param_index * 8;
            // codeql[cpp/commented-out-code]: first-pass formula, superseded by the stack_param_index fix noted near pass 2 below
            // va_st_start = 48 + (gp_count > 4 ? (gp_count - 4) : 0) * 8;
            switch (gp_count) {
            case 0: asm_mov_phyreg_rbp(cg_sec, X86_RCX, 8, va_reg_save_ofs); /* fallthrough */ /* movq %rcx, -%d(%rbp) */
            case 1: asm_mov_phyreg_rbp(cg_sec, X86_RDX, 8, va_reg_save_ofs - 8); /* fallthrough */ /* movq %rdx, -%d(%rbp) */
            case 2: asm_mov_phyreg_rbp(cg_sec, X86_R8, 8, va_reg_save_ofs - 16); /* fallthrough */ /* movq %r8, -%d(%rbp) */
            case 3: asm_mov_phyreg_rbp(cg_sec, X86_R9, 8, va_reg_save_ofs - 24); /* movq %r9, -%d(%rbp) */
            }
            // Save all 4 xmm regs unconditionally (caller puts FP in both GP and XMM)
            asm_movaps_rbp_xmm(cg_sec, 0, va_reg_save_ofs - 32); /* movaps %xmm0, -%d(%rbp) */
            asm_movaps_rbp_xmm(cg_sec, 1, va_reg_save_ofs - 48); /* movaps %xmm1, -%d(%rbp) */
            asm_movaps_rbp_xmm(cg_sec, 2, va_reg_save_ofs - 64); /* movaps %xmm2, -%d(%rbp) */
            asm_movaps_rbp_xmm(cg_sec, 3, va_reg_save_ofs - 80); /* movaps %xmm3, -%d(%rbp) */
#else
            va_reg_save_ofs = current_fn_stack_size + 176;
            va_gp_start = gp_count * 8;
            va_fp_start = va_fp * 16 + 48;
            va_st_start = 16 + stack_param_index * 8;

            switch (gp_count) {
            case 0: asm_mov_phyreg_rbp(cg_sec, X86_RDI, 8, va_reg_save_ofs); /* fallthrough */ /* movq %rdi, -%d(%rbp) */
            case 1: asm_mov_phyreg_rbp(cg_sec, X86_RSI, 8, va_reg_save_ofs - 8); /* fallthrough */ /* movq %rsi, -%d(%rbp) */
            case 2: asm_mov_phyreg_rbp(cg_sec, X86_RDX, 8, va_reg_save_ofs - 16); /* fallthrough */ /* movq %rdx, -%d(%rbp) */
            case 3: asm_mov_phyreg_rbp(cg_sec, X86_RCX, 8, va_reg_save_ofs - 24); /* fallthrough */ /* movq %rcx, -%d(%rbp) */
            case 4: asm_mov_phyreg_rbp(cg_sec, X86_R8, 8, va_reg_save_ofs - 32); /* fallthrough */ /* movq %r8, -%d(%rbp) */
            case 5: asm_mov_phyreg_rbp(cg_sec, X86_R9, 8, va_reg_save_ofs - 40); /* movq %r9, -%d(%rbp) */
            }
            if (va_fp < 8) {
                x86_test_rr(cg_sec, 1, X86_RAX, X86_RAX); // testb %%al, %%al
                {
                    size_t o = asm_jcc_label(cg_sec, X86_E); // jcc label
                    asm_fixup_add(cg_sec, o, format(".L.x%d", rcc_label_count), 1);
                }
                switch (va_fp) {
                case 0: asm_movaps_rbp_xmm(cg_sec, 0, va_reg_save_ofs - 48); /* fallthrough */ /* movaps %xmm0, -%d(%rbp) */
                case 1: asm_movaps_rbp_xmm(cg_sec, 1, va_reg_save_ofs - 64); /* fallthrough */ /* movaps %xmm1, -%d(%rbp) */
                case 2: asm_movaps_rbp_xmm(cg_sec, 2, va_reg_save_ofs - 80); /* fallthrough */ /* movaps %xmm2, -%d(%rbp) */
                case 3: asm_movaps_rbp_xmm(cg_sec, 3, va_reg_save_ofs - 96); /* fallthrough */ /* movaps %xmm3, -%d(%rbp) */
                case 4: asm_movaps_rbp_xmm(cg_sec, 4, va_reg_save_ofs - 112); /* fallthrough */ /* movaps %xmm4, -%d(%rbp) */
                case 5: asm_movaps_rbp_xmm(cg_sec, 5, va_reg_save_ofs - 128); /* fallthrough */ /* movaps %xmm5, -%d(%rbp) */
                case 6: asm_movaps_rbp_xmm(cg_sec, 6, va_reg_save_ofs - 144); /* fallthrough */ /* movaps %xmm6, -%d(%rbp) */
                case 7: asm_movaps_rbp_xmm(cg_sec, 7, va_reg_save_ofs - 160); /* movaps %xmm7, -%d(%rbp) */
                }
                cg_def_label(format(".L.x%d", rcc_label_count++)); // .L.x%d:
            }
#endif
        }
#else
        // Compute va_list init values for ARM64 (register saves are in Pass 2)
        if (fn->is_variadic) {
            int gp_param = 0;
            int fp_param = 0;
            int stack_param = 0;
            for (LVar *var = fn->params; var; var = var->param_next) {
                if (is_flonum(var->ty)) {
                    if (fp_param < 8) fp_param++;
                    else
                        stack_param++;
                } else if (gp_param < 8) {
                    gp_param++;
                } else {
                    stack_param++;
                }
            }
#ifdef __APPLE__
            // Apple ARM64: unnamed variadic args go on the stack, not in registers.
            // Set va offsets past the register save area so va_arg immediately
            // reads from the overflow (stack) area.
            va_gp_start = 64;
            va_fp_start = 192;
#else
            // AArch64 ABI: gr_offs = -(8 - gp_param) * 8, vr_offs = -(8 - fp_param) * 16
            va_gp_start = -(8 - gp_param) * 8;
            va_fp_start = -(8 - fp_param) * 16;
#endif
            va_st_start = 16 + stack_param * 8;
        }
#endif

        for (Node *n = fn->body; n; n = n->next) {
            VReg r = gen(n);
            if (r != -1) free_reg(r);
        }
        // Pass 2: Emit binary prologue, body, epilogue
        cg_sec = saved_sec;
        secbuf_free(&_dummy);
        cg_dry_run = false;
        used_regs = 0;
        spilled_regs = 0;
        spill_count = 0;
        memset(reg_owner, 0, sizeof(reg_owner));
        fn_struct_ret_off = 0; // reset for Pass 2 (fn_struct_ret_total already computed)
        fn_trampoline_off = 0; // reset for Pass 2 (fn_trampoline_total already computed)
        cg_label_ht_reset();
        asm_fixup_ht_reset();

#ifdef ARCH_ARM64
        // === ARM64 prologue ===
        // Callee-saved: x19-x28 are callee-saved among our 12 allocatable regs
        // x19-x24 = indices 6-11 in our reg arrays
        // If the function has cleanup vars and returns non-void, we use x19 to
        // preserve the return value across cleanup calls; mark x19 as used now
        // so the prologue saves/restores it correctly.
        bool fn_ret_nonvoid = fn->ty->return_ty && fn->ty->return_ty->kind != TY_VOID;
        bool fn_ret_gp_pair = fn->ty->return_ty && struct_returns_in_gp_regs(fn->ty->return_ty) && fn->ty->return_ty->size > 8;
        {
            for (LVar *var = fn->locals; var; var = var->next) {
                if (var_has_cleanup(var) && fn_ret_nonvoid) {
                    ever_used_regs |= (1 << 6); // x19 will be used in cleanup section
                    if (fn_ret_gp_pair)
                        ever_used_regs |= (1 << 7); // x20 preserves x1 (2nd eightbyte) across cleanup calls too
                    break;
                }
            }
        }
        int callee_mask = ((ever_used_regs >> 6) & 63);
        int n_callee_saved = 0;
        for (int j = 0; j < 6; j++)
            if (callee_mask & (1 << j)) n_callee_saved++;

        int need = fn->stack_size + fn_struct_ret_total + fn_trampoline_total + 32;
        // Spill slots grew from spill_reserved_base during Pass 1,
        // independent of `need`'s regions -- can overrun into locals under
        // heavy spill pressure. Re-anchor the excess on top of `need`;
        // Pass 2 replays the same push_spill_slot() sequence.
        int spill_bytes = next_spill_slot - spill_reserved_base;
        if (spill_bytes < 0) spill_bytes = 0;
        next_spill_slot = need;
        need += spill_bytes;
        int va_save_size = 0;
#ifndef __APPLE__
        // AAPCS64 (Linux): save all GP and FP arg regs for variadic functions
        if (fn->is_variadic) va_save_size = 192;
#endif
        // AAPCS64 passes `long double` (128-bit binary128) as a genuine
        // quad in a V register, but rcc's internal `long double` is a
        // plain 64-bit double everywhere else - incoming long double
        // params must be narrowed via libgcc's __trunctfdf2. That `bl`
        // clobbers every caller-saved register (x0-x17, v0-v7), so any
        // other not-yet-saved param sharing those registers would be
        // corrupted; reserve a small stash area to park each incoming
        // long double's raw quad until every other param is safely saved,
        // then narrow them one at a time from the stash (see below).
        int n_ldouble_params = 0;
        for (LVar *var = fn->params; var; var = var->param_next)
            if (var->ty->kind == TY_LDOUBLE && var->ty->size > 8) n_ldouble_params++;
        int ldouble_stash_size = n_ldouble_params * 16;
        int ldouble_stash_off = va_save_size;
        if (va_save_size > 0)
            need += va_save_size;
        if (ldouble_stash_size > 0)
            need += ldouble_stash_size;
        int frame_size = need + 16 + n_callee_saved * 8;
        // Round up to 16-byte alignment
        frame_size = (frame_size + 15) & ~15;
        // va_reg_save_ofs not used on ARM64; reg_save_area is stored as sp in va_start

        // Symbol linkage
        bool has_noninline_decl = false;
        bool had_extern_decl = false;
        if (fn->is_inline && !fn->is_extern) {
            for (LVar *g = prog->globals; g; g = g->next) {
                if (g->is_function && g->name == fn->name) {
                    if (g->has_init) has_noninline_decl = true;
                    if (g->is_extern && !g->is_weak) had_extern_decl = true;
                    break;
                }
            }
        }
        char *fn_label = fn->name;
        if (is_asm_reserved(fn->name))
            fn_label = format(".L_rcc_%s", fn->name);
        // C99 inline semantics: a plain non-static, non-extern `inline`
        // function has no external-linkage definition unless some other
        // declaration in this TU forces one (has_noninline_decl/
        // had_extern_decl). GNU89 inline semantics (real GCC's actual
        // behavior, and what `__attribute__((gnu_inline))` explicitly
        // opts into -- glibc/gperf's common `#ifdef __GNUC_STDC_INLINE__
        // __attribute__((gnu_inline)) #endif` portability idiom) differ:
        // a plain (non-extern) gnu_inline function DOES get an ordinary
        // global-linkage definition -- only `extern inline` + gnu_inline
        // (the declaration-only case, already special-cased earlier to
        // skip body emission entirely) suppresses it. Missing this made
        // e.g. gperf-generated hoedown_find_block_tag() in
        // hoedown/src/html_blocks.c compile fine per-TU but emit as a
        // local (non-.globl) symbol, leaving the final executable link
        // with an unresolved reference despite every object file
        // individually containing a full, correct definition.
        bool fn_exported = !fn->is_static && !fn_emitting_local_copy &&
            (!fn->is_inline || fn->is_extern || has_noninline_decl || had_extern_decl || fn->is_gnu_inline);
        // A mangled asm_name (GNU nested function, or __asm__("name")
        // renaming) is already the exact final symbol text — used as-is,
        // matching how call sites resolve it (gen_funcall, codegen.c
        // ~line 1054) and how sym_name_for()/asm_sym_name_for() (above)
        // already treat asm_name for other purposes.
        const char *fn_sym_name = fn->asm_name ? fn->asm_name : asm_sym_name(sym_name(fn_label));
        size_t fn_sym_start = cg_sec->len;
        if (fn->is_weak) {
            cg_weak_label(fn_sym_name); // .weak_definition %s
        } else if (fn_exported)
            cg_global_label(fn_sym_name); // .globl %s
        else
            cg_def_fn_label(fn_sym_name); // .weak %s

        // Stack frame: stp fp,lr; mov fp,sp; sub sp,sp,#frame_size
        asm_stp_fp_lr(cg_sec); // stp x29, x30, [sp, #-16]!
        asm_mov_fp_sp(cg_sec); // mov x29, sp
        if (frame_size <= 4095)
            arm64_sub_imm(cg_sec, 1, ARM64_SP, ARM64_SP, frame_size, 0); // sub sp, sp, #frame_size
        else {
            emit_mov_imm64(ARM64_X16, (uint64_t)frame_size); // mov x16, #frame_size
            arm64_sub_extreg(cg_sec, 1, ARM64_SP, ARM64_SP, ARM64_X16, ARM64_UXTX, 0); // sub sp, sp, x16
        }

        // GNU nested function: spill the incoming static-chain pointer
        // (physical x18, GCC's AArch64 STATIC_CHAIN_REGNUM) to its fixed
        // frame slot immediately, before any other codegen can clobber
        // x18 — mirrors the x86-64 prologue's identical block.
        if (fn->is_nested)
            arm64_stur(cg_sec, 1, ARM64_X18, ARM64_X29, -CHAIN_SLOT_OFFSET);

        // Nonlocal goto target support: a function whose __label__ is
        // actually reached by a nested descendant's nonlocal goto (see
        // is_goto_target_fn, set during parsing) records ITS OWN
        // just-established, stable sp into its own CHAIN_RSP_OFFSET slot
        // - see that comment in rcc.h for why this must be self-recorded
        // rather than propagated by the caller at each call site. Gated
        // (not unconditional) to avoid growing every function's
        // prologue by one instruction for no reason.
        if (is_goto_target_fn(fn->name)) {
            asm_mov_reg_sp(cg_sec, ARM64_X16); // mov x16, sp
            arm64_stur(cg_sec, 1, ARM64_X16, ARM64_X29, -CHAIN_RSP_OFFSET);
        }

        // Save variadic argument registers at the bottom of the frame (sp)
        // Skip on Apple ARM64: variadic args are already on the caller's stack.
        if (fn->is_variadic && va_save_size > 0) {
            asm_stp_sp(cg_sec, ARM64_X0, ARM64_X1, 0, false, false); // stp x0, x1, [sp]
            asm_stp_sp(cg_sec, ARM64_X2, ARM64_X3, 2, false, false); // stp x2, x3, [sp, #16]
            asm_stp_sp(cg_sec, ARM64_X4, ARM64_X5, 4, false, false); // stp x4, x5, [sp, #32]
            asm_stp_sp(cg_sec, ARM64_X6, ARM64_X7, 6, false, false); // stp x6, x7, [sp, #48]
            asm_stp_q_sp(cg_sec, 0, 1, 64); // stp q0, q1, [sp, #64]
            asm_stp_q_sp(cg_sec, 2, 3, 96); // stp q2, q3, [sp, #96]
            asm_stp_q_sp(cg_sec, 4, 5, 128); // stp q4, q5, [sp, #128]
            asm_stp_q_sp(cg_sec, 6, 7, 160); // stp q6, q7, [sp, #160]
            // Save original sp so va_start can find the reg_save_area even after alloca
            arm64_add_imm(cg_sec, 1, ARM64_X16, ARM64_SP, 0, 0); // mov x16, sp
            arm64_stur(cg_sec, 3, ARM64_X16, ARM64_X29, -8); // stur x16, [x29, #-8]
        }

        // Save callee-saved regs
        int cs_off = 16 + va_save_size + ldouble_stash_size;
        for (int j = 0; j < 6; j++) {
            if (callee_mask & (1 << j)) {
                arm64_str_uoff(cg_sec, 3, (Arm64Reg)(ARM64_X19 + j), ARM64_SP, cs_off / 8); // str x{cs}, [sp, #cs_off]
                cs_off += 8;
            }
        }

        // Save hidden struct return pointer if needed
        if (fn->ty->return_ty && (((fn->ty->return_ty->kind == TY_STRUCT || fn->ty->return_ty->kind == TY_UNION) && !struct_returns_in_gp_regs(fn->ty->return_ty)) || fn->ty->return_ty->kind == TY_COMPLEX)) {
            int retbuf_offset = 0;
            for (LVar *var = fn->locals; var; var = var->next) {
                if (var->name && var->name == kw_retbuf) {
                    retbuf_offset = var->offset;
                    break;
                }
            }
            if (retbuf_offset <= 4095)
                arm64_stur(cg_sec, 3, ARM64_X8, ARM64_X29, -retbuf_offset); // stur x8, [x29, #-retbuf_offset]
            else {
                int v = retbuf_offset;
                emit_mov_imm64(ARM64_X16, (uint64_t)v); // mov x16, #v
                arm64_sub_reg(cg_sec, 1, ARM64_X16, ARM64_X29, ARM64_X16, ARM64_LSL, 0); // sub x16, x29, x16
                arm64_str_uoff(cg_sec, 3, ARM64_X8, ARM64_X16, 0); // str x8, [x16]
            }
        }

        // Save incoming params to stack slots
        int gp_param = 0;
        int fp_param = 0;
        int stack_param = 0;
        // long double params (is_flonum, size>8) are deferred: narrowing
        // them via __trunctfdf2 clobbers every caller-saved register, so
        // all other params must be safely saved first (see the deferred
        // pass after this loop).
        LVar *ld_param_var[8];
        int ld_param_fpidx[8];
        int n_ld_deferred = 0;
        {
            for (LVar *var = fn->params; var; var = var->param_next) {
                int hfa_elem_size = 0;
                int hfa_count = (!is_flonum(var->ty) && (var->ty->kind == TY_STRUCT || var->ty->kind == TY_UNION || var->ty->kind == TY_COMPLEX))
                    ? arm64_hfa_count(var->ty, &hfa_elem_size)
                    : 0;
                if (hfa_count > 0 && fp_param + hfa_count <= 8) {
                    if (var->offset <= 4095)
                        asm_sub_x16_fp_imm(cg_sec, var->offset); // sub x16, x29, #var->offset
                    else {
                        int v = var->offset;
                        emit_mov_imm64(ARM64_X16, (uint64_t)v); // mov x16, #v
                        asm_sub_x16_fp_x16(cg_sec); // sub x16, x29, x16
                    }
                    for (int j = 0; j < hfa_count; j++) {
                        int off = j * hfa_elem_size;
                        if (hfa_elem_size == 4)
                            asm_str_s_x16_off(cg_sec, fp_param + j, off); // str s{fp_param+j}, [x16, #off]
                        else
                            asm_str_d_x16_off(cg_sec, fp_param + j, off); // str d{fp_param+j}, [x16, #off]
                    }
                    fp_param += hfa_count;
                } else if (hfa_count > 0) {
                    // HFA struct overflowed V registers — copy from caller's stack frame
                    int spoff = 16 + stack_param * 8;
                    int sz = var->ty->size;
                    // Use byte-copy from caller's stack to local slot
                    int c = ++rcc_label_count;
                    if (var->offset <= 4095)
                        arm64_sub_imm(cg_sec, 1, ARM64_X13, ARM64_X29, var->offset, 0); // sub x13, x29, #var->offset
                    else {
                        int v = var->offset;
                        asm_mov_imm(cg_sec, 13, 8, v & 0xffff);
                        v >>= 16;
                        int s = 16;
                        while (v) {
                            asm_movk(cg_sec, 13, 1, (uint16_t)(v & 0xffff), s);
                            v >>= 16;
                            s += 16;
                        }
                        arm64_sub_reg(cg_sec, 1, ARM64_X13, ARM64_X29, ARM64_X13, ARM64_LSL, 0); // sub x13, x29, x13
                    }
                    // x11 = source (caller's stack)
                    arm64_add_imm(cg_sec, 1, ARM64_X11, ARM64_X29, spoff, 0); // add x11, x29, #spoff
                    asm_mov_imm_phy(cg_sec, ARM64_X9, 8, sz); // mov x9, #sz
                    cg_label_ht_add(format(".L.pcopy.%d", c), cg_sec->len); // .L.pcopy.c:
                    arm64_subs_imm(cg_sec, 1, ARM64_XZR, ARM64_X9, 0, 0); // cmp x9, #0
                    {
                        size_t _off = asm_jcc_label(cg_sec, ARM64_EQ);
                        asm_fixup_add(cg_sec, _off, format(".L.pcopy_end.%d", c), 1);
                    } // b.eq .L.pcopy_end.c
                    arm64_sub_imm(cg_sec, 1, ARM64_X9, ARM64_X9, 1, 0); // sub x9, x9, #1
                    asm_ldrb_w16_x9_phy(cg_sec, ARM64_X11); // ldrb w16, [x11, x9]
                    asm_strb_w16_x9_phy(cg_sec, ARM64_X13); // strb w16, [x13, x9]
                    {
                        size_t _off = asm_jmp_label(cg_sec);
                        asm_fixup_add(cg_sec, _off, format(".L.pcopy.%d", c), 0);
                    } // b .L.pcopy.c
                    cg_label_ht_add(format(".L.pcopy_end.%d", c), cg_sec->len);
                    asm_fixup_resolve(cg_sec, format(".L.pcopy_end.%d", c), cg_sec->len); // .L.pcopy_end.c:
                    stack_param += (sz + 7) / 8;
                } else if (is_flonum(var->ty)) {
                    if (fp_param < 8) {
                        if (var->ty->kind == TY_LDOUBLE && var->ty->size > 8) {
                            // Defer: narrowing needs a `bl`, which would
                            // clobber every other not-yet-saved param's
                            // register (see the deferred pass below).
                            ld_param_var[n_ld_deferred] = var;
                            ld_param_fpidx[n_ld_deferred] = fp_param;
                            n_ld_deferred++;
                        } else if (var->ty->size == 4) {
                            if (fn->ty->is_oldstyle) {
                                asm_fcvt_s0_d(cg_sec, fp_param); // fcvt s0, d{fp_param}
                                asm_str_s0_fp_neg(cg_sec, var->offset); // str s0, [x29, #-offset]
                            } else {
                                asm_str_s_fp_neg(cg_sec, fp_param, var->offset); // str s{fp_param}, [x29, #-offset]
                            }
                        } else
                            asm_str_d_fp_neg(cg_sec, fp_param, var->offset); // str d{fp_param}, [x29, #-offset]
                        fp_param++;
                    } else {
                        // 9th+ named float/double parameter: the 8 V
                        // argument registers (v0-v7) are exhausted, so
                        // AAPCS64 places this on the stack exactly like an
                        // overflowing integer parameter (see the general
                        // "Stack argument" branch further below, whose
                        // is_flonum handling this mirrors) -- this branch
                        // was previously entirely missing: is_flonum
                        // params are dispatched here BEFORE the generic
                        // stack-argument branch ever sees them, so that
                        // branch's own (correct) is_flonum handling was
                        // unreachable dead code, and a 9th+ float/double
                        // parameter's local slot was left uninitialized
                        // (whatever was on the stack from a prior frame),
                        // silently reading as garbage/zero.
                        int spoff = 16 + stack_param * 8;
                        if (var->ty->size == 4) {
                            asm_ldr_s0_fp_off(cg_sec, spoff); // ldr s0, [x29, #spoff]
                            asm_str_s0_fp_neg(cg_sec, var->offset); // str s0, [x29, #-offset]
                        } else {
                            asm_ldr_d0_fp_off(cg_sec, spoff); // ldr d0, [x29, #spoff]
                            asm_str_d_fp_neg(cg_sec, 0, var->offset); // str d0, [x29, #-offset]
                        }
                        stack_param++;
                    }
                } else if (var->ty->kind == TY_INT128 || var->ty->kind == TY_DECIMAL128) {
                    // int128 / _Decimal128: two consecutive GP registers
                    if (gp_param + 1 < 8) {
                        /* golden:
                           printf("  str %s, [%s, #-%d]\n", gpreg[gp_param], FRAME_PTR, var->offset);
                           printf("  str %s, [%s, #-%d]\n", gpreg[gp_param + 1], FRAME_PTR, var->offset - 8);
                           GNU as converts str x{n}, [x29, #-offset] to stur */
                        arm64_stur(cg_sec, 1, (Arm64Reg)gp_param, ARM64_X29, -var->offset); // stur x{gp_param}, [x29, #-offset]
                        arm64_stur(cg_sec, 1, (Arm64Reg)(gp_param + 1), ARM64_X29, -(var->offset - 8)); // stur x{gp_param+1}, [x29, #-(offset-8)]
                        gp_param += 2;
                    } else {
                        // int128 on stack: two 8-byte slots from caller's frame
                        int spoff = 16 + stack_param * 8;
                        arm64_ldr_uoff(cg_sec, 3, ARM64_X11, ARM64_X29, (uint32_t)(spoff / 8)); // ldr x11, [x29, #spoff]
                        arm64_stur(cg_sec, 1, ARM64_X11, ARM64_X29, -var->offset); // stur x11, [x29, #-var->offset]
                        arm64_ldr_uoff(cg_sec, 3, ARM64_X11, ARM64_X29, (uint32_t)((spoff + 8) / 8)); // ldr x11, [x29, #spoff+8]
                        arm64_stur(cg_sec, 1, ARM64_X11, ARM64_X29, -(var->offset - 8)); // stur x11, [x29, #-(var->offset-8)]
                        stack_param += 2;
                    }
                } else if (gp_param < 8) {
                    int sf = var->ty->size <= 4 ? 0 : 1; // word or dword
                    if (is_complex(var->ty) && var->ty->size > 8) {
                        // _Complex double (16 bytes): two consecutive GP registers
                        if (gp_param + 1 < 8) {
                            arm64_stur(cg_sec, sf, (Arm64Reg)gp_param, ARM64_X29, -var->offset); // stur x{gp_param}, [x29, #-offset]
                            arm64_stur(cg_sec, sf, (Arm64Reg)(gp_param + 1), ARM64_X29, -(var->offset - 8)); // stur x{gp_param+1}, [x29, #-(offset-8)]
                            gp_param += 2;
                            continue;
                        } else {
                            // complex > 8 bytes on stack
                            int spoff = 16 + stack_param * 8;
                            arm64_ldr_uoff(cg_sec, 3, ARM64_X11, ARM64_X29, (uint32_t)(spoff / 8)); // ldr x11, [x29, #spoff]
                            arm64_stur(cg_sec, 1, ARM64_X11, ARM64_X29, -var->offset); // stur x11, [x29, #-offset]
                            arm64_ldr_uoff(cg_sec, 3, ARM64_X11, ARM64_X29, (uint32_t)((spoff + 8) / 8)); // ldr x11, [x29, #spoff+8]
                            arm64_stur(cg_sec, 1, ARM64_X11, ARM64_X29, -(var->offset - 8)); // stur x11, [x29, #-(offset-8)]
                            stack_param += 2;
                            continue;
                        }
                    }
                    if ((var->ty->kind == TY_STRUCT || var->ty->kind == TY_UNION) && var->ty->size > 8) {

                        int c = ++rcc_label_count;
                        arm64_add_imm(cg_sec, 1, ARM64_X16, gp_param, 0, 0); // mov x16, x{gp_param}
                        if (var->offset <= 4095)
                            arm64_sub_imm(cg_sec, 1, ARM64_X17, ARM64_X29, var->offset, 0); // sub x17, x29, #var->offset
                        else {
                            emit_mov_imm64(ARM64_X17, (uint64_t)var->offset); // mov x17, #var->offset
                            arm64_sub_reg(cg_sec, 1, ARM64_X17, ARM64_X29, ARM64_X17, ARM64_LSL, 0); // sub x17, x29, x17
                        }
                        emit_mov_imm64(ARM64_X9, (uint64_t)var->ty->size); // mov x9, #var->ty->size
                        cg_def_label(format(".L.param_copy.%d", c));
                        arm64_subs_imm(cg_sec, 1, ARM64_XZR, ARM64_X9, 0, 0); // cmp x9, #0
                        size_t cj = asm_jcc_label(cg_sec, ARM64_EQ); // beq .L.param_copy_end.%d
                        asm_fixup_add(cg_sec, cj, format(".L.param_copy_end.%d", c), 1);
                        arm64_sub_imm(cg_sec, 1, ARM64_X9, ARM64_X9, 1, 0); // sub x9, x9, #1
                        asm_ldur_phy(cg_sec, ARM64_X18, ARM64_X16, 0, 0); // ldurb w18, [x16]
                        asm_stur_phy(cg_sec, ARM64_X18, ARM64_X17, 0, 0); // sturb w18, [x17]
                        arm64_add_imm(cg_sec, 1, ARM64_X16, ARM64_X16, 1, 0); // add x16, x16, #1
                        arm64_add_imm(cg_sec, 1, ARM64_X17, ARM64_X17, 1, 0); // add x17, x17, #1
                        size_t cj2 = asm_jmp_label(cg_sec); // b .L.param_copy.%d
                        asm_fixup_add(cg_sec, cj2, format(".L.param_copy.%d", c), 0);
                        cg_def_label(format(".L.param_copy_end.%d", c));
                    } else {
                        int sf = var->ty->size <= 4 ? 2 : 3;
                        asm_stur_phy(cg_sec, gp_param, ARM64_X29, sf, -var->offset); // stur x/w{gp_param}, [x29, #-offset]
                    }
                    gp_param++;
                } else {
                    // Stack argument — load from caller's frame using correct width
                    int spoff = 16 + stack_param * 8;
                    if (is_flonum(var->ty)) {
                        if (var->ty->size == 4) {
                            asm_ldr_s0_fp_off(cg_sec, spoff); // ldr s0, [x29, #spoff]
                            asm_str_s0_fp_neg(cg_sec, var->offset); // str s0, [x29, #-offset]
                        } else {
                            asm_ldr_d0_fp_off(cg_sec, spoff); // ldr d0, [x29, #spoff]
                            asm_str_d_fp_neg(cg_sec, 0, var->offset); // str d0, [x29, #-offset]
                        }
                    } else if (var->ty->size <= 1) {
                        asm_ldrb_w11_fp_off(cg_sec, spoff); // ldrb w11, [x29, #spoff]
                        asm_strb_w11_fp_neg(cg_sec, var->offset); // strb w11, [x29, #-offset]
                    } else if (var->ty->size <= 2) {
                        asm_ldrh_w11_fp_off(cg_sec, spoff); // ldrh w11, [x29, #spoff]
                        asm_strh_w11_fp_neg(cg_sec, var->offset); // strh w11, [x29, #-offset]
                    } else if (var->ty->size <= 4) {
                        asm_ldr_w11_fp_off(cg_sec, spoff); // ldr w11, [x29, #spoff]
                        asm_str_w11_fp_neg(cg_sec, var->offset); // str w11, [x29, #-offset]
                    } else {
                        asm_ldr_x11_fp_off(cg_sec, spoff); // ldr x11, [x29, #spoff]
                        asm_str_x11_fp_neg(cg_sec, var->offset); // str x11, [x29, #-offset]
                    }
                    stack_param++;
                }
            }
        }

        // Deferred long double params: stash every raw incoming quad first
        // (plain register-to-memory stores, no calls - safe regardless of
        // how many there are), then narrow each one via __trunctfdf2 from
        // its stash slot and store the resulting double. By this point
        // every other param has already been saved to its permanent slot
        // above, so the `bl`s below are free to clobber any caller-saved
        // register.
        if (n_ld_deferred > 0) {
            for (int k = 0; k < n_ld_deferred; k++)
                arm64_str_fp(cg_sec, 4, (Arm64Reg)ld_param_fpidx[k], ARM64_SP,
                             (uint32_t)(ldouble_stash_off + k * 16)); // str q{fpidx}, [sp, #stash+k*16]
            for (int k = 0; k < n_ld_deferred; k++) {
                arm64_ldr_fp(cg_sec, 4, ASM_Q0, ARM64_SP,
                             (uint32_t)(ldouble_stash_off + k * 16)); // ldr q0, [sp, #stash+k*16]
                emit_direct_call("__trunctfdf2", false); // bl __trunctfdf2 (quad in v0 -> d0)
                asm_str_d_fp_neg(cg_sec, 0, ld_param_var[k]->offset); // str d0, [x29, #-offset]
            }
        }


        // Re-run gen() to emit binary body
        for (Node *n = fn->body; n; n = n->next) {
            asm_peep_node_start(cg_sec); // .L.return.%s:
            VReg r = gen(n);
            asm_peep_node_end(cg_sec); // .L.return.%s:
            if (r != -1) free_reg(r);
        }

        // === ARM64 epilogue ===
        cg_def_label(format(".L.return.%s", fn->name));

        // Cleanup calls
        bool has_cleanup = false;
        for (LVar *var = fn->locals; var; var = var->next)
            if (var_has_cleanup(var)) {
                has_cleanup = true;
                break;
            }
        if (has_cleanup && fn_ret_nonvoid) {
            // Save x0 (return value) to x19 (callee-saved) before cleanup calls.
            // x19 is guaranteed saved/restored by the prologue (see ever_used_regs |= above).
            asm_mov_phy_phy(cg_sec, ARM64_X19, ARM64_X0, 1); // mov x19, x0 (save return value)
            if (fn_ret_gp_pair)
                asm_mov_phy_phy(cg_sec, ARM64_X20, ARM64_X1, 1); // mov x20, x1 (save 2nd eightbyte too)
        }
        for (LVar *var = fn->locals; var; var = var->next) {
            if (var->defer_stmt) {
                VReg r = gen(var->defer_stmt);
                if (r != -1) free_reg(r);
                continue;
            }
            if (var_has_cleanup(var))
                emit_cleanup_var(var);
        }
        if (has_cleanup && fn_ret_nonvoid) {
            // Restore x0 from x19 after cleanup calls
            asm_mov_phy_phy(cg_sec, ARM64_X0, ARM64_X19, 1); // mov x0, x19 (restore return value)
            if (fn_ret_gp_pair)
                asm_mov_phy_phy(cg_sec, ARM64_X1, ARM64_X20, 1); // mov x1, x20 (restore 2nd eightbyte)
        }

        // VLA or alloca may have moved sp; restore to fixed frame position
        // before reading callee-saved regs (stored at sp+16..sp+n at frame entry)
        if (fn->dealloc_vla || fn_uses_alloca) {
            if (frame_size <= 4095)
                arm64_sub_imm(cg_sec, 1, ARM64_SP, ARM64_X29, frame_size, 0); // sub sp, x29, #frame_size
            else {
                emit_mov_imm64(ARM64_X16, (uint64_t)frame_size); // mov x16, #frame_size
                arm64_sub_extreg(cg_sec, 1, ARM64_SP, ARM64_X29, ARM64_X16, ARM64_UXTX, 0); // sub sp, x29, x16
            }
        }

        // Restore callee-saved
        cs_off = 16 + va_save_size + ldouble_stash_size;
        for (int j = 0; j < 6; j++) {
            if (callee_mask & (1 << j)) {
                arm64_ldr_uoff(cg_sec, 3, (Arm64Reg)(ARM64_X19 + j), ARM64_SP, cs_off / 8); // ldr x{cs}, [sp, #cs_off]
                cs_off += 8;
            }
        }

        if (frame_size <= 4095)
            arm64_add_imm(cg_sec, 1, ARM64_SP, ARM64_SP, frame_size, 0); // add sp, sp, #frame_size
        else {
            emit_mov_imm64(ARM64_X16, (uint64_t)frame_size); // mov x16, #frame_size
            arm64_add_extreg(cg_sec, 1, ARM64_SP, ARM64_SP, ARM64_X16, ARM64_UXTX, 0); // add sp, sp, x16
        }
        asm_ldp_fp_lr(cg_sec); // ldp x29, x30, [sp], #16
        asm_ret(cg_sec); // ret
        cg_patch_fn_size(fn_sym_name, fn_sym_start);

#else
        // === x86_64 prologue ===
        // Determine which callee-saved regs need saving.
        // Windows: rbx,r12,r13,r14,r15,rsi (indices 2-7)
        // Linux SysV: rbx,r12,r13,r14,r15 only (rsi is caller-saved)
#ifdef _WIN32
        int callee_count = 6;
#else
        int callee_count = 5;
#endif
        int callee_mask = (ever_used_regs >> 2) & ((1 << callee_count) - 1);
        int n_pushes = 0;
        for (int j = 0; j < callee_count; j++)
            if (callee_mask & (1 << j)) n_pushes++;

        // Calculate stack frame size
        // Total space below rbp must cover: locals + spills + shadow (32)
        int need = fn->stack_size + fn_struct_ret_total + fn_trampoline_total + 32;
        if (fn->is_variadic)
            need = va_reg_save_ofs;
        // Spill slots grew from spill_reserved_base during Pass 1,
        // independent of `need`'s regions -- can overrun into locals under
        // heavy spill pressure. Re-anchor the excess on top of `need`;
        // Pass 2 replays the same push_spill_slot() sequence.
        int spill_bytes = next_spill_slot - spill_reserved_base;
        if (spill_bytes < 0) spill_bytes = 0;
        // __cleanup__ epilogue (below) stashes RAX (+RDX for a GP-pair
        // struct return) via spill_offset() *after* Pass 2's body walk --
        // invisible to Pass 1's discovery. Budget it explicitly so it
        // can't grow past `need` into the callee-saved area.
        bool fn_has_cleanup = false;
        for (LVar *cv = fn->locals; cv; cv = cv->next)
            if (var_has_cleanup(cv)) {
                fn_has_cleanup = true;
                break;
            }
        if (fn_has_cleanup) {
            spill_bytes += 8;
#ifndef _WIN32
            if (fn->ty->return_ty && struct_returns_in_gp_regs(fn->ty->return_ty) && fn->ty->return_ty->size > 8)
                spill_bytes += 8;
#endif
        }
        next_spill_slot = need;
        need += spill_bytes;
        // Callee-saved registers are stored in a dedicated save area BELOW
        // the locals/spill region, at [rbp - need - 8 .. rbp - need -
        // push_bytes]. They must NOT be pushed at [rbp-8..]: locals and
        // spill slots are addressed downwards from rbp starting at offset
        // 8, so pushed registers would overlap them and any local/spill
        // store would corrupt a saved register, returning garbage to the
        // caller when the epilogue restores it.
        int push_bytes = n_pushes * 8;
        int sub_amount = need + push_bytes;
        // Fix 16-byte alignment (rsp is 16-aligned after push rbp)
        if (sub_amount % 16 != 0) sub_amount += 8;

        // Emit prologue - handle is_weak, inline, and static linkage.
        // For inline functions, check:
        // 1. If there's a non-inline declaration (has_init) in globals
        // 2. If prior non-weak declaration had extern (LVar is_extern=true)
        bool has_noninline_decl = false;
        bool had_extern_decl = false;
        if (fn->is_inline && !fn->is_extern) {
            for (LVar *g = prog->globals; g; g = g->next) {
                if (g->is_function && g->name == fn->name) {
                    if (g->has_init)
                        has_noninline_decl = true;
                    // Only consider non-weak extern declarations
                    if (g->is_extern && !g->is_weak)
                        had_extern_decl = true;
                    break;
                }
            }
        }
        char *fn_label = fn->name;
        if (is_asm_reserved(fn->name))
            fn_label = format(".L_rcc_%s", fn->name);
        // See the ARM64 fn_exported computation above for the GNU89
        // gnu_inline rationale.
        bool fn_exported = !fn->is_static && !fn_emitting_local_copy &&
            (!fn->is_inline || fn->is_extern || has_noninline_decl || had_extern_decl || fn->is_gnu_inline);
        const char *fn_sym_name = fn->asm_name ? fn->asm_name : asm_sym_name(sym_name(fn_label));
        size_t fn_sym_start = cg_sec->len;
        if (fn->is_weak) {
            cg_weak_label(fn_sym_name); // .weak_definition %s
        } else if (fn_exported) {
            cg_global_label(fn_sym_name); // .globl %s
        } else {
            cg_def_fn_label(fn_sym_name); // .weak %s
        }
#ifdef _WIN32
        uw_begin(); // .seh_proc
#endif
        asm_push(cg_sec, X86_RBP); // push rX86_RBP
#ifdef _WIN32
        uw_pushreg(X86_RBP); // .seh_pushreg %rbp
#endif
        asm_mov_rsp_rbp(cg_sec); // movq %rsp, %rbp
#ifdef _WIN32
        uw_set_fpreg(X86_RBP); // .seh_setframe %rbp, 0
#endif

        // Callee-saved registers are NOT pushed here; they are saved into
        // dedicated slots below the locals/spill region right after the
        // stack allocation (see the push_bytes comment above).
#ifdef _WIN32
        // Windows requires probing the stack one page at a time when growing
        // it by more than a page: a single large `sub %rsp` can jump over the
        // guard page straight into unmapped memory. mingw-w64 and MSVC call
        // ___chkstk_ms to touch each page in turn.
        if (sub_amount >= 4096) {
            x86_mov_ri(cg_sec, 4, X86_RAX, sub_amount); // movl $sub_amount, %eax
            emit_direct_call("___chkstk_ms", false); // call ___chkstk_ms
            x86_sub_rr(cg_sec, 8, X86_RSP, X86_RAX); // subq %rax, %rsp
        } else
#endif
            asm_sub_rsp_imm(cg_sec, sub_amount); // subq $sub_amount, %rsp
#ifdef _WIN32
        uw_stackalloc(sub_amount); // .seh_stackalloc sub_amount
        uw_endprologue(); // .seh_endprologue
#endif

        // Save the callee-saved registers that were actually used into
        // their dedicated slots below the locals/spill region.
        for (int j = 0, slot = need; j < callee_count; j++)
            if (callee_mask & (1 << j)) {
                slot += 8;
                asm_mov_phyreg_rbp(cg_sec, REG(j + 2), 8, slot); // movq %reg, -slot(%rbp)
            }

        // GNU nested function: spill the incoming static-chain pointer
        // (physical %r10, the SysV convention this codebase reuses) to its
        // fixed frame slot immediately, before any other codegen can
        // clobber r10. This is a raw physical-register store, not an
        // alloc_reg()'d VReg, so it has zero effect on Pass 1's
        // register-usage discovery — both passes emit it identically.
        if (fn->is_nested)
            asm_mov_phyreg_rbp(cg_sec, X86_R10, 8, CHAIN_SLOT_OFFSET);

        // Nonlocal goto target support: a function whose __label__ is
        // actually reached by a nested descendant's nonlocal goto (see
        // is_goto_target_fn, set during parsing) records ITS OWN
        // just-established, stable rsp into its own CHAIN_RSP_OFFSET
        // slot. A descendant's nonlocal goto (ND_GOTO in codegen.c)
        // chain-walks to this function's rbp, then reads this exact
        // value back - see CHAIN_RSP_OFFSET's comment in rcc.h for why
        // this must be self-recorded rather than propagated by the
        // caller at each call site (call-site-local register/argument
        // staging can transiently perturb rsp around the very call that
        // would otherwise capture it, corrupting the propagated value).
        // Gated (not unconditional) to avoid growing every function's
        // prologue by one instruction for no reason.
        if (is_goto_target_fn(fn->name))
            asm_mov_phyreg_rbp(cg_sec, X86_RSP, 8, CHAIN_RSP_OFFSET);

        // Save variadic argument registers to the reg_save_area
        // (must happen before param saves, which may clobber xmm0 via cvtsd2ss)
        if (fn->is_variadic) {
#ifdef _WIN32
            // Windows x64 ABI: save rcx, rdx, r8, r9 and xmm0-xmm3
            int gp_count = va_gp_start / 8;
            switch (gp_count) {
            case 0: asm_mov_phyreg_rbp(cg_sec, X86_RCX, 8, va_reg_save_ofs); /* fallthrough */ /* movq %rcx, -%d(%rbp) */
            case 1: asm_mov_phyreg_rbp(cg_sec, X86_RDX, 8, va_reg_save_ofs - 8); /* fallthrough */ /* movq %rdx, -%d(%rbp) */
            case 2: asm_mov_phyreg_rbp(cg_sec, X86_R8, 8, va_reg_save_ofs - 16); /* fallthrough */ /* movq %r8, -%d(%rbp) */
            case 3: asm_mov_phyreg_rbp(cg_sec, X86_R9, 8, va_reg_save_ofs - 24); /* movq %r9, -%d(%rbp) */
            }
            asm_movaps_rbp_xmm(cg_sec, 0, va_reg_save_ofs - 32); /* movaps %xmm0, -%d(%rbp) */
            asm_movaps_rbp_xmm(cg_sec, 1, va_reg_save_ofs - 48); /* movaps %xmm1, -%d(%rbp) */
            asm_movaps_rbp_xmm(cg_sec, 2, va_reg_save_ofs - 64); /* movaps %xmm2, -%d(%rbp) */
            asm_movaps_rbp_xmm(cg_sec, 3, va_reg_save_ofs - 80); /* movaps %xmm3, -%d(%rbp) */
#else
            // Linux: save all 6 GP registers: rdi, rsi, rdx, rcx, r8, r9
            asm_mov_phyreg_rbp(cg_sec, X86_RDI, 8, va_reg_save_ofs); /* movq %rdi, -%d(%rbp) */
            asm_mov_phyreg_rbp(cg_sec, X86_RSI, 8, va_reg_save_ofs - 8); /* movq %rsi, -%d(%rbp) */
            asm_mov_phyreg_rbp(cg_sec, X86_RDX, 8, va_reg_save_ofs - 16); /* movq %rdx, -%d(%rbp) */
            asm_mov_phyreg_rbp(cg_sec, X86_RCX, 8, va_reg_save_ofs - 24); /* movq %rcx, -%d(%rbp) */
            asm_mov_phyreg_rbp(cg_sec, X86_R8, 8, va_reg_save_ofs - 32); /* movq %r8, -%d(%rbp) */
            asm_mov_phyreg_rbp(cg_sec, X86_R9, 8, va_reg_save_ofs - 40); /* movq %r9, -%d(%rbp) */
            // Save all 8 XMM registers: xmm0-xmm7
            asm_movaps_rbp_xmm(cg_sec, 0, va_reg_save_ofs - 48); /* movaps %xmm0, -%d(%rbp) */
            asm_movaps_rbp_xmm(cg_sec, 1, va_reg_save_ofs - 64); /* movaps %xmm1, -%d(%rbp) */
            asm_movaps_rbp_xmm(cg_sec, 2, va_reg_save_ofs - 80); /* movaps %xmm2, -%d(%rbp) */
            asm_movaps_rbp_xmm(cg_sec, 3, va_reg_save_ofs - 96); /* movaps %xmm3, -%d(%rbp) */
            asm_movaps_rbp_xmm(cg_sec, 4, va_reg_save_ofs - 112); /* movaps %xmm4, -%d(%rbp) */
            asm_movaps_rbp_xmm(cg_sec, 5, va_reg_save_ofs - 128); /* movaps %xmm5, -%d(%rbp) */
            asm_movaps_rbp_xmm(cg_sec, 6, va_reg_save_ofs - 144); /* movaps %xmm6, -%d(%rbp) */
            asm_movaps_rbp_xmm(cg_sec, 7, va_reg_save_ofs - 160); /* movaps %xmm7, -%d(%rbp) */
#endif
        }

        // Save incoming params from ABI regs to stack slots
        {
#ifdef _WIN32
            X86Reg greg[] = {X86_RCX, X86_RDX, X86_R8, X86_R9};
            int max_gp = 4;
#else
            X86Reg greg[] = {X86_RDI, X86_RSI, X86_RDX, X86_RCX, X86_R8, X86_R9};
            int max_gp = 6;
#endif
            int gp = fn->ty->return_ty &&
                    (
#ifdef _WIN32
                         ((fn->ty->return_ty->kind == TY_STRUCT || fn->ty->return_ty->kind == TY_UNION) && fn->ty->return_ty->size > 8) || (fn->ty->return_ty->kind == TY_COMPLEX && fn->ty->return_ty->size > 8)
#else
                         (fn->ty->return_ty->kind == TY_STRUCT || fn->ty->return_ty->kind == TY_UNION) && !struct_returns_in_gp_regs(fn->ty->return_ty)
#endif
                             )
                ? 1
                : 0;
            int xfp = 0;
            int stack_param_index2 = 0;
            for (LVar *var = fn->params; var; var = var->param_next) {
#ifndef _WIN32
                // SysV x86-64 ABI: `long double` (80-bit x87 extended)
                // is ALWAYS classified MEMORY -- passed on the stack in
                // a 16-byte-aligned, 2-eightbyte slot, never in an XMM
                // register. This is Pass 2's own real-emission copy of
                // the identical fix applied to Pass 1's dry-run layout
                // computation above (this function generates its
                // prologue twice: once with cg_dry_run=true just to
                // discover stack layout, once for real -- the two
                // copies use entirely different local counter names,
                // `xfp`/`stack_param_index2` here vs
                // `param_xmm_index`/`stack_param_index` there, so both
                // needed the fix independently). Without this, a
                // `long double` parameter read its value from an XMM
                // register no caller ever populates for it, reading
                // garbage -- confirmed via `ldnear(long double x,
                // long double y)` (gcc torture's conversion.c): with
                // two long double parameters, only the one whose XMM
                // register happened to still coincidentally hold the
                // right bit pattern (register reuse in the caller's own
                // prior computation) read correctly; the other did not.
                if (var->ty->kind == TY_LDOUBLE) {
                    if (stack_param_index2 & 1)
                        stack_param_index2++;
                    int stack_off2 = 16 + stack_param_index2 * 8;
                    x86_fldt_m(cg_sec, x86_mem(X86_RBP, stack_off2)); // fldt stack_off2(%rbp)
                    x86_fstpl_m(cg_sec, x86_mem(X86_RBP, -var->offset)); // fstpl -var->offset(%rbp)
                    stack_param_index2 += 2;
                    continue;
                }
#endif
#ifndef _WIN32
                // Linux SysV: _Complex float/double are passed in SSE regs,
                // _Complex integer types in GP regs, matching the call-site
                // classification in gen_funcall.
                if (is_complex(var->ty)) {
                    bool cfloat = var->ty->base && is_flonum(var->ty->base);
                    if (cfloat) {
                        int base_sz = var->ty->base ? var->ty->base->size : 8;
                        if (var->ty->size <= 8) {
                            // _Complex float (or other <=8-byte floating complex):
                            // packed in one xmm reg; store the whole 8 bytes.
                            if (xfp < 8) {
                                x86_movsd_mr(cg_sec, x86_mem(X86_RBP, -var->offset), (X86XmmReg)xfp);
                                xfp++;
                            } else {
                                int stack_off2 = 16 + stack_param_index2 * 8;
                                x86_movsd_rm(cg_sec, X86_XMM0, x86_mem(X86_RBP, stack_off2));
                                x86_movsd_mr(cg_sec, x86_mem(X86_RBP, -var->offset), X86_XMM0);
                                stack_param_index2++;
                            }
                        } else {
                            // _Complex double: real in xmm{xfp}, imag in xmm{xfp+1}
                            if (xfp + 1 < 8) {
                                x86_movsd_mr(cg_sec, x86_mem(X86_RBP, -var->offset), (X86XmmReg)xfp);
                                x86_movsd_mr(cg_sec, x86_mem(X86_RBP, -(var->offset - base_sz)), (X86XmmReg)(xfp + 1));
                                xfp += 2;
                            } else {
                                int stack_off2 = 16 + stack_param_index2 * 8;
                                if (stack_param_index2 & 1) stack_param_index2++;
                                stack_off2 = 16 + stack_param_index2 * 8;
                                x86_movsd_rm(cg_sec, X86_XMM0, x86_mem(X86_RBP, stack_off2));
                                x86_movsd_mr(cg_sec, x86_mem(X86_RBP, -var->offset), X86_XMM0);
                                x86_movsd_rm(cg_sec, X86_XMM0, x86_mem(X86_RBP, stack_off2 + base_sz));
                                x86_movsd_mr(cg_sec, x86_mem(X86_RBP, -(var->offset - base_sz)), X86_XMM0);
                                stack_param_index2 += 2;
                            }
                        }
                    } else {
                        // Integer _Complex: one or two GP regs
                        if (var->ty->size <= 8) {
                            if (gp < max_gp) {
                                x86_mov_mr(cg_sec, 8, x86_mem(X86_RBP, -var->offset), greg[gp]);
                                gp++;
                            } else {
                                int stack_off2 = 16 + stack_param_index2 * 8;
                                x86_mov_rm(cg_sec, 8, X86_RAX, x86_mem(X86_RBP, stack_off2));
                                x86_mov_mr(cg_sec, 8, x86_mem(X86_RBP, -var->offset), X86_RAX);
                                stack_param_index2++;
                            }
                        } else {
                            if (gp + 1 < max_gp) {
                                int base_sz = var->ty->base ? var->ty->base->size : 8;
                                x86_mov_mr(cg_sec, base_sz <= 4 ? 4 : 8, x86_mem(X86_RBP, -var->offset), greg[gp]);
                                x86_mov_mr(cg_sec, base_sz <= 4 ? 4 : 8, x86_mem(X86_RBP, -(var->offset - base_sz)), greg[gp + 1]);
                                gp += 2;
                            } else {
                                int stack_off2 = 16 + stack_param_index2 * 8;
                                if (stack_param_index2 & 1) stack_param_index2++;
                                stack_off2 = 16 + stack_param_index2 * 8;
                                x86_mov_rm(cg_sec, 8, X86_RAX, x86_mem(X86_RBP, stack_off2));
                                x86_mov_mr(cg_sec, 8, x86_mem(X86_RBP, -var->offset), X86_RAX);
                                x86_mov_rm(cg_sec, 8, X86_RAX, x86_mem(X86_RBP, stack_off2 + 8));
                                x86_mov_mr(cg_sec, 8, x86_mem(X86_RBP, -(var->offset - 8)), X86_RAX);
                                stack_param_index2 += 2;
                            }
                        }
                    }
                    continue;
                }
                // Linux SysV: __int128 / _Decimal128 occupy two consecutive
                // GP registers/stack slots
                if (var->ty->kind == TY_INT128 || var->ty->kind == TY_DECIMAL128) {
                    if (gp + 1 < max_gp) {
                        x86_mov_mr(cg_sec, 8, x86_mem(X86_RBP, -var->offset), greg[gp]); // movq greg[gp], -off(%rbp)
                        x86_mov_mr(cg_sec, 8, x86_mem(X86_RBP, -(var->offset - 8)), greg[gp + 1]); // movq greg[gp+1], -(off-8)(%rbp)
                        gp += 2;
                    } else {
                        if (stack_param_index2 & 1) stack_param_index2++;
                        int stack_off2 = 16 + stack_param_index2 * 8;
                        x86_mov_rm(cg_sec, 8, X86_RAX, x86_mem(X86_RBP, stack_off2)); // movq stack_off(%rbp), %rax
                        x86_mov_mr(cg_sec, 8, x86_mem(X86_RBP, -var->offset), X86_RAX); // movq %rax, -off(%rbp)
                        x86_mov_rm(cg_sec, 8, X86_RAX, x86_mem(X86_RBP, stack_off2 + 8)); // movq stack_off+8(%rbp), %rax
                        x86_mov_mr(cg_sec, 8, x86_mem(X86_RBP, -(var->offset - 8)), X86_RAX); // movq %rax, -(off-8)(%rbp)
                        stack_param_index2 += 2;
                    }
                    continue;
                }
#endif
#ifdef _WIN32
                // Win64: __int128/_Decimal128 is passed by pointer (like large struct)
                if (var->ty->kind == TY_INT128 || var->ty->kind == TY_DECIMAL128) {
                    if (gp < max_gp) {
                        x86_mov_rm(cg_sec, 8, X86_RAX, x86_mem(greg[gp], 0)); // movq (greg[gp]), %rax
                        x86_mov_mr(cg_sec, 8, x86_mem(X86_RBP, -var->offset), X86_RAX); // movq %rax, -off(%rbp)
                        x86_mov_rm(cg_sec, 8, X86_RAX, x86_mem(greg[gp], 8)); // movq 8(greg[gp]), %rax
                        x86_mov_mr(cg_sec, 8, x86_mem(X86_RBP, -(var->offset - 8)), X86_RAX); // movq %rax, -(off-8)(%rbp)
                        gp++;
                    } else {
                        int stack_off2 = 48 + stack_param_index2 * 8;
                        x86_mov_rm(cg_sec, 8, X86_RAX, x86_mem(X86_RBP, stack_off2)); // movq stack_off(%rbp), %rax
                        x86_mov_rm(cg_sec, 8, X86_RCX, x86_mem(X86_RAX, 0)); // movq (%rax), %rcx (lo)
                        x86_mov_mr(cg_sec, 8, x86_mem(X86_RBP, -var->offset), X86_RCX); // movq %rcx, -off(%rbp)
                        x86_mov_rm(cg_sec, 8, X86_RCX, x86_mem(X86_RAX, 8)); // movq 8(%rax), %rcx (hi)
                        x86_mov_mr(cg_sec, 8, x86_mem(X86_RBP, -(var->offset - 8)), X86_RCX); // movq %rcx, -(off-8)(%rbp)
                        stack_param_index2++;
                    }
                    continue;
                }
                // Win64: _Complex > 8 bytes passed by pointer like large struct
                bool is_large_complex = is_complex(var->ty) && var->ty->size > 8;
                if (is_large_complex && gp < max_gp) {
                    // Complex > 8 in GP reg: copy pointee to local slot
                    int c = ++rcc_label_count;
                    x86_mov_rr(cg_sec, 8, X86_R11, greg[gp]); // movq greg[gp], %r11
                    x86_mov_ri(cg_sec, 8, X86_R10, var->ty->size); // movq $size, %r10
                    cg_def_label(format(".L.param2.%d", c));
                    x86_cmp_ri(cg_sec, 8, X86_R10, 0);
                    size_t js = cg_sec->len;
                    x86_jcc_rel32(cg_sec, X86_E, 0);
                    asm_fixup_add(cg_sec, js, format(".L.param2_end.%d", c), 1);
                    asm_movb_r11_r10_al(cg_sec, -1);
                    asm_movb_al_rbp_r10(cg_sec, var->offset);
                    x86_sub_ri(cg_sec, 8, X86_R10, 1);
                    size_t jp = cg_sec->len;
                    x86_jmp_rel32(cg_sec, 0);
                    asm_fixup_add(cg_sec, jp, format(".L.param2.%d", c), 0);
                    cg_def_label(format(".L.param2_end.%d", c));
                    gp++;
                    continue;
                }
#endif
                if (!is_flonum(var->ty) && gp < max_gp && !((var->ty->kind == TY_STRUCT || var->ty->kind == TY_UNION || is_complex(var->ty)) && var->ty->size > 8)) {
                    int sz = var->ty->size <= 4 ? 4 : 8;
                    x86_mov_mr(cg_sec, sz, x86_mem(X86_RBP, -var->offset), greg[gp]); // %s:
                    gp++;
                } else if (is_flonum(var->ty) &&
#ifdef _WIN32
                           gp < max_gp // Win64: float and int share the same 4-reg slots
#else
                           xfp < 8
#endif
                ) {
                    // Save float/double param from xmm register to stack slot
#ifdef _WIN32
                    int xmm_idx = gp; // Win64: float uses slot gp (combined counter)
#else
                    int xmm_idx = xfp;
#endif
                    if (var->ty->size <= 4) {
                        if (fn->ty->is_oldstyle) {
                            // K&R oldstyle definition: ANY caller (having
                            // no prototype to bind against, by definition
                            // of "oldstyle") applies C's default argument
                            // promotion and sends a genuine DOUBLE here,
                            // never a narrow float -- narrow it down to
                            // match this parameter's declared `float`
                            // storage, mirroring the modern-ABI callee's
                            // widen-then-store for a `double` param below.
                            x86_cvtsd2ss(cg_sec, X86_XMM0, (X86XmmReg)xmm_idx); // cvtsd2ss %xmm{n}, %xmm0
                            x86_movss_mr(cg_sec, x86_mem(X86_RBP, -var->offset), X86_XMM0); // movss %xmm0, -off(%rbp)
                        } else {
                            // ABI: the caller (per the call-site fix in
                            // gen_funcall) now places a genuine narrow
                            // float in the low bits of the incoming xmm
                            // register; store it directly, no narrowing
                            // needed.
                            x86_movss_mr(cg_sec, x86_mem(X86_RBP, -var->offset), (X86XmmReg)xmm_idx); // movss %xmm{n}, -off(%rbp)
                        }
                    } else {
                        x86_movsd_mr(cg_sec, x86_mem(X86_RBP, -var->offset), (X86XmmReg)xmm_idx); // movsd %xmm{n}, -off(%rbp)
                    }
#ifdef _WIN32
                    gp++; // Win64: advance combined position counter
#endif
                    xfp++;
                } else if ((var->ty->kind == TY_STRUCT || var->ty->kind == TY_UNION || is_complex(var->ty)) && var->ty->size > 8 && gp < max_gp) {
                    int c = ++rcc_label_count;
                    x86_mov_rr(cg_sec, 8, X86_R11, greg[gp]);
                    x86_mov_ri(cg_sec, 8, X86_R10, var->ty->size);
                    cg_def_label(format(".L.param2.%d", c));
                    x86_cmp_ri(cg_sec, 8, X86_R10, 0);
                    size_t js = cg_sec->len;
                    x86_jcc_rel32(cg_sec, X86_E, 0);
                    asm_fixup_add(cg_sec, js, format(".L.param2_end.%d", c), 1);
                    asm_movb_r11_r10_al(cg_sec, -1);
                    asm_movb_al_rbp_r10(cg_sec, var->offset);
                    x86_sub_ri(cg_sec, 8, X86_R10, 1);
                    size_t jp = cg_sec->len;
                    x86_jmp_rel32(cg_sec, 0);
                    asm_fixup_add(cg_sec, jp, format(".L.param2.%d", c), 0);
                    cg_def_label(format(".L.param2_end.%d", c));
                    gp++;
                } else {
                    // Stack argument — passed above return address
#ifdef _WIN32
                    int stack_off2 = 48 + stack_param_index2 * 8;
#else
                    int stack_off2 = 16 + stack_param_index2 * 8;
#endif
                    if (is_flonum(var->ty)) {
                        if (var->ty->size <= 4) {
                            if (fn->ty->is_oldstyle) {
                                // K&R oldstyle: caller always promotes to
                                // double (see the register-arg branch's
                                // comment above for why); narrow it.
                                x86_movsd_rm(cg_sec, X86_XMM0, x86_mem(X86_RBP, stack_off2)); // movsd stack_off2(%rbp), xmm0
                                x86_cvtsd2ss(cg_sec, X86_XMM0, X86_XMM0); // cvtsd2ss %xmm0, %xmm0
                                x86_movss_mr(cg_sec, x86_mem(X86_RBP, -var->offset), X86_XMM0); // movss xmm0, -var->offset(%rbp)
                            } else {
                                // ABI: the caller (per the call-site
                                // stack-arg fix in gen_funcall) now stores
                                // a genuine narrow float on the stack; load
                                // it directly, no narrowing needed.
                                x86_movss_rm(cg_sec, X86_XMM0, x86_mem(X86_RBP, stack_off2)); // movss stack_off2(%rbp), xmm0
                                x86_movss_mr(cg_sec, x86_mem(X86_RBP, -var->offset), X86_XMM0); // movss xmm0, -var->offset(%rbp)
                            }
                        } else {
                            x86_movsd_rm(cg_sec, X86_XMM0, x86_mem(X86_RBP, stack_off2)); // movsd stack_off2(%rbp), xmm0
                            x86_movsd_mr(cg_sec, x86_mem(X86_RBP, -var->offset), X86_XMM0); // movsd xmm0, -var->offset(%rbp)
                        }
                    } else if ((var->ty->kind == TY_STRUCT || var->ty->kind == TY_UNION || is_complex(var->ty)) && var->ty->size > 8) {
                        // Structs > 8 bytes (and 16-byte _Complex double) are passed
                        // by pointer even on the stack; load the pointer and copy
                        // the pointee into the local slot.
                        int c = ++rcc_label_count;
                        x86_mov_rm(cg_sec, 8, X86_R11, x86_mem(X86_RBP, stack_off2)); // movq stack_off2(%rbp), %r11
                        x86_mov_ri(cg_sec, 8, X86_R10, var->ty->size);
                        cg_def_label(format(".L.param2.%d", c));
                        x86_cmp_ri(cg_sec, 8, X86_R10, 0);
                        size_t js2 = cg_sec->len;
                        x86_jcc_rel32(cg_sec, X86_E, 0);
                        asm_fixup_add(cg_sec, js2, format(".L.param2_end.%d", c), 1);
                        asm_movb_r11_r10_al(cg_sec, -1); // movb -1(%r11,%r10), %%al
                        asm_movb_al_rbp_r10(cg_sec, var->offset);
                        x86_sub_ri(cg_sec, 8, X86_R10, 1);
                        size_t jp2 = cg_sec->len;
                        x86_jmp_rel32(cg_sec, 0);
                        asm_fixup_add(cg_sec, jp2, format(".L.param2.%d", c), 0);
                        cg_def_label(format(".L.param2_end.%d", c));
                    } else {
                        int psz = var->ty->size <= 4 ? 4 : 8;
                        x86_mov_rm(cg_sec, psz, X86_RAX, x86_mem(X86_RBP, stack_off2)); // mov stack_off2(%rbp), %rax
                        x86_mov_mr(cg_sec, psz, x86_mem(X86_RBP, -var->offset), X86_RAX); // mov %rax, -var->offset(%rbp)
                    }
                    stack_param_index2++;
                }
            }
#ifdef _WIN32
            // Fix va_st_start using correct stack_param_index2 from pass 2
            va_st_start = 48 + stack_param_index2 * 8;
#endif
        }

        bool has_retbuf = fn->ty->return_ty &&
            (fn->ty->return_ty->kind == TY_STRUCT || fn->ty->return_ty->kind == TY_UNION
#ifdef _WIN32
             || (fn->ty->return_ty->kind == TY_COMPLEX && fn->ty->return_ty->size > 8)
#endif
                 )
#ifdef _WIN32
            && fn->ty->return_ty->size > 8
#else
            && !struct_returns_in_gp_regs(fn->ty->return_ty)
#endif
            ;
        if (has_retbuf) {
            int retbuf_offset = 0;
            for (LVar *var = fn->locals; var; var = var->next) {
                if (var->name && var->name == kw_retbuf) {
                    retbuf_offset = var->offset;
                    break;
                }
            }
            x86_mov_mr(cg_sec, 8, x86_mem(X86_RBP, -retbuf_offset),
#ifdef _WIN32
                       X86_RCX
#else
                       X86_RDI
#endif
            ); // mov retreg, -retbuf_offset(%rbp)
        }

        // Re-run gen() to emit binary body
        for (Node *n = fn->body; n; n = n->next) {
            asm_peep_node_start(cg_sec); // mov %s, -%d(%rbp)
            VReg r = gen(n);
            asm_peep_node_end(cg_sec); // .L.return.%s:
            if (r != -1) free_reg(r);
        }
        // Emit epilogue
        cg_def_label(format(".L.return.%s", fn->name)); // mov %s, -%d(%rbp)

        // Emit __cleanup__ calls (LIFO: locals list is in reverse declaration order)
        bool has_cleanup = false;
        for (LVar *var = fn->locals; var; var = var->next)
            if (var_has_cleanup(var)) {
                has_cleanup = true;
                break;
            }
#ifndef _WIN32
        bool save_rdx_for_cleanup = has_cleanup && fn->ty->return_ty && struct_returns_in_gp_regs(fn->ty->return_ty) && fn->ty->return_ty->size > 8;
#else
        bool save_rdx_for_cleanup = false;
#endif
        if (has_cleanup)
            asm_mov_phyreg_rbp(cg_sec, X86_RAX, 8, spill_offset(0)); // mov [rbp-spill_offset(0], X86_RAX)
        if (save_rdx_for_cleanup)
            asm_mov_phyreg_rbp(cg_sec, X86_RDX, 8, spill_offset(1)); // mov [rbp-spill_offset(1], X86_RDX)
        for (LVar *var = fn->locals; var; var = var->next) {
            if (var->defer_stmt) {
                // A top-level (block_depth == 1) defer is exclusively
                // fired here: parser.c's `defer` handling advances
                // current_fn_scope_locals past its own marker, the same
                // way an ordinary top-level declaration does, so no
                // return's own emit_cleanup_range range includes it --
                // every exit path (fall-through and every return alike)
                // converges on .L.return<fn>, reaching this once. A
                // defer nested inside a block never reaches fn->locals
                // (popped off when that block's own scope exits) and
                // stays covered by whichever return's own range is
                // still in scope for it instead.
                VReg r = gen(var->defer_stmt);
                if (r != -1) free_reg(r);
                continue;
            }
            if (var_has_cleanup(var))
                emit_cleanup_var(var);
        }
        if (has_cleanup)
            asm_mov_rbp(cg_sec, X86_RAX, 8, spill_offset(0)); // mov [rbp-spill_offset(0], RAX)
        if (save_rdx_for_cleanup)
            asm_mov_rbp(cg_sec, X86_RDX, 8, spill_offset(1)); // mov [rbp-spill_offset(1], RDX)
        // Reload the callee-saved registers from their dedicated slots below
        // the locals/spill region (mirrors the prologue saves) BEFORE
        // deallocating the frame. These are pure rbp-relative reads and
        // never depend on rsp, so there is no need to drop rsp first — and
        // doing so is actively wrong: it leaves the slots below the (now
        // current) rsp, a region a signal handler or anything else touching
        // sub-rsp memory is free to clobber before the reads execute,
        // silently corrupting the restored registers. Restore first, then
        // collapse rsp back to rbp (uniform for VLA/alloca/normal frames).
        for (int j = 0, slot = need; j < callee_count; j++)
            if (callee_mask & (1 << j)) {
                slot += 8;
                asm_mov_rbp(cg_sec, REG(j + 2), 8, slot); // movq -slot(%rbp), %reg
            }
        asm_mov_rbp_rsp(cg_sec); // movq %rbp, %rsp
        asm_pop(cg_sec, X86_RBP); // popq %rbp
        asm_ret(cg_sec); // ret
        cg_patch_fn_size(fn_sym_name, fn_sym_start);
#ifdef _WIN32
        uw_endproc(); // .seh_endproc
#endif
#endif
        // Resolve every queued label-address-DIFFERENCE static-initializer
        // patch whose labels belong to THIS function, now that its body has
        // been fully generated (every ND_LABEL's cg_def_label() call has
        // run, so cg_label_ht holds its final .text offsets). Must happen
        // here, before the next iteration's cg_label_ht_reset() above wipes
        // them — cg_label_ht holds one function's labels at a time, never
        // accumulated across the translation unit. See queue_label_diff_patch's
        // doc comment for why this can't just run once after the whole loop.
        for (LabelDiffPatch **_pp = &pending_label_diffs; *_pp;) {
            LabelDiffPatch *_p = *_pp;
            if (_p->owner_fn && strcmp(_p->owner_fn, current_fn) == 0) {
                size_t off_hi = cg_label_ht_get(_p->label_hi);
                size_t off_lo = cg_label_ht_get(_p->label_lo);
                if (off_hi != (size_t)-1 && off_lo != (size_t)-1) {
                    int64_t diff = (int64_t)off_hi - (int64_t)off_lo;
                    SecBuf *_sb = objfile_section_buf(cg_obj, _p->section);
                    if (_sb) memcpy(_sb->data + _p->patch_off, &diff, (size_t)_p->size);
                }
                *_pp = _p->next; // consumed
            } else {
                _pp = &_p->next;
            }
        }
    }
    // Emit constructor/destructor entries
    bool has_ctor = false, has_dtor = false;
    for (TLItem *item = prog->items; item; item = item->next) {
        if (item->kind == TL_FUNC) {
            if (item->fn->is_constructor) has_ctor = true;
            if (item->fn->is_destructor) has_dtor = true;
        }
    }
    if (has_dtor) {
#if defined(__APPLE__)
        // macOS: emit destructor stubs that call __cxa_atexit at startup
        for (TLItem *item = prog->items; item; item = item->next) {
            if (item->kind == TL_FUNC && item->fn->is_destructor) {
                cg_set_section(SEC_TEXT);
                cg_def_label(format("___GLOBAL_dtor_%s", item->fn->name));
                asm_stp_fp_lr(cg_sec);
                asm_mov_fp_sp(cg_sec);
                { // adrp x0, dtor_label@PAGE; add x0, x0, dtor_label@PAGEOFF
                    const char *lbl = asm_sym_name(sym_name(item->fn->name));
                    int sidx = objfile_find_sym(cg_obj, lbl);
                    if (sidx < 0) sidx = objfile_add_sym(cg_obj, lbl, SEC_UNDEF, 0, 0, SB_LOCAL, ST_NOTYPE);
                    size_t a_off = cg_sec->len;
                    asm_adrp(cg_sec, ARM64_X0);
                    objfile_add_reloc(cg_obj, SEC_TEXT, a_off, sidx, R_AARCH64_ADR_PREL_PG_HI21, 0);
                    size_t d_off = cg_sec->len;
                    asm_add_rd_rd_0(cg_sec, ARM64_X0);
                    objfile_add_reloc(cg_obj, SEC_TEXT, d_off, sidx, R_AARCH64_ADD_ABS_LO12_NC, 0);
                }
                asm_mov_imm_phy(cg_sec, ARM64_X1, 8, 0);
                { // adrp x2, ___dso_handle@PAGE; add x2, x2, ___dso_handle@PAGEOFF
                    const char *lbl = "___dso_handle";
                    int sidx = objfile_find_sym(cg_obj, lbl);
                    if (sidx < 0) sidx = objfile_add_sym(cg_obj, lbl, SEC_UNDEF, 0, 0, SB_LOCAL, ST_NOTYPE);
                    size_t a_off = cg_sec->len;
                    asm_adrp(cg_sec, ARM64_X2);
                    objfile_add_reloc(cg_obj, SEC_TEXT, a_off, sidx, R_AARCH64_ADR_PREL_PG_HI21, 0);
                    size_t d_off = cg_sec->len;
                    asm_add_rd_rd_0(cg_sec, ARM64_X2);
                    objfile_add_reloc(cg_obj, SEC_TEXT, d_off, sidx, R_AARCH64_ADD_ABS_LO12_NC, 0);
                }
                size_t call_off = asm_call_label(cg_sec);
                int cxa_sidx = objfile_find_sym(cg_obj, "___cxa_atexit");
                if (cxa_sidx < 0)
                    cxa_sidx = objfile_add_sym(cg_obj, "___cxa_atexit", SEC_UNDEF, 0, 0, SB_GLOBAL, ST_FUNC);
                objfile_add_reloc(cg_obj, SEC_TEXT, call_off, cxa_sidx, R_AARCH64_CALL26, 0);
                asm_ldp_fp_lr(cg_sec);
                asm_ret(cg_sec);
            }
        }
#endif
    }
#if defined(__APPLE__)
    if (has_ctor || has_dtor) {
        cg_set_section(SEC_INIT_ARRAY);
        secbuf_align(cg_sec, 8);
        for (TLItem *item = prog->items; item; item = item->next) {
            if (item->kind == TL_FUNC && item->fn->is_constructor) {
                size_t off = cg_sec->len;
                secbuf_emit64le(cg_sec, 0); // .quad constructor
                int sidx = objfile_find_sym(cg_obj, asm_sym_name(sym_name(item->fn->name)));
                if (sidx < 0)
                    sidx = objfile_add_sym(cg_obj, asm_sym_name(sym_name(item->fn->name)), SEC_UNDEF, 0, 0, SB_GLOBAL, ST_NOTYPE);
                objfile_add_reloc(cg_obj, SEC_INIT_ARRAY, off, sidx, R_AARCH64_ABS64, 0);
            }
        }
        for (TLItem *item = prog->items; item; item = item->next) {
            if (item->kind == TL_FUNC && item->fn->is_destructor) {
                size_t off = cg_sec->len;
                secbuf_emit64le(cg_sec, 0);
                const char *stub = format("___GLOBAL_dtor_%s", item->fn->name);
                int sidx = objfile_find_sym(cg_obj, stub);
                if (sidx < 0)
                    sidx = objfile_add_sym(cg_obj, stub, SEC_UNDEF, 0, 0, SB_LOCAL, ST_NOTYPE);
                objfile_add_reloc(cg_obj, SEC_INIT_ARRAY, off, sidx, R_AARCH64_ABS64, 0);
            }
        }
    }
#else
    if (has_ctor) {
        cg_set_section(SEC_INIT_ARRAY);
        for (TLItem *item = prog->items; item; item = item->next) {
            if (item->kind == TL_FUNC && item->fn->is_constructor) {
                size_t off = cg_sec->len;
                secbuf_emit64le(cg_sec, 0); // .quad %s
                int sidx = objfile_find_sym(cg_obj, asm_sym_name(sym_name(item->fn->name)));
                if (sidx < 0)
                    sidx = objfile_add_sym(cg_obj, asm_sym_name(sym_name(item->fn->name)), SEC_UNDEF, 0, 0, SB_GLOBAL, ST_NOTYPE);
#ifdef ARCH_ARM64
                objfile_add_reloc(cg_obj, SEC_INIT_ARRAY, off, sidx, R_AARCH64_ABS64, 0);
#else
                objfile_add_reloc(cg_obj, SEC_INIT_ARRAY, off, sidx, R_X86_64_64, 0);
#endif
            }
        }
#if defined(__APPLE__)
        // Register destructor stubs in __mod_init_func too
        for (TLItem *item = prog->items; item; item = item->next) {
            if (item->kind == TL_FUNC && item->fn->is_destructor)
                (void)0 /* directive: .quad */;
        }
#endif
    }
#if !defined(__APPLE__)
    if (has_dtor) {
        cg_set_section(SEC_FINI_ARRAY);
        for (TLItem *item = prog->items; item; item = item->next) {
            if (item->kind == TL_FUNC && item->fn->is_destructor) {
                size_t off = cg_sec->len;
                secbuf_emit64le(cg_sec, 0); // .quad %s
                int sidx = objfile_find_sym(cg_obj, asm_sym_name(sym_name(item->fn->name)));
                if (sidx < 0)
                    sidx = objfile_add_sym(cg_obj, asm_sym_name(sym_name(item->fn->name)), SEC_UNDEF, 0, 0, SB_GLOBAL, ST_NOTYPE);
#ifdef ARCH_ARM64
                objfile_add_reloc(cg_obj, SEC_FINI_ARRAY, off, sidx, R_AARCH64_ABS64, 0);
#else
                objfile_add_reloc(cg_obj, SEC_FINI_ARRAY, off, sidx, R_X86_64_64, 0);
#endif
            }
        }
    }
#endif
#endif
    if (alloca_needed)
        emit_alloca();

    // Emit float literal constants after all functions
    if (float_lits) {
        cg_set_section(SEC_RODATA);
        (void)0 /* directive: .balign */;
        for (FloatLit *fl = float_lits; fl; fl = fl->next) {
            cg_def_label_sec(format(".LF%d", fl->id), SEC_RODATA);
            if (fl->size == 4) {
                float f = (float)fl->val;
                uint32_t bits;
                memcpy(&bits, &f, 4);
                secbuf_emit32le(cg_sec, bits);
            } else {
                uint64_t bits;
                memcpy(&bits, &fl->val, 8);
                secbuf_emit64le(cg_sec, bits);
            }
        }
    }
    // Resolve pending aliases after all symbols are emitted
    for (LVar *var = prog->globals; var; var = var->next) {
        if (!var->alias_target) continue;
        const char *alias_name = asm_sym_name(sym_name(var->name));
        const char *target_name = asm_sym_name(sym_name(var->alias_target));
        int tidx = objfile_find_sym(cg_obj, target_name);
        if (tidx >= 0 && cg_obj->syms[tidx].section != SEC_UNDEF) {
            // Update existing UNDEF alias symbol to match target
            int aidx = objfile_find_sym(cg_obj, alias_name);
            if (aidx >= 0 && cg_obj->syms[aidx].section == SEC_UNDEF) {
                cg_obj->syms[aidx].section = cg_obj->syms[tidx].section;
                cg_obj->syms[aidx].offset = cg_obj->syms[tidx].offset;
                cg_obj->syms[aidx].size = cg_obj->syms[tidx].size;
                cg_obj->syms[aidx].bind = cg_obj->syms[tidx].bind;
                cg_obj->syms[aidx].type = cg_obj->syms[tidx].type;
            } else if (aidx < 0) {
                objfile_add_sym(cg_obj, alias_name,
                                cg_obj->syms[tidx].section,
                                cg_obj->syms[tidx].offset,
                                cg_obj->syms[tidx].size,
                                cg_obj->syms[tidx].bind,
                                cg_obj->syms[tidx].type);
            }
        }
    }
    // FMV IFUNC resolvers: for functions with __attribute__((target_clones(...))),
    // rename the emitted function body to "foo.default", emit a resolver that
    // returns the best clone, and make "foo" an IFUNC symbol pointing to it.
    // ELF and COFF both support STT_GNU_IFUNC (MinGW binutils ≥ 2.38).
#if 1
    for (TLItem *item = prog->items; item; item = item->next) {
        if (item->kind != TL_FUNC) continue;
        Function *fn = item->fn;
        if (!fn->target_clones || fn->n_target_clones == 0) continue;
        if (!fn->body) continue;

        const char *fn_name = fn->asm_name ? fn->asm_name : fn->name;
        const char *fn_sym = asm_sym_name(sym_name(fn_name));
        const char *default_name = format("%s.default", fn_name);
        const char *default_sym = asm_sym_name(sym_name(default_name));
        const char *resolver_name = format("%s.resolver", fn_name);
        const char *resolver_sym = asm_sym_name(sym_name(resolver_name));

        // Rename existing function symbol from "foo" to "foo.default"
        int fidx = objfile_find_sym(cg_obj, fn_sym);
        if (fidx < 0) continue; // shouldn't happen
        free(cg_obj->syms[fidx].name);
        cg_obj->syms[fidx].name = strdup(default_sym);

        // Emit resolver returning the default clone.
        // Use fidx (renamed symbol index) directly to avoid picking up
        // a duplicate UNDEF entry with the same name from a call-site.
        cg_set_section(SEC_TEXT);
        size_t res_start = cg_sec->len;
        int dsidx = fidx;

#ifdef ARCH_ARM64
        // ARM64: adrp x0, default@PAGE; add x0, x0, default@PAGEOFF; ret
        {
            size_t a_off = cg_sec->len;
            asm_adrp(cg_sec, ARM64_X0);
            objfile_add_reloc(cg_obj, SEC_TEXT, a_off, dsidx, R_AARCH64_ADR_PREL_PG_HI21, 0);
            size_t d_off = cg_sec->len;
            asm_add_rd_rd_0(cg_sec, ARM64_X0);
            objfile_add_reloc(cg_obj, SEC_TEXT, d_off, dsidx, R_AARCH64_ADD_ABS_LO12_NC, 0);
        }
#else
        // x86-64: lea foo.default(%rip), %rax
        secbuf_emit8(cg_sec, 0x48); // REX.W
        secbuf_emit8(cg_sec, 0x8d); // LEA r64, m
        secbuf_emit8(cg_sec, 0x05); // ModRM: rip-relative
        objfile_add_reloc(cg_obj, SEC_TEXT, cg_sec->len, dsidx, R_X86_64_PC32, -4);
        secbuf_emit32le(cg_sec, 0); // placeholder
#endif
        asm_ret(cg_sec);
        size_t res_end = cg_sec->len;

        objfile_add_sym(cg_obj, resolver_sym,
                        SEC_TEXT, res_start, res_end - res_start,
                        SB_GLOBAL, ST_FUNC);

        // Create "foo" as IFUNC pointing to resolver
        objfile_add_sym(cg_obj, fn_sym,
                        SEC_TEXT, res_start, res_end - res_start,
                        SB_GLOBAL, ST_GNU_IFUNC);
    }
#endif
    // Flush DWARF debug line info
    if (objfile_has_debug(cg_obj))
        objfile_flush_debug_line(cg_obj, cg_obj->text.len);
    return cg_obj;
}
