diff --git a/Cargo.toml b/Cargo.toml index 5f258d8..fb282c3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ volt-ir = { path = "src/ir" } volt-target = { path = "src/target" } volt-emulator = { path = "src/emulator" } -object = { version = "0.40.0", default-features = false, features = ["read_core", "write_core", "elf"] } +object = { version = "0.40.0", default-features = false, features = ["read_core", "write_core", "elf", "macho"] } libc = "0.2" parking_lot = "0.12" winnow-args = "0.1" diff --git a/README.md b/README.md index 9fcb730..7bab141 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,41 @@ The disk form supplies `root=/dev/vda rw` by default. `--cpus`, `--idle`, `--fast`, and `--spin` control the virtual CPU count and WFI policy. Use `volt-boot --help` for the complete generated interface. +## XNU boot + +`volt-boot` also boots an XNU kernel. A Mach-O image is detected by its magic, so no +flag is needed: + +```sh +cargo run --release --bin volt-boot -- \ + path/to/kernel.development.qemu 20 --fast +``` + +The loader follows the `bootxnu` U-Boot command: it places the Mach-O segments at a +`physBase` that is congruent to the link-time `virtBase` modulo a 32 MiB L2 block, +builds an Apple flattened device tree (the equivalent of `mkafdt.py`) and a `boot_args` +block, and enters at the kernel entry with the MMU off and `x0` pointing at +`boot_args`. The machine is the QEMU `virt` layout XNU's QEMU platform expects: +PL011 `uart0` at `0x09000000`, and a GICv3 (distributor `0x08000000`, redistributor +`0x080a0000`, `ICC_*` system registers) that forwards the virtual timer as a FIQ. The +last 512 KiB of RAM is the panic log (`chosen/pram`), and `memSize` excludes it. + +Defaults for a Mach-O kernel are 1 GiB of RAM and the boot-args +`-v serial=3 debug=0x14e keepsyms=1 serial-device-name=uart0`. To pass your own, the +first word must not start with a dash (`serial=3 -v ...`), or it is read as a flag. +`--initrd`, `--disk` and `--cpus` above 1 are rejected for XNU. + +The bare kernel boots through the pmap and VM bootstrap, zone and IOKit start-up, the +scheduler and IONVRAM, and then panics in `read_random`: the corecrypto kext that +registers the kernel PRNG is not part of the kernel image. That is the kernel's own +behavior on any machine, and the panic message and backtrace appear on the console. +Going further needs a kernelcache with corecrypto prelinked. + +Not modelled: privileged-access-never is stored and saved in `SPSR_EL1` but not +enforced by translation, the physical timer is stored but never fires, floating point +rounds to nearest even and sets no `FPSR` flags, and the GICv3 carries only the +virtual timer (no SPIs, SGIs or group 1). + ## Performance Translating a block costs much more than running it once, and a kernel boot runs most diff --git a/src/emulator/cli/src/boot.rs b/src/emulator/cli/src/boot.rs index a695f4f..8832cc9 100644 --- a/src/emulator/cli/src/boot.rs +++ b/src/emulator/cli/src/boot.rs @@ -1,4 +1,4 @@ -//! Raw Linux arm64 Image boot through Volt's full-system JIT with VirtIO-Blk. +//! Raw Linux arm64 Image, or XNU Mach-O kernel, boot through Volt's full-system JIT. use std::{ io::Write, @@ -7,7 +7,10 @@ use std::{ time::Duration, }; use volt_emulator::devices::BlockBackend; -use volt_emulator::system::{DEFAULT_CMDLINE, IdleMode, RAM_SIZE, SystemConfig, smp::SmpConfig}; +use volt_emulator::system::{ + DEFAULT_CMDLINE, DEFAULT_XNU_CMDLINE, IdleMode, RAM_SIZE, SystemConfig, XNU_RAM_SIZE, + smp::SmpConfig, +}; use winnow_args::{Args, ValueEnum}; #[derive(ValueEnum, Clone, Copy, Debug, Default, PartialEq, Eq)] @@ -33,10 +36,10 @@ impl From for IdleMode { #[derive(Args, Debug)] #[arg( name = "volt-boot", - about = "AArch64 Linux boot with PL011, GICv2, and VirtIO-Blk" + about = "AArch64 Linux or XNU boot with PL011, GICv2/GICv3, and VirtIO-Blk (Linux)" )] struct Cli { - /// Path to the raw Linux AArch64 Image. + /// Path to a raw Linux AArch64 Image, or an XNU kernel (Mach-O). #[arg(positional)] image: PathBuf, @@ -44,7 +47,7 @@ struct Cli { #[arg(positional)] seconds: Option, - /// Kernel command line. + /// Kernel command line (Linux) or boot-args (XNU; start with a non-dash word). #[arg(positional)] cmdline: Option, @@ -189,15 +192,18 @@ fn run(cli: Cli) -> Result<(), String> { }; let seconds = cli.seconds.unwrap_or(20); - let default_cmdline = if cli.disk.is_some() { + let image = std::fs::read(&cli.image).map_err(|e| format!("{}: {e}", cli.image.display()))?; + // A Mach-O image is an XNU kernel; anything else is a raw Linux Image. + let xnu = volt_emulator::system::xnu::is_macho(&image); + let default_cmdline = if xnu { + DEFAULT_XNU_CMDLINE + } else if cli.disk.is_some() { "root=/dev/vda rw console=ttyAMA0 earlycon=pl011,0x9000000 nokaslr loglevel=8" } else { DEFAULT_CMDLINE }; let cmdline = cli.cmdline.unwrap_or_else(|| default_cmdline.to_string()); - let image = std::fs::read(&cli.image).map_err(|e| format!("{}: {e}", cli.image.display()))?; - let initrd_data = match &cli.initrd { Some(path) => Some(load_initrd(path)?), None => None, @@ -205,6 +211,8 @@ fn run(cli: Cli) -> Result<(), String> { let default_ram = if let Some(initrd) = &initrd_data { RAM_SIZE.max((initrd.len() * 3 + (128 << 20)).next_power_of_two()) + } else if xnu { + XNU_RAM_SIZE } else { RAM_SIZE }; diff --git a/src/emulator/src/aarch64/cache.rs b/src/emulator/src/aarch64/cache.rs index d144515..350fbc7 100644 --- a/src/emulator/src/aarch64/cache.rs +++ b/src/emulator/src/aarch64/cache.rs @@ -344,8 +344,17 @@ impl Cache { let word = u32::from_le_bytes(room.try_into().unwrap()); let instruction = match decode(word) { Ok(insn) => insn, + // Run the decoded prefix first: the undecodable word then heads the + // next block, where it is reported at its own address. + Err(_) if count > 4 => { + count -= 4; + break; + } Err(_) => { - eprintln!("cache decode failure at pc {at:x}: word {word:08x}"); + // Architecturally undefined words are the guest's to handle. + if !super::decode::is_undefined(word) { + eprintln!("cache decode failure at pc {at:x}: word {word:08x}"); + } return Err(Error::Decode { word, pc: at }); } }; diff --git a/src/emulator/src/aarch64/compile.rs b/src/emulator/src/aarch64/compile.rs index 279bdbb..b1365ad 100644 --- a/src/emulator/src/aarch64/compile.rs +++ b/src/emulator/src/aarch64/compile.rs @@ -966,6 +966,10 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< l.trap(pc.wrapping_add(4), Trap::Load); } } + Instruction::SimdLiteral(a) => { + let address = l.k(l.u64, pc.wrapping_add(a.offset as u64)); + l.simd_trap_access(address, a.bytes as u64, a.rt, None, None, pc, Trap::Load); + } Instruction::Exclusive(a) => { setup_memory( &l, @@ -995,6 +999,85 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< }, ); } + Instruction::Atomic(a) => { + // Read-modify-write needs a host atomic, so it always takes the machine + // slow path. `dests` receive the words read (31 discards) and `operands` + // are what the machine combines with memory (see `AtomicKind::apply`). + let (dests, operands): ([u8; 2], &[u8]) = match a.kind { + AtomicKind::Casp => ([a.rs, a.rs + 1], &[a.rs, a.rs + 1, a.rt, a.rt + 1]), + AtomicKind::Cas => ([a.rs, 31], &[a.rs, a.rt]), + AtomicKind::LoadPair => ([a.rt, a.rt2], &[]), + AtomicKind::StorePair => ([31, 31], &[a.rt, a.rt2]), + _ => ([a.rt, 31], &[a.rs]), + }; + l.store(offset_of!(Cpu, address), l.reg(l.u64, a.rn, false)); + l.byte(offset_of!(Cpu, width), a.size as u64); + l.byte(offset_of!(Cpu, dest), dests[0] as u64); + l.byte(offset_of!(Cpu, atomic_kind), a.kind as u64); + for (slot, &dest) in dests.iter().enumerate() { + l.byte(offset_of!(Cpu, atomic_dests) + slot, dest as u64); + } + for (slot, ®) in operands.iter().enumerate() { + // Register numbers past 30 read as XZR. + l.store( + offset_of!(Cpu, atomic_operands) + slot * 8, + l.reg(l.u64, reg, true), + ); + } + if a.kind == AtomicKind::StorePair { + l.byte(offset_of!(Cpu, status_dest), a.rs as u64); + } + l.trap(pc.wrapping_add(4), Trap::Atomic); + } + Instruction::SimdStruct(a) => { + use super::simd_struct::StructDesc; + // Element-wise accesses with lane merging need the machine slow path. + let d = a.desc; + let address = l.reg(l.u64, a.rn, false); + l.store(offset_of!(Cpu, address), address); + for (field, value) in [ + (offset_of!(StructDesc, store), u64::from(d.store)), + (offset_of!(StructDesc, shape), d.shape as u64), + (offset_of!(StructDesc, registers), u64::from(d.registers)), + ( + offset_of!(StructDesc, interleaved), + u64::from(d.interleaved), + ), + (offset_of!(StructDesc, esize), u64::from(d.esize)), + (offset_of!(StructDesc, lane), u64::from(d.lane)), + (offset_of!(StructDesc, q), u64::from(d.q)), + (offset_of!(StructDesc, rt), u64::from(d.rt)), + ] { + l.byte(offset_of!(Cpu, structure) + field, value); + } + let step = match a.post { + StructPost::None => None, + StructPost::Immediate => Some(l.k(l.u64, d.bytes())), + StructPost::Register(rm) => Some(l.reg(l.u64, rm, true)), + }; + if let Some(step) = step { + l.store( + offset_of!(Cpu, writeback_value), + l.bin(l.u64, B::Add, address, step), + ); + l.byte(offset_of!(Cpu, writeback_dest), a.rn as u64); + l.byte(offset_of!(Cpu, writeback), 1); + } + l.trap(pc.wrapping_add(4), Trap::SimdStruct); + } + Instruction::AddressTranslate(a) => { + l.store(offset_of!(Cpu, address), l.reg(l.u64, a.rt, true)); + l.store( + offset_of!(Cpu, value), + l.k(l.u64, u64::from(a.write) | u64::from(a.user) << 1), + ); + l.trap(pc.wrapping_add(4), Trap::AddressTranslate); + } + Instruction::Host(word) => { + // The host runs this word: see `host::run`. + l.store(offset_of!(Cpu, value), l.k(l.u64, word as u64)); + l.trap(pc.wrapping_add(4), Trap::Host); + } Instruction::Pair(a) => { let trap = if a.op == MemoryOp::Store { Trap::Store @@ -1022,8 +1105,28 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< Instruction::Sev | Instruction::Sevl => l.byte(offset_of!(Cpu, event_set), 1), Instruction::SimdImm(a) => { let at = offset_of!(Cpu, v) + a.rd as usize * 16; - l.store(at, l.k(l.u64, a.low)); - l.store(at + 8, l.k(l.u64, a.high)); + match a.combine { + ImmCombine::Set => { + l.store(at, l.k(l.u64, a.low)); + l.store(at + 8, l.k(l.u64, a.high)); + } + ImmCombine::Or | ImmCombine::AndNot => { + // A 64-bit form (`q == false`) also clears the upper half. + let (op, imm) = match a.combine { + ImmCombine::Or => (B::BitOr, a.low), + _ => (B::BitAnd, !a.low), + }; + l.store(at, l.imm(l.u64, op, l.load(l.u64, at), imm)); + l.store( + at + 8, + if a.q { + l.imm(l.u64, op, l.load(l.u64, at + 8), imm) + } else { + l.k(l.u64, 0) + }, + ); + } + } } Instruction::SimdMove { rd, rn } => { let at_d = offset_of!(Cpu, v) + rd as usize * 16; @@ -1058,6 +1161,45 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< l.bin(l.u64, B::BitOr, cleared, l.imm(l.u64, B::Shl, value, shift)), ); } + Instruction::SimdInsertElement(a) => { + // Copy one lane of Vn over one lane of Vd. The source is read before + // the store, so `rd == rn` is fine. + let ebits = (1u64 << a.esize) * 8; + let emask = if ebits == 64 { + u64::MAX + } else { + (1u64 << ebits) - 1 + }; + let per_dw = (64 / ebits) as usize; + let at_d = offset_of!(Cpu, v) + a.rd as usize * 16; + let at_n = offset_of!(Cpu, v) + a.rn as usize * 16; + let (src_word, src_shift) = ( + a.src as usize / per_dw, + (a.src as usize % per_dw) as u64 * ebits, + ); + let (dst_word, dst_shift) = ( + a.dst as usize / per_dw, + (a.dst as usize % per_dw) as u64 * ebits, + ); + let source = l.load(l.u64, at_n + src_word * 8); + let value = l.imm( + l.u64, + B::BitAnd, + l.imm(l.u64, B::Shr, source, src_shift), + emask, + ); + let old = l.load(l.u64, at_d + dst_word * 8); + let cleared = l.imm(l.u64, B::BitAnd, old, !(emask << dst_shift)); + l.store( + at_d + dst_word * 8, + l.bin( + l.u64, + B::BitOr, + cleared, + l.imm(l.u64, B::Shl, value, dst_shift), + ), + ); + } Instruction::SimdExtract(a) => { // Move one lane into the low bits of a GPR, zero-extended. let ebits = (1u64 << a.esize) * 8; @@ -1180,97 +1322,6 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< l.store(at_d, lo); l.store(at_d + 8, hi); } - Instruction::SimdPermute(a) => { - use crate::aarch64::decode::PermuteOp; - let at_d = offset_of!(Cpu, v) + a.rd as usize * 16; - let at_n = offset_of!(Cpu, v) + a.rn as usize * 16; - let at_m = offset_of!(Cpu, v) + a.rm as usize * 16; - let vn_lo = l.load(l.u64, at_n); - let vn_hi = l.load(l.u64, at_n + 8); - let vm_lo = l.load(l.u64, at_m); - let vm_hi = l.load(l.u64, at_m + 8); - let (lo, hi) = match a.op { - PermuteOp::Uzp1 | PermuteOp::Zip1 | PermuteOp::Trn1 if a.size == 3 => { - (vn_lo, vm_lo) - } - PermuteOp::Uzp2 | PermuteOp::Zip2 | PermuteOp::Trn2 if a.size == 3 => { - (vn_hi, vm_hi) - } - PermuteOp::Uzp1 if a.size == 2 => { - let lo = l.bin( - l.u64, - B::BitOr, - l.imm(l.u64, B::BitAnd, vn_lo, 0xffff_ffff), - l.imm(l.u64, B::Shl, vn_hi, 32), - ); - let hi = l.bin( - l.u64, - B::BitOr, - l.imm(l.u64, B::BitAnd, vm_lo, 0xffff_ffff), - l.imm(l.u64, B::Shl, vm_hi, 32), - ); - (lo, hi) - } - PermuteOp::Uzp2 if a.size == 2 => { - let lo = l.bin( - l.u64, - B::BitOr, - l.imm(l.u64, B::Shr, vn_lo, 32), - l.imm(l.u64, B::BitAnd, vn_hi, 0xffff_ffff_0000_0000), - ); - let hi = l.bin( - l.u64, - B::BitOr, - l.imm(l.u64, B::Shr, vm_lo, 32), - l.imm(l.u64, B::BitAnd, vm_hi, 0xffff_ffff_0000_0000), - ); - (lo, hi) - } - PermuteOp::Zip1 if a.size == 2 => { - let lo = l.bin( - l.u64, - B::BitOr, - l.imm(l.u64, B::BitAnd, vn_lo, 0xffff_ffff), - l.imm( - l.u64, - B::Shl, - l.imm(l.u64, B::BitAnd, vm_lo, 0xffff_ffff), - 32, - ), - ); - let hi = l.bin( - l.u64, - B::BitOr, - l.imm(l.u64, B::Shr, vn_lo, 32), - l.imm(l.u64, B::BitAnd, vm_lo, 0xffff_ffff_0000_0000), - ); - (lo, hi) - } - PermuteOp::Zip2 if a.size == 2 => { - let lo = l.bin( - l.u64, - B::BitOr, - l.imm(l.u64, B::BitAnd, vn_hi, 0xffff_ffff), - l.imm( - l.u64, - B::Shl, - l.imm(l.u64, B::BitAnd, vm_hi, 0xffff_ffff), - 32, - ), - ); - let hi = l.bin( - l.u64, - B::BitOr, - l.imm(l.u64, B::Shr, vn_hi, 32), - l.imm(l.u64, B::BitAnd, vm_hi, 0xffff_ffff_0000_0000), - ); - (lo, hi) - } - _ => (vn_lo, vm_lo), - }; - l.store(at_d, lo); - l.store(at_d + 8, hi); - } Instruction::SimdCompareZero(a) => { let at_n = offset_of!(Cpu, v) + a.rn as usize * 16; let at_d = offset_of!(Cpu, v) + a.rd as usize * 16; @@ -1412,31 +1463,58 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< Instruction::SimdAlu(a) => { use crate::aarch64::decode::SimdAluOp; // Bitwise logical operations work on whole halves, lane width free. - if matches!(a.op, SimdAluOp::AddV) { - // Horizontal add: sum every lane of the full 128-bit source - // and store the result in lane 0, zeroing the rest. + if matches!( + a.op, + SimdAluOp::AddV | SimdAluOp::UMinV | SimdAluOp::UMaxV | SimdAluOp::UAddLV + ) { + // Horizontal add, minimum, maximum or widening add: reduce every + // lane of the source and store the result in lane 0, zeroing the + // rest. `UAddLV` keeps twice the lane width. let ebits = (1u64 << a.size) * 8; - let emask = if ebits == 64 { - u64::MAX + let mask_of = |bits: u64| { + if bits >= 64 { + u64::MAX + } else { + (1u64 << bits) - 1 + } + }; + let emask = mask_of(ebits); + let result_mask = if a.op == SimdAluOp::UAddLV { + mask_of(ebits * 2) } else { - (1u64 << ebits) - 1 + emask }; let at_n = offset_of!(Cpu, v) + a.rn as usize * 16; let at_d = offset_of!(Cpu, v) + a.rd as usize * 16; let vn_lo = l.load(l.u64, at_n); let vn_hi = l.load(l.u64, at_n + 8); let halves = if a.q { 2 } else { 1 }; - let mut total = l.k(l.u64, 0); + let mut total = if a.op == SimdAluOp::UMinV { + l.k(l.u64, emask) + } else { + l.k(l.u64, 0) + }; for half in 0..halves { let v = if half == 0 { vn_lo } else { vn_hi }; for lane in 0..(8 >> a.size) { let shift = lane as u64 * ebits; let field = l.imm(l.u64, B::BitAnd, l.imm(l.u64, B::Shr, v, shift), emask); - total = l.bin(l.u64, B::Add, total, field); + total = match a.op { + SimdAluOp::UMinV => { + l.sel(l.u64, l.cmp(C::Lt, field, total), field, total) + } + SimdAluOp::UMaxV => { + l.sel(l.u64, l.cmp(C::Gt, field, total), field, total) + } + _ => l.bin(l.u64, B::Add, total, field), + }; } } - l.store(at_d, l.bin(l.u64, B::BitAnd, total, l.k(l.u64, emask))); + l.store( + at_d, + l.bin(l.u64, B::BitAnd, total, l.k(l.u64, result_mask)), + ); l.store(at_d + 8, l.k(l.u64, 0)); } else if matches!( a.op, @@ -2355,6 +2433,15 @@ fn system_offset(r: SystemRegister) -> Option { ElrEl1 => offset_of!(CpuSystem, elr_el1), EsrEl1 => offset_of!(CpuSystem, esr_el1), FarEl1 => offset_of!(CpuSystem, far_el1), + ParEl1 => offset_of!(CpuSystem, par_el1), + IsrEl1 => offset_of!(CpuSystem, isr_el1), + IccSreEl1 => offset_of!(CpuSystem, icc_sre_el1), + IccPmrEl1 => offset_of!(CpuSystem, icc_pmr_el1), + IccBpr0El1 => offset_of!(CpuSystem, icc_bpr0_el1), + IccCtlrEl1 => offset_of!(CpuSystem, icc_ctlr_el1), + IccIgrpen0El1 => offset_of!(CpuSystem, icc_igrpen0_el1), + CntpCtlEl0 => offset_of!(CpuSystem, cntp_ctl_el0), + ActlrEl1 => offset_of!(CpuSystem, actlr_el1), VbarEl1 => offset_of!(CpuSystem, vbar_el1), CpacrEl1 => offset_of!(CpuSystem, cpacr_el1), SctlrEl1 => offset_of!(CpuSystem, sctlr_el1), @@ -2368,6 +2455,8 @@ fn system_offset(r: SystemRegister) -> Option { CntvCtlEl0 => offset_of!(CpuSystem, cntv_ctl_el0), CntvCvalEl0 => offset_of!(CpuSystem, cntv_cval_el0), TpidrEl1 => offset_of!(CpuSystem, tpidr_el1), + ContextidrEl1 => offset_of!(CpuSystem, contextidr_el1), + CsselrEl1 => offset_of!(CpuSystem, csselr_el1), MdscrEl1 => offset_of!(CpuSystem, mdscr_el1), CntkctlEl1 => offset_of!(CpuSystem, cntkctl_el1), OslarEl1 => offset_of!(CpuSystem, oslar_el1), @@ -2430,11 +2519,53 @@ fn lower_system(l: &Lower, a: System) { 2, ), ), + Pan | PanSet => { + // PSTATE.PAN lives in bit 22 of the register form. + let pan = at + offset_of!(CpuSystem, pan); + if a.read { + l.put(a.rt, l.imm(l.u64, B::Shl, l.load(l.u64, pan), 22)); + } else if a.register == PanSet { + l.store(pan, l.k(l.u64, u64::from(a.immediate & 1))); + } else { + let bit = l.imm(l.u64, B::Shr, l.reg(l.u64, a.rt, true), 22); + l.store(pan, l.imm(l.u64, B::BitAnd, bit, 1)); + } + } RazWi => { if a.read { l.put(a.rt, l.k(l.u64, 0)); } } + SpSel | SpSelSet => { + let selected = l.load(l.u8, at + offset_of!(CpuSystem, spsel)); + if a.read { + l.put(a.rt, l.cv(l.u64, selected)); + return; + } + let new = if a.register == SpSelSet { + l.k(l.u8, u64::from(a.immediate & 1)) + } else { + l.cv(l.u8, l.imm(l.u64, B::BitAnd, l.reg(l.u64, a.rt, true), 1)) + }; + // The live stack pointer belongs to bank `SP_EL1` when SPSel is 1 + // and `SP_EL0` when it is 0. Park it in its bank, then load the + // other one. Selecting the bank already in use is a store and a + // reload of the same slot. + let banks = at + offset_of!(CpuSystem, sp_el); + let bank = |select: Value| { + l.sel( + l.ptr, + l.cmp(C::Ne, select, l.k(l.u8, 0)), + l.addr(banks + 8), + l.addr(banks), + ) + }; + let live = l.load(l.u64, Lower::reg_at(31)); + l.store_ptr(bank(selected), live); + let incoming = l.load_ptr(l.u64, bank(new)); + l.store(Lower::reg_at(31), incoming); + l.store(at + offset_of!(CpuSystem, spsel), new); + } DaifSet | DaifClear => { let p = at + offset_of!(CpuSystem, daif); let before = l.load(l.u64, p); @@ -2479,6 +2610,35 @@ fn lower_system(l: &Lower, a: System) { let locked = l.cmp(C::Ne, either, l.k(l.u64, 0)); l.put(a.rt, l.sel(l.u64, locked, l.k(l.u64, 2), l.k(l.u64, 0))); } + CcsidrEl1 => { + // CSSELR bit 0 is InD, bits 3:1 the level - 1; L2 has no separate I-side. + use super::identification::{CCSIDR_L1D, CCSIDR_L1I, CCSIDR_L2}; + let select = l.imm( + l.u64, + B::BitAnd, + l.load(l.u64, at + offset_of!(CpuSystem, csselr_el1)), + 0xf, + ); + let is = |n: u64| l.cmp(C::Eq, select, l.k(l.u64, n)); + let l1_instruction = l.sel(l.u64, is(1), l.k(l.u64, CCSIDR_L1I), l.k(l.u64, CCSIDR_L2)); + l.put( + a.rt, + l.sel(l.u64, is(0), l.k(l.u64, CCSIDR_L1D), l1_instruction), + ); + } + CntvTvalEl0 => { + // The timer value is the signed 32-bit distance to the compare value: + // reads give the low 32 bits of `CVAL - count`, writes `CVAL = count + sext(value)`. + let counter = l.load(l.u64, at + offset_of!(CpuSystem, cntvct_el0)); + let compare = at + offset_of!(CpuSystem, cntv_cval_el0); + if a.read { + let distance = l.bin(l.u64, B::Sub, l.load(l.u64, compare), counter); + l.put(a.rt, l.imm(l.u64, B::BitAnd, distance, 0xffff_ffff)); + } else { + let delta = l.extend(l.reg(l.u64, a.rt, true), Extend::Sxtw); + l.store(compare, l.bin(l.u64, B::Add, counter, delta)); + } + } Daif => { let p = at + offset_of!(CpuSystem, daif); if a.read { @@ -2627,24 +2787,6 @@ mod tests { assert_eq!(cpu.trap, Trap::Svc, "continued past the load"); } #[test] - fn inline_ld1_multi_register_loads_lanes() { - let mut ram = [0u8; 4096]; - // Source bytes 0x00..0x3f at offset 8. - for (i, cell) in ram[8..72].iter_mut().enumerate() { - *cell = i as u8; - } - let mut cpu = Cpu::default(); - map_page(&mut cpu, &mut ram, false); - cpu.x[8] = 0x10008; - // ld1 {v1.16b, v2.16b}, [x8]; svc - run_inline(&mut cpu, &[0x4c40a101, SVC]); - assert_eq!(cpu.v[1][0], 0x0706050403020100); - assert_eq!(cpu.v[1][1], 0x0f0e0d0c0b0a0908); - assert_eq!(cpu.v[2][0], 0x1716151413121110); - assert_eq!(cpu.v[2][1], 0x1f1e1d1c1b1a1918); - assert_eq!(cpu.trap, Trap::Svc, "continued past the load"); - } - #[test] fn inline_load_miss_defers_to_the_machine() { let mut cpu = Cpu::default(); cpu.x[0] = 0x10008; diff --git a/src/emulator/src/aarch64/cpu.rs b/src/emulator/src/aarch64/cpu.rs index 84a9c06..8e37c64 100644 --- a/src/emulator/src/aarch64/cpu.rs +++ b/src/emulator/src/aarch64/cpu.rs @@ -1,4 +1,5 @@ -use super::decode::SignExtend; +use super::decode::{AtomicKind, SignExtend}; +use super::simd_struct::StructDesc; #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] #[repr(u8)] @@ -29,6 +30,15 @@ pub enum Trap { DcZva, Eret, Undefined, + /// LSE atomic (`ldadd`, `cas`, `swp`, ...); see `Cpu::atomic_kind`. + Atomic, + /// SIMD structure load/store (`ld2`, `st4`, `ld1r`, ...); see `Cpu::structure`. + SimdStruct, + /// `at s1e*`: translate `Cpu::address` (`value` bit 0 = write, bit 1 = as EL0) + /// into `PAR_EL1`. + AddressTranslate, + /// An instruction the host runs: `Cpu::value` holds its word (see `host::run`). + Host, } /// Number of data-TLB entries. The JIT indexes with `(va >> 12) & (DTLB_ENTRIES - 1)`. @@ -102,6 +112,16 @@ pub struct Cpu { pub dest: u8, pub load_signed: SignExtend, pub vector_dest: bool, + /// LSE atomic in flight (`Trap::Atomic`): the operation, with `address`, + /// `width` and `dest` as for a load. `atomic_operands` is `[Rs]` for the memory + /// ops, `[Rs, Rt]` for CAS and `[Rs, Rs+1, Rt, Rt+1]` for CASP. + pub atomic_kind: AtomicKind, + pub atomic_operands: [u64; 4], + /// Registers that receive the words read (31 discards): `[Rt, Rt2]` for `LDXP`, + /// `[Rs, Rs+1]` for `CASP`, `[Rt, _]` for the memory operations. + pub atomic_dests: [u8; 2], + /// SIMD structure access in flight (`Trap::SimdStruct`), with `address` as its base. + pub structure: StructDesc, pub exclusive: Exclusive, pub status_dest: u8, @@ -113,6 +133,8 @@ pub struct Cpu { /// stays correct with other vCPUs and inline stores that never touch a /// shared monitor. pub monitor_value: u64, + /// The second word of an exclusive pair (`LDXP`); `monitor_value` is the first. + pub monitor_value2: u64, pub event_set: bool, pub second_pending: bool, pub second_address: u64, @@ -143,6 +165,7 @@ pub struct System { pub elr_el1: u64, pub esr_el1: u64, pub far_el1: u64, + pub par_el1: u64, pub vbar_el1: u64, pub cpacr_el1: u64, pub sctlr_el1: u64, @@ -153,6 +176,8 @@ pub struct System { pub tpidr_el0: u64, pub tpidrro_el0: u64, pub tpidr_el1: u64, + pub contextidr_el1: u64, + pub csselr_el1: u64, pub mdscr_el1: u64, pub cntkctl_el1: u64, pub oslar_el1: u64, @@ -163,6 +188,17 @@ pub struct System { pub cntv_ctl_el0: u64, pub cntv_cval_el0: u64, pub daif: u64, + /// PSTATE.PAN (0 or 1). Saved in `SPSR_EL1` bit 22 and set on exception entry + /// when `SCTLR_EL1.SPAN` is clear; not enforced by translation. + pub pan: u64, + pub isr_el1: u64, + pub icc_sre_el1: u64, + pub icc_pmr_el1: u64, + pub icc_bpr0_el1: u64, + pub icc_ctlr_el1: u64, + pub icc_igrpen0_el1: u64, + pub cntp_ctl_el0: u64, + pub actlr_el1: u64, } impl Default for System { fn default() -> Self { @@ -174,6 +210,7 @@ impl Default for System { elr_el1: 0, esr_el1: 0, far_el1: 0, + par_el1: 0, vbar_el1: 0, cpacr_el1: 0, sctlr_el1: 0, @@ -184,6 +221,8 @@ impl Default for System { tpidr_el0: 0, tpidrro_el0: 0, tpidr_el1: 0, + contextidr_el1: 0, + csselr_el1: 0, mdscr_el1: 0, cntkctl_el1: 0, oslar_el1: 1, @@ -194,6 +233,16 @@ impl Default for System { cntv_ctl_el0: 0, cntv_cval_el0: 0, daif: 0xf, + pan: 0, + isr_el1: 0, + icc_sre_el1: 0, + icc_pmr_el1: 0, + icc_bpr0_el1: 0, + // PRIbits 7: eight priority bits. + icc_ctlr_el1: 7 << 8, + icc_igrpen0_el1: 0, + cntp_ctl_el0: 0, + actlr_el1: 0, } } } @@ -227,6 +276,34 @@ mod tests { } impl Cpu { + /// Complete `LDXP`/`STXP` against the `memory` pair just read at `address`, and + /// return the pair to leave in memory. The monitor is value-based like the + /// single-register exclusives: `STXP` succeeds if the monitor is held on the same + /// address and the pair still has the values `LDXP` saw. The status goes to + /// `status_dest`. + pub fn exclusive_pair(&mut self, kind: AtomicKind, memory: [u64; 2]) -> [u64; 2] { + if kind == AtomicKind::LoadPair { + self.monitor_valid = true; + self.monitor_address = self.address; + self.monitor_width = self.width; + self.monitor_value = memory[0]; + self.monitor_value2 = memory[1]; + return memory; + } + let held = self.monitor_valid + && self.monitor_address == self.address + && self.monitor_width == self.width + && [self.monitor_value, self.monitor_value2] == memory; + self.monitor_valid = false; + if self.status_dest != 31 { + self.x[usize::from(self.status_dest)] = u64::from(!held); + } + if held { + [self.atomic_operands[0], self.atomic_operands[1]] + } else { + memory + } + } /// Reads the byte-sized memory-request fields JIT code stores one byte at a /// time. Plain reads of these adjacent bytes get merged into a single wide /// load, which cannot be store-forwarded from several narrow stores and diff --git a/src/emulator/src/aarch64/decode.rs b/src/emulator/src/aarch64/decode.rs index 4757fd4..8d10f55 100644 --- a/src/emulator/src/aarch64/decode.rs +++ b/src/emulator/src/aarch64/decode.rs @@ -1,3 +1,5 @@ +use super::simd_struct::{Shape, StructDesc}; + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum DecodeError { UnsupportedInstruction, @@ -176,6 +178,13 @@ pub struct MultiplyHigh { pub rd: u8, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct SimdLiteral { + /// 4, 8 or 16. + pub bytes: u8, + pub rt: u8, + pub offset: i64, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Literal { pub size: Size, pub rt: u8, @@ -189,17 +198,134 @@ pub struct Exclusive { pub rt: u8, pub rs: u8, } + +/// Operation of an LSE atomic. Acquire and release ordering are not modelled +/// separately: each host read-modify-write is sequentially consistent. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +#[repr(u8)] +pub enum AtomicKind { + #[default] + Add, + Clr, + Eor, + Set, + SMax, + SMin, + UMax, + UMin, + Swap, + /// `CAS`, `CASB`, `CASH`: compare with `Rs`, store `Rt` on a match. + Cas, + /// `CASP`: compare `Rs`, `Rs+1` with the pair, store `Rt`, `Rt+1` on a match. + Casp, + /// `LDXP`: load a pair and set the exclusive monitor on it. + LoadPair, + /// `STXP`: store a pair if the monitor still holds and memory is unchanged. + StorePair, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Atomic { + pub kind: AtomicKind, + pub size: Size, + pub rs: u8, + pub rn: u8, + pub rt: u8, + /// The second register of `LoadPair` and `StorePair`. + pub rt2: u8, +} +impl AtomicKind { + /// Number of adjacent words the operation reads and writes. + pub fn words(self) -> usize { + if matches!(self, Self::Casp | Self::LoadPair | Self::StorePair) { + 2 + } else { + 1 + } + } + /// `LDXP` or `STXP`: the monitor-based pair accesses. + pub fn is_exclusive_pair(self) -> bool { + matches!(self, Self::LoadPair | Self::StorePair) + } + /// The words left in memory after the operation. `memory` holds the words read + /// (the second one only matters to `Casp`). `operands` are `[Rs]` for the memory + /// ops, `[Rs, Rt]` for `Cas`, and `[Rs, Rs+1, Rt, Rt+1]` for `Casp`. Results are + /// masked to `width` bytes. + pub fn apply(self, memory: [u64; 2], operands: [u64; 4], width: u8) -> [u64; 2] { + let mask = if width >= 8 { + u64::MAX + } else { + (1u64 << (width * 8)) - 1 + }; + let [old, second] = memory.map(|v| v & mask); + let [a, b, c, d] = operands.map(|v| v & mask); + match self { + Self::Add => [old.wrapping_add(a) & mask, second], + Self::Clr => [old & !a, second], + Self::Eor => [old ^ a, second], + Self::Set => [old | a, second], + Self::SMax => [ + if sext_width(old, width) >= sext_width(a, width) { + old + } else { + a + }, + second, + ], + Self::SMin => [ + if sext_width(old, width) <= sext_width(a, width) { + old + } else { + a + }, + second, + ], + Self::UMax => [old.max(a), second], + Self::UMin => [old.min(a), second], + Self::Swap => [a, second], + Self::Cas => [if old == a { b } else { old }, second], + // Exclusive pairs are completed by `Cpu::exclusive_pair`, which owns the + // monitor; memory is not combined with anything here. + Self::LoadPair | Self::StorePair => [old, second], + Self::Casp => { + if old == a && second == b { + [c, d] + } else { + [old, second] + } + } + } + } +} +/// Sign-extend the low `width` bytes of `value`. +fn sext_width(value: u64, width: u8) -> i64 { + let drop = 64 - 8 * u32::from(width); + ((value << drop) as i64) >> drop +} #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Address { pub page: bool, pub rd: u8, pub offset: i64, } +/// How a modified-immediate instruction combines its immediate with `rd`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ImmCombine { + /// `movi`, `mvni`, `fmov`: replace the register. + Set, + /// `orr`: `rd |= imm`. + Or, + /// `bic`: `rd &= !imm`. + AndNot, +} #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct SimdImmediate { pub rd: u8, + /// The expanded immediate: the low and high 64 bits (the high half is zero for + /// a 64-bit `q == false` form, which also clears the register's upper half). pub low: u64, pub high: u64, + pub combine: ImmCombine, + pub q: bool, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct SimdPair { @@ -216,11 +342,30 @@ pub struct SimdMemory { pub bytes: u8, pub rn: u8, pub rt: u8, - /// Consecutive vector registers touched, starting at `rt`. The - /// multi-structure forms use 2..4; the single-register forms use 1. - pub registers: u8, pub addressing: Addressing, } +/// How a SIMD structure access updates its base register afterwards. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum StructPost { + None, + /// Advance by the bytes transferred. + Immediate, + Register(u8), +} +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct SimdStruct { + pub desc: StructDesc, + pub rn: u8, + pub post: StructPost, +} +/// `at s1e{1,0}{r,w}, Xt`: a stage-1 walk whose result lands in `PAR_EL1`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct AddressTranslate { + pub rt: u8, + pub write: bool, + /// Translate as EL0 (`s1e0*`) rather than EL1. + pub user: bool, +} #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct SimdCompareZero { pub rd: u8, @@ -256,6 +401,16 @@ pub struct SimdInsert { pub esize: u8, pub index: u8, } +/// `mov Vd.T[dst], Vn.T[src]`: copy one lane, keeping the rest of the destination. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct SimdInsertElement { + pub rd: u8, + pub rn: u8, + /// Log2 of the element width in bytes. + pub esize: u8, + pub dst: u8, + pub src: u8, +} /// `mov , .[index]` / `umov`: extract one lane into a general register. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct SimdExtract { @@ -274,23 +429,6 @@ pub struct SimdFmov { /// True for 64-bit (`x`/`d`), false for 32-bit (`w`/`s`). pub double: bool, } -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum PermuteOp { - Uzp1, - Uzp2, - Zip1, - Zip2, - Trn1, - Trn2, -} -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct SimdPermute { - pub op: PermuteOp, - pub rd: u8, - pub rn: u8, - pub rm: u8, - pub size: u8, -} /// Three-registers-same integer vector operation (`U`, `size` and `opcode` /// fields verified against `llvm-mc` encodings). #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -339,6 +477,12 @@ pub enum SimdAluOp { /// Horizontal add of all lanes (only the low byte/half/word of the result /// is meaningful; the rest of the destination is zeroed). AddV, + /// Minimum (unsigned) across all lanes; result layout matches `AddV`. + UMinV, + /// Maximum (unsigned) across all lanes; result layout matches `AddV`. + UMaxV, + /// Sum of all lanes into a lane twice as wide (`uaddlv`). + UAddLV, /// Adjacent-lane pair operations between two registers. UMaxP, SMaxP, @@ -381,6 +525,7 @@ pub enum SystemRegister { ElrEl1, EsrEl1, FarEl1, + ParEl1, VbarEl1, CpacrEl1, SctlrEl1, @@ -393,7 +538,9 @@ pub enum SystemRegister { CntvctEl0, CntvCtlEl0, CntvCvalEl0, + CntvTvalEl0, TpidrEl1, + ContextidrEl1, MdscrEl1, CntkctlEl1, OslarEl1, @@ -409,6 +556,9 @@ pub enum SystemRegister { DczidEl0, CntfrqEl0, ClidrEl1, + /// Geometry of the cache selected by `CsselrEl1` (see `identification`). + CcsidrEl1, + CsselrEl1, IdAa64pfr0El1, IdAa64pfr1El1, IdAa64pfr2El1, @@ -429,6 +579,26 @@ pub enum SystemRegister { IdAa64mmfr4El1, DaifSet, DaifClear, + /// `mrs`/`msr SPSel, Xt`. + SpSel, + /// `msr SPSel, #imm`. + SpSelSet, + /// `mrs`/`msr PAN, Xt`: PSTATE.PAN, stored but not enforced by the page walk. + Pan, + /// `msr PAN, #imm`. + PanSet, + /// Pending-interrupt status (bit 7 IRQ, bit 6 FIQ), kept current by the machine. + IsrEl1, + /// GICv3 CPU interface registers, held as plain state: the redistributor and + /// the machine consult them when deciding whether the timer FIQ is delivered. + IccSreEl1, + IccPmrEl1, + IccBpr0El1, + IccCtlrEl1, + IccIgrpen0El1, + /// Stored, but the physical timer never fires. + CntpCtlEl0, + ActlrEl1, Nzcv, RazWi, } @@ -445,6 +615,8 @@ impl SystemRegister { | DczidEl0 | CntfrqEl0 | ClidrEl1 + | CcsidrEl1 + | IsrEl1 | IdAa64pfr0El1 | IdAa64pfr1El1 | IdAa64pfr2El1 @@ -623,11 +795,14 @@ pub enum Instruction { MulLong(MultiplyLong), MulHigh(MultiplyHigh), Literal(Literal), + SimdLiteral(SimdLiteral), Exclusive(Exclusive), + Atomic(Atomic), Clrex, Adr(Address), System(System), DcZva(u8), + AddressTranslate(AddressTranslate), Eret, CacheOp, Tlbi, @@ -652,15 +827,26 @@ pub enum Instruction { SimdImm(SimdImmediate), SimdPair(SimdPair), SimdMemory(SimdMemory), + SimdStruct(SimdStruct), SimdCompareZero(SimdCompareZero), SimdFmov(SimdFmov), - SimdMove { rd: u8, rn: u8 }, - SimdDup { rd: u8, rn: u8, esize: u8, q: bool }, + SimdMove { + rd: u8, + rn: u8, + }, + SimdDup { + rd: u8, + rn: u8, + esize: u8, + q: bool, + }, SimdDupElement(SimdDupElement), + /// An instruction the host executes from its raw word (see `host`). + Host(u32), SimdInsert(SimdInsert), + SimdInsertElement(SimdInsertElement), SimdExtract(SimdExtract), SimdExt(SimdExt), - SimdPermute(SimdPermute), SimdAlu(SimdAlu), SimdTable(SimdTable), Svc, @@ -692,10 +878,13 @@ impl Instruction { self, Memory(_) | Exclusive(_) + | Atomic(_) | Literal(_) + | SimdLiteral(_) | Pair(_) | SimdPair(_) | SimdMemory(_) + | SimdStruct(_) | B(_) | BCond(_) | TestBranch(_) @@ -708,12 +897,27 @@ impl Instruction { | Trap | Isb | DcZva(_) + | AddressTranslate(_) + | Host(_) | Eret | Tlbi ) } } +/// Whether `word` is architecturally UNDEFINED on this CPU, as opposed to an +/// instruction the emulator does not implement. Execution takes an Undefined +/// Instruction exception (XNU's `TRAP_DEBUGGER` word `0xe7ffdeff` relies on it). These +/// are the permanently-undefined `udf` space and the unallocated top-level groups, plus +/// the SVE group: the CPU does not advertise SVE. +pub fn is_undefined(word: u32) -> bool { + match (word >> 25) & 0xf { + // `udf #imm16` is the reserved group with bits 31:16 clear. + 0b0000 => word >> 16 == 0, + 0b0001 | 0b0010 | 0b0011 => true, + _ => false, + } +} fn reg(word: u32, shift: u32) -> u8 { ((word >> shift) & 31) as u8 } @@ -810,6 +1014,73 @@ fn bitmask(word: u32, width: Width) -> Result { } Ok(immediate) } +/// Decode an Advanced SIMD structure load/store word (`word & 0xbe00_0000 == +/// 0x0c00_0000`) into its descriptor and post-index, or `None` for the +/// whole-register LD1/ST1 forms that `SimdMemory` handles. +fn simd_struct(word: u32) -> Result, DecodeError> { + use DecodeError::UnsupportedInstruction as Bad; + let q = word & 0x4000_0000 != 0; + let rm = reg(word, 16); + let size = ((word >> 10) & 3) as u8; + let post = if word & 0x0080_0000 == 0 { + // Without post-index the Rm field is zero. + if rm != 0 { + return Err(Bad); + } + StructPost::None + } else if rm == 31 { + StructPost::Immediate + } else { + StructPost::Register(rm) + }; + let mut desc = StructDesc { + store: word & 0x0040_0000 == 0, + q, + rt: reg(word, 0), + esize: 1 << size, + ..StructDesc::default() + }; + if word & 0x0100_0000 == 0 { + if word & 0x0020_0000 != 0 { + return Err(Bad); + } + (desc.registers, desc.interleaved) = match (word >> 12) & 0xf { + 0b0000 => (4, true), + 0b0100 => (3, true), + 0b1000 => (2, true), + 0b0010 => (4, false), + 0b0110 => (3, false), + 0b1010 => (2, false), + 0b0111 => (1, false), + _ => return Err(Bad), + }; + // `.1d` exists only for LD1/ST1. + if desc.interleaved && !q && size == 3 { + return Err(Bad); + } + if desc.registers == 1 && !matches!(post, StructPost::Register(_)) { + return Ok(None); + } + } else { + // Bits 15:13 give the element scale (and, with R, the structure count); + // the lane index is spread over Q, S and size. + let opcode = (word >> 13) & 7; + let s = u8::from(word & 0x1000 != 0); + let qbit = u8::from(q); + desc.registers = ((opcode & 1) << 1 | u32::from(word & 0x0020_0000 != 0)) as u8 + 1; + let (shape, esize, lane) = match opcode >> 1 { + 0 => (Shape::Lane, 1, qbit << 3 | s << 2 | size), + 1 if size & 1 == 0 => (Shape::Lane, 2, qbit << 2 | s << 1 | size >> 1), + 2 if size == 0 => (Shape::Lane, 4, qbit << 1 | s), + 2 if size == 1 && s == 0 => (Shape::Lane, 8, qbit), + // Replication is a load, and has no lane or S bit. + 3 if !desc.store && s == 0 => (Shape::Replicate, 1 << size, 0), + _ => return Err(Bad), + }; + (desc.shape, desc.esize, desc.lane) = (shape, esize, lane); + } + Ok(Some((desc, post))) +} fn system_register( op0: u32, op1: u32, @@ -821,6 +1092,12 @@ fn system_register( if op0 == 2 && op1 == 0 && crn == 0 && (4..=7).contains(&op2) { return Ok(RazWi); } + if op1 == 0 && crn == 4 && op2 == 5 { + return Ok(SpSelSet); + } + if op1 == 0 && crn == 4 && op2 == 4 { + return Ok(PanSet); + } if op1 == 3 && crn == 4 { if op2 == 6 { return Ok(DaifSet); @@ -838,6 +1115,7 @@ fn system_register( 0x0401 => ElrEl1, 0x0520 => EsrEl1, 0x0600 => FarEl1, + 0x0740 => ParEl1, 0x0c00 => VbarEl1, 0x0102 => CpacrEl1, 0x0100 => SctlrEl1, @@ -850,7 +1128,9 @@ fn system_register( 0x3e02 => CntvctEl0, 0x3e31 => CntvCtlEl0, 0x3e32 => CntvCvalEl0, + 0x3e30 => CntvTvalEl0, 0x0d04 => TpidrEl1, + 0x0d01 => ContextidrEl1, 0x0022 => MdscrEl1, 0x0104 => OslarEl1, 0x0134 => OsdlrEl1, @@ -859,6 +1139,16 @@ fn system_register( 0x3440 => Fpcr, 0x3441 => Fpsr, 0x3421 => Daif, + 0x0420 => SpSel, + 0x0423 => Pan, + 0x0c10 => IsrEl1, + 0x0cc5 => IccSreEl1, + 0x0460 => IccPmrEl1, + 0x0c83 => IccBpr0El1, + 0x0cc4 => IccCtlrEl1, + 0x0cc6 => IccIgrpen0El1, + 0x3e21 => CntpCtlEl0, + 0x0101 => ActlrEl1, 0x0000 => MidrEl1, 0x0005 => MpidrEl1, 0x0006 => RevidrEl1, @@ -866,6 +1156,8 @@ fn system_register( 0x3007 => DczidEl0, 0x3e00 => CntfrqEl0, 0x1001 => ClidrEl1, + 0x1000 => CcsidrEl1, + 0x2000 => CsselrEl1, 0x0040 => IdAa64pfr0El1, 0x0041 => IdAa64pfr1El1, 0x0042 => IdAa64pfr2El1, @@ -1093,9 +1385,109 @@ pub fn decode(word: u32) -> Result { offset: sign_extend(word >> 5, 19) << 2, })); } + // LDR (literal, SIMD&FP): S, D or Q register from a PC-relative address. + if word & 0x3f00_0000 == 0x1c00_0000 { + let bytes = match word >> 30 { + 0 => 4, + 1 => 8, + 2 => 16, + _ => return Err(UnsupportedInstruction), + }; + return Ok(SimdLiteral(self::SimdLiteral { + bytes, + rt: rd, + offset: sign_extend(word >> 5, 19) << 2, + })); + } + // LSE atomics (ARMv8.1): LDADD/LDCLR/LDEOR/LDSET/LDSMAX/LDSMIN/LDUMAX/LDUMIN and + // SWP share one encoding, split by o3 (bit 15) and opc (bits 14:12). LDAPR, the + // RCpc load, is the o3=1, opc=100 slot with Rs unused. + if word & 0x3f20_0c00 == 0x3820_0000 { + let kind = match (word & 0x8000 != 0, (word >> 12) & 7) { + (false, 0) => AtomicKind::Add, + (false, 1) => AtomicKind::Clr, + (false, 2) => AtomicKind::Eor, + (false, 3) => AtomicKind::Set, + (false, 4) => AtomicKind::SMax, + (false, 5) => AtomicKind::SMin, + (false, 6) => AtomicKind::UMax, + (false, 7) => AtomicKind::UMin, + (true, 0) => AtomicKind::Swap, + (true, 4) if rm == 31 => { + return Ok(Memory(self::Memory { + op: MemoryOp::Load, + size: size(word), + rn, + rt: rd, + addressing: Addressing::Offset(0), + signed: SignExtend::None, + })); + } + _ => return Err(UnsupportedInstruction), + }; + return Ok(Atomic(self::Atomic { + kind, + size: size(word), + rs: rm, + rn, + rt: rd, + rt2: 0, + })); + } + // CAS/CASP (ARMv8.1): compare with Rs (the pair Rs, Rs+1 for CASP); store Rt on a match. + if word & 0x3fa0_7c00 == 0x08a0_7c00 { + return Ok(Atomic(self::Atomic { + kind: AtomicKind::Cas, + size: size(word), + rs: rm, + rn, + rt: rd, + rt2: 0, + })); + } + if word & 0xbfa0_7c00 == 0x0820_7c00 { + // CASP register pairs must be even; odd pairs are unallocated. + if rm % 2 != 0 || rd % 2 != 0 { + return Err(UnsupportedInstruction); + } + let size = if word & 0x4000_0000 != 0 { + Size::Double + } else { + Size::Word + }; + return Ok(Atomic(self::Atomic { + kind: AtomicKind::Casp, + size, + rs: rm, + rn, + rt: rd, + rt2: 0, + })); + } if word & 0x3f00_0000 == 0x0800_0000 { if word & 0x0020_0000 != 0 { - return Err(UnsupportedInstruction); + // LDXP/LDAXP/STXP/STLXP: a pair of words or doublewords; the load form has + // no status register. + let load = word & 0x0040_0000 != 0; + if word & 0x8000_0000 == 0 || word & 0x0080_0000 != 0 || (load && rm != 31) { + return Err(UnsupportedInstruction); + } + return Ok(Atomic(self::Atomic { + kind: if load { + AtomicKind::LoadPair + } else { + AtomicKind::StorePair + }, + size: if word & 0x4000_0000 != 0 { + Size::Double + } else { + Size::Word + }, + rs: rm, + rn, + rt: rd, + rt2: reg(word, 10), + })); } let size = size(word); let load = word & 0x0040_0000 != 0; @@ -1132,47 +1524,13 @@ pub fn decode(word: u32) -> Result { rs, })); } - // LD1/ST1 (multiple structures): the opcode field at bits 15:12 selects - // how many consecutive vector registers the structure spans, starting at - // `rt`: `1010` -> 2, `0110` -> 3, `0010` -> 4. The `0111` opcode is the - // single-register form handled below; the remaining encodings are the - // interleaved LD2/LD3/LD4 single-structure forms, which spread one - // element across registers and are not supported here. - if word & 0x3e00_0000 == 0x0c00_0000 && word & 0x0000_f000 != 0x0000_7000 { - let registers = match (word >> 12) & 0xf { - 0b1010 => 2, - 0b0110 => 3, - 0b0010 => 4, - _ => return Err(UnsupportedInstruction), - }; - // Each register holds the full vector width selected by Q. - let bytes = if word & 0x4000_0000 != 0 { 16 } else { 8 }; - let post_index = word & 0x0080_0000 != 0; - let index = (word >> 16) & 0x3f; - let span = (registers as u32 * bytes) as u64; - let addressing = if post_index { - if index != 31 { - return Err(UnsupportedInstruction); - } - Addressing::PostIndex(span as i64) - } else { - if index != 0 { - return Err(UnsupportedInstruction); - } - Addressing::Offset(0) - }; - return Ok(SimdMemory(self::SimdMemory { - op: if word & 0x0040_0000 != 0 { - MemoryOp::Load - } else { - MemoryOp::Store - }, - bytes: bytes as u8, - rn, - rt: rd, - registers: registers as u8, - addressing, - })); + // LD1..LD4 / ST1..ST4 (multiple and single structures) and LD1R..LD4R. + // A single-register LD1/ST1 without a register post-index is left to the + // `SimdMemory` blocks below, which the JIT inlines. + if word & 0xbe00_0000 == 0x0c00_0000 + && let Some((desc, post)) = simd_struct(word)? + { + return Ok(SimdStruct(self::SimdStruct { desc, rn, post })); } // LD1/ST1 (one structure): a full or half-width SIMD register. // @@ -1203,7 +1561,6 @@ pub fn decode(word: u32) -> Result { bytes: bytes as u8, rn, rt: rd, - registers: 1, addressing, })); } @@ -1265,7 +1622,6 @@ pub fn decode(word: u32) -> Result { bytes, rn, rt: rd, - registers: 1, addressing, })); } @@ -1343,6 +1699,15 @@ pub fn decode(word: u32) -> Result { if word & 0xffff_ffe0 == 0xd50b_7420 { return Ok(DcZva(rd)); } + // AT S1E1R / S1E1W / S1E0R / S1E0W: op1 = 0, CRn = 7, CRm = 8, op2 = 0..=3. + if word & 0xffff_ff00 == 0xd508_7800 && (word >> 5) & 7 < 4 { + let op2 = (word >> 5) & 7; + return Ok(AddressTranslate(self::AddressTranslate { + rt: rd, + write: op2 & 1 != 0, + user: op2 & 2 != 0, + })); + } if word & 0xfff8_0000 == 0xd508_0000 { let crn = (word >> 12) & 15; let crm = (word >> 8) & 15; @@ -1367,7 +1732,10 @@ pub fn decode(word: u32) -> Result { return Err(UnsupportedInstruction); } let expected = match register { - SystemRegister::DaifSet | SystemRegister::DaifClear => 0, + SystemRegister::DaifSet + | SystemRegister::DaifClear + | SystemRegister::SpSelSet + | SystemRegister::PanSet => 0, SystemRegister::MdscrEl1 | SystemRegister::OslarEl1 | SystemRegister::OsdlrEl1 @@ -1513,43 +1881,59 @@ pub fn decode(word: u32) -> Result { }, })); } - if word & 0x9f00_0400 == 0x0f00_0400 { - let q = (word >> 30) & 1; - let op = (word >> 29) & 1; + // Advanced SIMD modified immediate: MOVI, MVNI, ORR, BIC and FMOV (vector). + // `AdvSIMDExpandImm` selects the lane pattern from `cmode` and `op`. + // `immh` (bits 22:19) is zero here; a non-zero `immh` is a shift by immediate. + if word & 0x9ff8_0400 == 0x0f00_0400 { + let q = word & 0x4000_0000 != 0; + let op = word & 0x2000_0000 != 0; let cmode = (word >> 12) & 0xf; - let abc = (word >> 16) & 7; - let defgh = (word >> 5) & 0x1f; - let imm8 = (abc << 5) | defgh; + let imm8 = u64::from((((word >> 16) & 7) << 5) | ((word >> 5) & 0x1f)); let rd = (word & 0x1f) as u8; - - let val32: u64 = match cmode { - 0 | 1 => imm8 as u64, - 2 | 3 => (imm8 as u64) << 8, - 4 | 5 => (imm8 as u64) << 16, - 6 | 7 => (imm8 as u64) << 24, - 8 | 9 => { - let v16 = imm8 as u64; - v16 | (v16 << 16) + let twice = |v: u64| v | (v << 32); + let imm = match cmode { + // 32-bit lanes, shifted left by 0, 8, 16 or 24. + 0..=7 => twice(imm8 << (8 * (cmode >> 1))), + // 16-bit lanes, shifted left by 0 or 8. + 8..=11 => { + let lane = imm8 << (8 * ((cmode >> 1) & 1)); + twice(lane | (lane << 16)) } - 10 | 11 => { - let v16 = (imm8 as u64) << 8; - v16 | (v16 << 16) - } - 14 => { - let b = imm8 as u64; - b | (b << 8) | (b << 16) | (b << 24) - } - _ => imm8 as u64, + // 32-bit lanes with the vacated low bits set to ones. + 12 => twice((imm8 << 8) | 0xff), + 13 => twice((imm8 << 16) | 0xffff), + // Bytes: imm8 in every byte, or each bit of imm8 as a 0x00/0xff byte. + 14 if !op => imm8 * 0x0101_0101_0101_0101, + 14 => (0..8).fold(0, |acc, bit| { + acc | (if (imm8 >> bit) & 1 != 0 { 0xff } else { 0 }) << (8 * bit) + }), + // FMOV: the 8-bit float immediate widened to f32 (twice) or f64. + _ if !op => twice(super::fp::expand_immediate(imm8, false)), + _ if q => super::fp::expand_immediate(imm8, true), + _ => return Err(UnsupportedInstruction), }; - - let mut val64 = val32 | (val32 << 32); - if op == 1 { - val64 = !val64; - } - - let low = val64; - let high = if q == 1 { val64 } else { 0 }; - return Ok(SimdImm(SimdImmediate { rd, low, high })); + // Odd cmode below 12 is ORR/BIC; MOVI becomes MVNI (inverted) with op set. + let (value, combine) = if cmode < 12 && cmode & 1 == 1 { + ( + imm, + if op { + ImmCombine::AndNot + } else { + ImmCombine::Or + }, + ) + } else if cmode < 14 && op { + (!imm, ImmCombine::Set) + } else { + (imm, ImmCombine::Set) + }; + return Ok(SimdImm(SimdImmediate { + rd, + low: value, + high: if q { value } else { 0 }, + combine, + q, + })); } if word & 0xbf3f_fc00 == 0x0e20_9800 { let q = word & 0x4000_0000 != 0; @@ -1573,8 +1957,44 @@ pub fn decode(word: u32) -> Result { double: double64, })); } + // Scalar floating point and integer SIMD the JIT does not compile: `host::run` + // executes the word. + if super::host::is_supported(word) { + return Ok(Host(word)); + } + // Across-lanes reductions: bits 21:17 = 11000, bits 11:10 = 10. They share + // the three-same class bits, so they are matched first. ADDV (U=0, opcode + // 11011) and, with U=1, UMAXV (01010), UMINV (11010) and UADDLV (00011). + if word & 0xbf3f_fc00 == 0x0e31_b800 + || word & 0xbf3f_fc00 == 0x2e30_a800 + || word & 0xbf3f_fc00 == 0x2e31_a800 + || word & 0xbf3f_fc00 == 0x2e30_3800 + { + let q = word & 0x4000_0000 != 0; + let size = ((word >> 22) & 3) as u8; + // 64-bit lanes need Q, and `.2s` is reserved for every reduction. + if size == 3 || (size == 2 && !q) { + return Err(UnsupportedInstruction); + } + let op = match (word & 0x2000_0000 != 0, (word >> 12) & 0x1f) { + (false, 0b11011) => SimdAluOp::AddV, + (true, 0b01010) => SimdAluOp::UMaxV, + (true, 0b11010) => SimdAluOp::UMinV, + (true, 0b00011) => SimdAluOp::UAddLV, + _ => return Err(UnsupportedInstruction), + }; + return Ok(SimdAlu(self::SimdAlu { + op, + rd, + rn, + rm: 0, + size, + q, + })); + } // Integer three-registers-same (`U`, `size`, `opcode` verified with llvm-mc). - if word & 0x9f20_0000 == 0x0e20_0000 { + // Bit 10 is 1 in this class; bit-10-clear encodings are handled above. + if word & 0x9f20_0400 == 0x0e20_0400 { use SimdAluOp::*; let q = word & 0x4000_0000 != 0; let u = word & 0x2000_0000 != 0; @@ -1617,13 +2037,11 @@ pub fn decode(word: u32) -> Result { (true, 0b01001) => UQShl, (false, 0b01011) => SQRShl, (true, 0b01011) => UQRShl, - (false, 0b10111) if rm == 0b10001 => AddV, - (false, 0b10111) if rm != 0b10001 => AddP, + (false, 0b10111) => AddP, (true, 0b10100) => UMaxP, (false, 0b10100) => SMaxP, (true, 0b10101) => UMinP, (false, 0b10101) => SMinP, - (false, 0b10111) if size != 3 => AddP, // The two low opcode bits select AND/BIC/ORR/ORN (and, with U=1, // EOR/BSL/BIT/BIF). (false, 0b00011) if size == 0 => And, @@ -1684,6 +2102,21 @@ pub fn decode(word: u32) -> Result { } } } + // `mov Vd.T[i], Vn.T[j]` (`ins`, element): imm5 gives the size and the + // destination index; imm4 gives the source index, scaled by the size. + if word & 0xffe0_8400 == 0x6e00_0400 { + let imm5 = (word >> 16) & 0x1f; + let esize = imm5.trailing_zeros(); + if imm5 != 0 && esize <= 3 { + return Ok(SimdInsertElement(self::SimdInsertElement { + rd, + rn, + esize: esize as u8, + dst: (imm5 >> (esize + 1)) as u8, + src: (((word >> 11) & 0xf) >> esize) as u8, + })); + } + } // `dup` from an element: the imm5 field encodes the element size as the // position of its lowest set bit and the index in the bits above. if word & 0xbf80_fc00 == 0x0e00_0400 { @@ -1741,28 +2174,6 @@ pub fn decode(word: u32) -> Result { let rd = (word & 0x1f) as u8; return Ok(SimdExt(self::SimdExt { rd, rn, rm, imm })); } - if word & 0xbf20_8400 == 0x0e00_0000 { - let op = match (word >> 10) & 0x3f { - 0b000110 => PermuteOp::Uzp1, - 0b001110 => PermuteOp::Uzp2, - 0b000101 => PermuteOp::Zip1, - 0b001101 => PermuteOp::Zip2, - 0b000111 => PermuteOp::Trn1, - 0b001111 => PermuteOp::Trn2, - _ => return Err(UnsupportedInstruction), - }; - let size = ((word >> 22) & 3) as u8; - let rm = ((word >> 16) & 0x1f) as u8; - let rn = ((word >> 5) & 0x1f) as u8; - let rd = (word & 0x1f) as u8; - return Ok(SimdPermute(self::SimdPermute { - op, - rd, - rn, - rm, - size, - })); - } if word & 0x7fff_0000 == 0x5ac0_0000 { let op = match (word >> 10) & 63 { 0 => UnaryOp::Rbit, @@ -2015,6 +2426,8 @@ mod tests { (0xd53be040, CntvctEl0), (0xd53b4220, Daif), (0xd53b4200, Nzcv), + (0xd5390000, CcsidrEl1), + (0xd53a0000, CsselrEl1), ] { assert_eq!( decode(word), @@ -2031,6 +2444,8 @@ mod tests { (0xd51bd040, TpidrEl0), (0xd5184000, SpsrEl1), (0xd518c000, VbarEl1), + (0xd518d020, ContextidrEl1), + (0xd51a0000, CsselrEl1), ] { assert_eq!( decode(word), @@ -2042,7 +2457,10 @@ mod tests { })) ); } - for word in [0xd5380060, 0xd5383000, 0xd53b0000, 0xd5385000, 0xd5180000] { + // Writing the read-only CCSIDR_EL1 is unallocated. + for word in [ + 0xd5380060, 0xd5383000, 0xd53b0000, 0xd5385000, 0xd5180000, 0xd5190000, + ] { assert_eq!(decode(word), Err(DecodeError::UnsupportedInstruction)); } assert_eq!( @@ -2119,7 +2537,6 @@ mod tests { 0x13808000, // W EXTR bit position 32 0x5400000e, // unconditional condition in conditional branch 0xd61f0001, // reserved low branch bits - 0x887f7c00, // pair-exclusive class 0xb8c00000, // reserved signed word to W load 0xf8620800, // unsupported index extension 0x5ac00c00, // W REV64 @@ -2143,6 +2560,94 @@ mod tests { assert_eq!(simd.high, 0); } #[test] + fn simd_modified_immediates_follow_the_architectural_expansion() { + // (word, low, high, combine): the vector `movi`/`mvni`/`orr`/`bic`/`fmov` families. + use ImmCombine::*; + for (word, low, high, combine) in [ + // `movi v0.2d, #0` is cmode 1110 with op 1: a per-bit byte mask, not a NOT. + (0x6f00e400, 0, 0, Set), + ( + 0x6f05e540, + 0xff00_ff00_ff00_ff00, + 0xff00_ff00_ff00_ff00, + Set, + ), + // `movi d1, #imm` is the scalar form: the upper half is cleared. + (0x2f04e421, 0xff00_0000_0000_00ff, 0, Set), + ( + 0x4f02e6a1, + 0x5555_5555_5555_5555, + 0x5555_5555_5555_5555, + Set, + ), + ( + 0x6f000422, + 0xffff_fffe_ffff_fffe, + 0xffff_fffe_ffff_fffe, + Set, + ), + (0x2f006489, 0xfbff_ffff_fbff_ffff, 0, Set), + ( + 0x4f00a468, + 0x0300_0300_0300_0300, + 0x0300_0300_0300_0300, + Set, + ), + // The ones-shifting `msl` forms. + ( + 0x4f00c427, + 0x0000_01ff_0000_01ff, + 0x0000_01ff_0000_01ff, + Set, + ), + ( + 0x4f00d427, + 0x0001_ffff_0001_ffff, + 0x0001_ffff_0001_ffff, + Set, + ), + ( + 0x4f03f605, + 0x3f80_0000_3f80_0000, + 0x3f80_0000_3f80_0000, + Set, + ), + ( + 0x6f04f406, + 0xc000_0000_0000_0000, + 0xc000_0000_0000_0000, + Set, + ), + // `orr`/`bic` combine with the register instead of replacing it. + (0x4f001443, 0x0000_0002_0000_0002, 0x0000_0002_0000_0002, Or), + ( + 0x6f00b424, + 0x0100_0100_0100_0100, + 0x0100_0100_0100_0100, + AndNot, + ), + ] { + let Ok(Instruction::SimdImm(simd)) = decode(word) else { + panic!("{word:08x} is not a simd immediate") + }; + assert_eq!( + (simd.low, simd.high, simd.combine), + (low, high, combine), + "{word:08x}" + ); + } + // `fmov v.2d` needs Q = 1; `fmov d, #imm` is a different instruction. + assert_eq!(decode(0x2f04f406), Err(DecodeError::UnsupportedInstruction)); + // `ushll.8h v1, v0.8b, #0`, `shl.4s` and `sshr.16b` share the class bits of a + // modified immediate but have a non-zero `immh`; they are not `movi`. + for word in [0x2f08a401, 0x4f255420, 0x4f090420, 0x0f0c8420] { + assert!( + matches!(decode(word), Ok(Instruction::Host(_))), + "{word:08x}" + ); + } + } + #[test] fn signed_branches_and_address_generation_preserve_negative_offsets() { assert_eq!(decode(0x17ffffff), Ok(Instruction::B(-4))); assert_eq!( @@ -2213,7 +2718,6 @@ mod tests { bytes: 16, rn: 1, rt: 0, - registers: 1, addressing: Addressing::Offset(0), })) ); @@ -2224,12 +2728,297 @@ mod tests { bytes: 16, rn: 1, rt: 0, - registers: 1, addressing: Addressing::PostIndex(16), })) ); } #[test] + fn only_architecturally_undefined_words_are_undefined() { + // `udf #0`, XNU's `TRAP_DEBUGGER`, and an SVE word (the CPU has no SVE). + for word in [0x0000_0000, 0x0000_ffff, 0xe7ff_deff, 0x0420_0000] { + assert!(is_undefined(word), "{word:08x}"); + assert!(decode(word).is_err(), "{word:08x} is not an instruction"); + } + // Valid instructions, and an instruction that is real but not implemented + // (half-precision `fadd`), are not undefined. + for word in [0xd503_201f, 0x8b00_0000, 0x1ee2_2820] { + assert!(!is_undefined(word), "{word:08x}"); + } + } + #[test] + fn exclusive_pairs_decode_with_both_transfer_registers() { + let pair = |kind, size, rs, rn, rt, rt2| { + Ok(Instruction::Atomic(Atomic { + kind, + size, + rs, + rn, + rt, + rt2, + })) + }; + // ldxp x9, x21, [x8]; stxp w11, x19, x10, [x8]; ldxp w0, wzr, [x0] + // (verified against llvm-mc). + assert_eq!( + decode(0xc87f5509), + pair(AtomicKind::LoadPair, Size::Double, 31, 8, 9, 21) + ); + assert_eq!( + decode(0xc82b2913), + pair(AtomicKind::StorePair, Size::Double, 11, 8, 19, 10) + ); + assert_eq!( + decode(0x887f7c00), + pair(AtomicKind::LoadPair, Size::Word, 31, 0, 0, 31) + ); + // The load form has no status register; byte and halfword pairs do not exist. + assert!(decode(0xc87e5509).is_err()); + assert!(decode(0x487f5509).is_err()); + } + #[test] + fn structure_loads_and_stores_decode_to_descriptors() { + use StructPost::*; + // (word, store, shape, registers, interleaved, esize, lane, q, rt, rn, post), + // every word verified against llvm-mc. + let cases = [ + ( + 0x4c400044, + false, + Shape::Multiple, + 4, + true, + 1, + 0, + true, + 4, + 2, + None, + ), // ld4.16b + ( + 0x4cdf4c20, + false, + Shape::Multiple, + 3, + true, + 8, + 0, + true, + 0, + 1, + Immediate, + ), // ld3.2d, #48 + ( + 0x4c9f8c7e, + true, + Shape::Multiple, + 2, + true, + 8, + 0, + true, + 30, + 3, + Immediate, + ), // st2.2d + ( + 0x0c408481, + false, + Shape::Multiple, + 2, + true, + 2, + 0, + false, + 1, + 4, + None, + ), // ld2.4h + // LD1 of whole registers is a structure access too; only with one register + // and no register post-index does it stay a `SimdMemory`. + ( + 0x4c402d50, + false, + Shape::Multiple, + 4, + false, + 8, + 0, + true, + 16, + 10, + None, + ), + ( + 0x4cc2a800, + false, + Shape::Multiple, + 2, + false, + 4, + 0, + true, + 0, + 0, + Register(2), + ), + ( + 0x0d4090a3, + false, + Shape::Lane, + 1, + false, + 4, + 1, + false, + 3, + 5, + None, + ), // ld1.s[1] + ( + 0x4ddf84a3, + false, + Shape::Lane, + 1, + false, + 8, + 1, + true, + 3, + 5, + Immediate, + ), // ld1.d[1] + ( + 0x4dc61ca3, + false, + Shape::Lane, + 1, + false, + 1, + 15, + true, + 3, + 5, + Register(6), + ), // ld1.b[15] + ( + 0x4d4058a3, + false, + Shape::Lane, + 1, + false, + 2, + 7, + true, + 3, + 5, + None, + ), // ld1.h[7] + ( + 0x4d40cc20, + false, + Shape::Replicate, + 1, + false, + 8, + 0, + true, + 0, + 1, + None, + ), // ld1r.2d + ( + 0x0ddfc020, + false, + Shape::Replicate, + 1, + false, + 1, + 0, + false, + 0, + 1, + Immediate, + ), // ld1r.8b + ( + 0x4d40e820, + false, + Shape::Replicate, + 3, + false, + 4, + 0, + true, + 0, + 1, + None, + ), // ld3r.4s + ( + 0x4d60a020, + false, + Shape::Lane, + 4, + false, + 4, + 2, + true, + 0, + 1, + None, + ), // ld4.s[2] + ( + 0x4d9fb020, + true, + Shape::Lane, + 3, + false, + 4, + 3, + true, + 0, + 1, + Immediate, + ), // st3.s[3] + ( + 0x0d008020, + true, + Shape::Lane, + 1, + false, + 4, + 0, + false, + 0, + 1, + None, + ), // st1.s[0] + ]; + for (word, store, shape, registers, interleaved, esize, lane, q, rt, rn, post) in cases { + let expected = Instruction::SimdStruct(SimdStruct { + desc: StructDesc { + store, + shape, + registers, + interleaved, + esize, + lane, + q, + rt, + }, + rn, + post, + }); + assert_eq!(decode(word), Ok(expected), "{word:08x}"); + } + // A single register with no or an immediate post-index stays the inlinable form. + assert!(matches!(decode(0x4c407020), Ok(Instruction::SimdMemory(_)))); + assert!(matches!(decode(0x4c9f7020), Ok(Instruction::SimdMemory(_)))); + // `ld2`..`ld4` of `.1d` and a stray Rm without post-index are reserved. + assert!(decode(0x0c408c00).is_err()); + assert!(decode(0x4c428c00).is_err()); + // `st1r` does not exist: replicate is a load. + assert!(decode(0x4d00c020).is_err()); + } + #[test] fn decodes_simd_compare_zero() { assert_eq!( decode(0x4e209801), diff --git a/src/emulator/src/aarch64/exception.rs b/src/emulator/src/aarch64/exception.rs index 7935256..b0c5260 100644 --- a/src/emulator/src/aarch64/exception.rs +++ b/src/emulator/src/aarch64/exception.rs @@ -34,6 +34,7 @@ pub mod status { pub fn take(cpu: &mut Cpu, reason: Reason, fault_address: u64) -> u64 { let from = cpu.system.el; cpu.system.spsr_el1 = (u64::from(cpu.flags) & 0xf000_0000) + | (cpu.system.pan << 22) | ((cpu.system.daif & 15) << 6) | (u64::from(from & 3) << 2) | u64::from(cpu.system.spsel); @@ -64,6 +65,10 @@ pub fn take(cpu: &mut Cpu, reason: Reason, fault_address: u64) -> u64 { cpu.system.spsel = true; cpu.sp = cpu.system.sp_el[1]; cpu.system.daif = 15; + // SCTLR_EL1.SPAN clear: entry to EL1 sets PAN. + if cpu.system.sctlr_el1 & (1 << 23) == 0 { + cpu.system.pan = 1; + } cpu.monitor_valid = false; cpu.fault_address = fault_address; cpu.pc = vector; @@ -84,6 +89,7 @@ pub fn eret(cpu: &mut Cpu) -> u64 { let live_stack = if to == 1 && !cpu.system.spsel { 0 } else { to }; cpu.sp = cpu.system.sp_el[live_stack as usize]; cpu.system.daif = (spsr >> 6) & 15; + cpu.system.pan = (spsr >> 22) & 1; cpu.flags = (spsr & 0xf000_0000) as u32; cpu.pc = cpu.system.elr_el1; cpu.pc @@ -93,6 +99,31 @@ pub fn eret(cpu: &mut Cpu) -> u64 { mod tests { use super::*; #[test] + fn pan_is_set_on_entry_unless_span_and_comes_back_with_eret() { + let mut cpu = Cpu::default(); + cpu.pc = 0x4000; + cpu.system.vbar_el1 = 0x10000; + // SCTLR_EL1.SPAN clear: entry sets PAN, and the SPSR records the old value. + cpu.system.pan = 0; + take(&mut cpu, Reason::default(), 0); + assert_eq!(cpu.system.pan, 1); + assert_eq!((cpu.system.spsr_el1 >> 22) & 1, 0); + cpu.system.pan = 1; + take(&mut cpu, Reason::default(), 0); + assert_eq!((cpu.system.spsr_el1 >> 22) & 1, 1); + // SCTLR_EL1.SPAN set: PAN keeps the interrupted value. + cpu.system.sctlr_el1 = 1 << 23; + cpu.system.pan = 0; + take(&mut cpu, Reason::default(), 0); + assert_eq!(cpu.system.pan, 0); + // ERET restores PAN from SPSR bit 22. + for saved in [1u64, 0] { + cpu.system.spsr_el1 = saved << 22; + eret(&mut cpu); + assert_eq!(cpu.system.pan, saved); + } + } + #[test] fn el1t_exception_banks_both_stacks_and_restores_status() { let mut cpu = Cpu::default(); cpu.pc = 0x4000; diff --git a/src/emulator/src/aarch64/fp.rs b/src/emulator/src/aarch64/fp.rs new file mode 100644 index 0000000..176b5b6 --- /dev/null +++ b/src/emulator/src/aarch64/fp.rs @@ -0,0 +1,1000 @@ +//! Scalar floating point (single and double precision) and the vector `fabs`/`fneg` +//! forms, executed with host `f32`/`f64` arithmetic. +//! +//! Decode only classifies a word with [`parse`]; the JIT stores the word in +//! `Cpu::value` and traps, and the machine (or user mode) runs [`run`] on the CPU. +//! Results follow the Arm rules where they differ from the host's: an invalid +//! operation yields the default NaN (positive, quiet), a NaN operand propagates +//! quietened, and `fmax(+0, -0)` is `+0`. +//! +//! Not modelled: half precision, `FPCR` rounding modes and flush-to-zero (the +//! rounding is always to nearest, ties to even), and the `FPSR` exception flags. + +use super::Cpu; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Precision { + Single, + Double, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Unary { + Move, + Abs, + Neg, + Sqrt, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Binary { + Mul, + Div, + Add, + Sub, + Max, + Min, + MaxNum, + MinNum, + Nmul, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Fused { + Madd, + Msub, + Nmadd, + Nmsub, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Op { + Unary { + kind: Unary, + p: Precision, + rd: u8, + rn: u8, + }, + /// `fcvt`: between single and double; `p` is the source precision. + Convert { + p: Precision, + rd: u8, + rn: u8, + }, + Binary { + kind: Binary, + p: Precision, + rd: u8, + rn: u8, + rm: u8, + }, + Fused { + kind: Fused, + p: Precision, + rd: u8, + rn: u8, + rm: u8, + ra: u8, + }, + /// `fcmp`/`fcmpe`; no `rm` compares with zero. + Compare { + p: Precision, + rn: u8, + rm: Option, + }, + CondCompare { + p: Precision, + rn: u8, + rm: u8, + cond: u32, + nzcv: u32, + }, + Select { + p: Precision, + rd: u8, + rn: u8, + rm: u8, + cond: u32, + }, + Immediate { + p: Precision, + rd: u8, + imm8: u64, + }, + /// `scvtf`/`ucvtf` from a general register. + IntToFloat { + signed: bool, + p: Precision, + wide: bool, + rd: u8, + rn: u8, + }, + /// `fcvtzs`/`fcvtzu` to a general register. + FloatToInt { + signed: bool, + p: Precision, + wide: bool, + rd: u8, + rn: u8, + }, + /// `scvtf`/`ucvtf` between SIMD&FP registers (the source holds an integer). + ScalarIntToFloat { + signed: bool, + p: Precision, + rd: u8, + rn: u8, + }, + /// `fcvtzs`/`fcvtzu` between SIMD&FP registers. + ScalarFloatToInt { + signed: bool, + p: Precision, + rd: u8, + rn: u8, + }, + /// Vector `fabs`/`fneg`. + VectorSign { + negate: bool, + double: bool, + q: bool, + rd: u8, + rn: u8, + }, +} + +fn precision(ty: u32) -> Option { + match ty { + 0 => Some(Precision::Single), + 1 => Some(Precision::Double), + // 3 is half precision, which is not modelled; 2 is reserved. + _ => None, + } +} + +/// The 32- or 64-bit pattern of the 8-bit floating-point immediate `imm8` +/// (`VFPExpandImm`), as `fmov` and the vector `fmov` take it. +pub fn expand_immediate(imm8: u64, double: bool) -> u64 { + let sign = imm8 >> 7; + let b = (imm8 >> 6) & 1; + let fraction = imm8 & 0x3f; + if double { + (sign << 63) | ((b ^ 1) << 62) | (b * 0xff << 54) | (fraction << 48) + } else { + (sign << 31) | ((b ^ 1) << 30) | (b * 0x1f << 25) | (fraction << 19) + } +} + +fn parse(word: u32) -> Option { + let rd = (word & 31) as u8; + let rn = ((word >> 5) & 31) as u8; + let rm = ((word >> 16) & 31) as u8; + let ty = (word >> 22) & 3; + // Scalar floating point: `M=0 S=0 01111 0 type 1 ...`. + if word & 0xff20_0c00 == 0x1e20_0800 { + let kind = match (word >> 12) & 15 { + 0 => Binary::Mul, + 1 => Binary::Div, + 2 => Binary::Add, + 3 => Binary::Sub, + 4 => Binary::Max, + 5 => Binary::Min, + 6 => Binary::MaxNum, + 7 => Binary::MinNum, + 8 => Binary::Nmul, + _ => return None, + }; + return Some(Op::Binary { + kind, + p: precision(ty)?, + rd, + rn, + rm, + }); + } + // One-source: bits 14:10 = 10000; the opcode (bits 20:15) includes bit 15. + if word & 0xff20_7c00 == 0x1e20_4000 { + let p = precision(ty)?; + let kind = match (word >> 15) & 0x3f { + 0 => Unary::Move, + 1 => Unary::Abs, + 2 => Unary::Neg, + 3 => Unary::Sqrt, + // `fcvt` to single from double, to double from single. + 4 if p == Precision::Double => return Some(Op::Convert { p, rd, rn }), + 5 if p == Precision::Single => return Some(Op::Convert { p, rd, rn }), + _ => return None, + }; + return Some(Op::Unary { kind, p, rd, rn }); + } + if word & 0xff20_fc07 == 0x1e20_2000 { + let zero = word & 8 != 0; + return Some(Op::Compare { + p: precision(ty)?, + rn, + rm: (!zero).then_some(rm), + }); + } + if word & 0xff20_0c10 == 0x1e20_0400 { + return Some(Op::CondCompare { + p: precision(ty)?, + rn, + rm, + cond: (word >> 12) & 15, + nzcv: word & 15, + }); + } + if word & 0xff20_0c00 == 0x1e20_0c00 { + return Some(Op::Select { + p: precision(ty)?, + rd, + rn, + rm, + cond: (word >> 12) & 15, + }); + } + if word & 0xff20_1fe0 == 0x1e20_1000 { + return Some(Op::Immediate { + p: precision(ty)?, + rd, + imm8: u64::from((word >> 13) & 0xff), + }); + } + if word & 0xff00_0000 == 0x1f00_0000 { + let kind = match ((word >> 21) & 1, (word >> 15) & 1) { + (0, 0) => Fused::Madd, + (0, _) => Fused::Msub, + (_, 0) => Fused::Nmadd, + _ => Fused::Nmsub, + }; + return Some(Op::Fused { + kind, + p: precision(ty)?, + rd, + rn, + rm, + ra: ((word >> 10) & 31) as u8, + }); + } + // Conversions between floating point and general registers. The `fmov` forms + // (rmode 0, opcode 6 and 7) are `SimdFmov`. + if word & 0x7f20_fc00 == 0x1e20_0000 { + let (p, wide) = (precision(ty)?, word & 0x8000_0000 != 0); + return match ((word >> 19) & 3, (word >> 16) & 7) { + (0, 2) => Some(Op::IntToFloat { + signed: true, + p, + wide, + rd, + rn, + }), + (0, 3) => Some(Op::IntToFloat { + signed: false, + p, + wide, + rd, + rn, + }), + (3, 0) => Some(Op::FloatToInt { + signed: true, + p, + wide, + rd, + rn, + }), + (3, 1) => Some(Op::FloatToInt { + signed: false, + p, + wide, + rd, + rn, + }), + _ => None, + }; + } + // The same conversions between SIMD&FP registers: bit 22 is `sz`. + let p = if word & 0x0040_0000 != 0 { + Precision::Double + } else { + Precision::Single + }; + match word & 0xffbf_fc00 { + 0x5e21_d800 => { + return Some(Op::ScalarIntToFloat { + signed: true, + p, + rd, + rn, + }); + } + 0x7e21_d800 => { + return Some(Op::ScalarIntToFloat { + signed: false, + p, + rd, + rn, + }); + } + 0x5ea1_b800 => { + return Some(Op::ScalarFloatToInt { + signed: true, + p, + rd, + rn, + }); + } + 0x7ea1_b800 => { + return Some(Op::ScalarFloatToInt { + signed: false, + p, + rd, + rn, + }); + } + _ => {} + } + // Vector `fabs` (U=0) and `fneg` (U=1). + if word & 0x9fbf_fc00 == 0x0ea0_f800 { + let (q, double) = (word & 0x4000_0000 != 0, word & 0x0040_0000 != 0); + // `.1d` is reserved. + if double && !q { + return None; + } + return Some(Op::VectorSign { + negate: word & 0x2000_0000 != 0, + double, + q, + rd, + rn, + }); + } + None +} + +/// Whether [`run`] implements `word`. +pub fn is_supported(word: u32) -> bool { + parse(word).is_some() +} + +trait Float: Copy + PartialOrd { + const SIGN: u64; + const QUIET: u64; + fn from_u64(bits: u64) -> Self; + fn to_u64(self) -> u64; + fn is_nan(self) -> bool; + fn default_nan() -> Self; + fn add(self, other: Self) -> Self; + fn sub(self, other: Self) -> Self; + fn mul(self, other: Self) -> Self; + fn div(self, other: Self) -> Self; + fn sqrt(self) -> Self; + /// `self * a + b`, rounded once. + fn fma(self, a: Self, b: Self) -> Self; + fn negative(self) -> bool { + self.to_u64() & Self::SIGN != 0 + } + fn neg(self) -> Self { + Self::from_u64(self.to_u64() ^ Self::SIGN) + } + fn quiet(self) -> Self { + Self::from_u64(self.to_u64() | Self::QUIET) + } +} +macro_rules! float { + ($t:ty, $bits:ty, $sign:expr, $quiet:expr, $nan:expr) => { + impl Float for $t { + const SIGN: u64 = $sign; + const QUIET: u64 = $quiet; + fn from_u64(bits: u64) -> Self { + <$t>::from_bits(bits as $bits) + } + fn to_u64(self) -> u64 { + u64::from(<$t>::to_bits(self)) + } + fn is_nan(self) -> bool { + <$t>::is_nan(self) + } + fn default_nan() -> Self { + <$t>::from_bits($nan) + } + fn add(self, other: Self) -> Self { + self + other + } + fn sub(self, other: Self) -> Self { + self - other + } + fn mul(self, other: Self) -> Self { + self * other + } + fn div(self, other: Self) -> Self { + self / other + } + fn sqrt(self) -> Self { + <$t>::sqrt(self) + } + fn fma(self, a: Self, b: Self) -> Self { + <$t>::mul_add(self, a, b) + } + } + }; +} +float!(f32, u32, 1 << 31, 1 << 22, 0x7fc0_0000); +float!(f64, u64, 1 << 63, 1 << 51, 0x7ff8_0000_0000_0000); + +/// An operation that produced a NaN returns the first NaN operand, quietened, or +/// the default NaN when the operands were numbers (the host's NaN has its own sign). +fn nan_or(result: T, operands: &[T]) -> T { + if !result.is_nan() { + return result; + } + operands + .iter() + .find(|o| o.is_nan()) + .map_or(T::default_nan(), |o| o.quiet()) +} + +fn extremum(a: T, b: T, max: bool, numbers_win: bool) -> T { + if a.is_nan() || b.is_nan() { + // `fmaxnm`/`fminnm` prefer the number over a quiet NaN. + if numbers_win && !(a.is_nan() && b.is_nan()) { + return if a.is_nan() { b } else { a }; + } + return nan_or(T::default_nan(), &[a, b]); + } + if a == b { + // Equal values: only the zeros differ, by sign. `max` prefers +0, `min` -0. + return if a.negative() == max { b } else { a }; + } + if (a > b) == max { a } else { b } +} + +fn binary(kind: Binary, a: T, b: T) -> T { + let result = match kind { + Binary::Mul => a.mul(b), + Binary::Div => a.div(b), + Binary::Add => a.add(b), + Binary::Sub => a.sub(b), + Binary::Max => return extremum(a, b, true, false), + Binary::Min => return extremum(a, b, false, false), + Binary::MaxNum => return extremum(a, b, true, true), + Binary::MinNum => return extremum(a, b, false, true), + Binary::Nmul => { + let product = nan_or(a.mul(b), &[a, b]); + return if product.is_nan() { + product + } else { + product.neg() + }; + } + }; + nan_or(result, &[a, b]) +} + +fn fused(kind: Fused, n: T, m: T, a: T) -> T { + // fmadd: a + n*m; fmsub: a - n*m; fnmadd: -a - n*m; fnmsub: n*m - a. + let (n, a) = match kind { + Fused::Madd => (n, a), + Fused::Msub => (n.neg(), a), + Fused::Nmadd => (n.neg(), a.neg()), + Fused::Nmsub => (n, a.neg()), + }; + nan_or(n.fma(m, a), &[n, m, a]) +} + +fn unary(kind: Unary, a: T) -> T { + match kind { + Unary::Move => a, + Unary::Abs if a.is_nan() => a, + Unary::Abs => T::from_u64(a.to_u64() & !T::SIGN), + Unary::Neg => a.neg(), + Unary::Sqrt => nan_or(a.sqrt(), &[a]), + } +} + +/// `NZCV` (in bits 31:28) of comparing `a` with `b`. +fn compare(a: T, b: T) -> u32 { + if a.is_nan() || b.is_nan() { + 0x3000_0000 + } else if a == b { + 0x6000_0000 + } else if a < b { + 0x8000_0000 + } else { + 0x2000_0000 + } +} + +fn condition_holds(cond: u32, flags: u32) -> bool { + let (n, z, c, v) = ( + flags >> 31 & 1 != 0, + flags >> 30 & 1 != 0, + flags >> 29 & 1 != 0, + flags >> 28 & 1 != 0, + ); + let base = match cond >> 1 { + 0 => z, + 1 => c, + 2 => n, + 3 => v, + 4 => c && !z, + 5 => n == v, + 6 => n == v && !z, + _ => true, + }; + // Odd conditions invert, except `nv` (0b1111), which also always holds. + if cond & 1 != 0 && cond != 15 { + !base + } else { + base + } +} + +fn read(cpu: &Cpu, p: Precision, reg: u8) -> u64 { + match p { + Precision::Single => cpu.v[usize::from(reg)][0] & 0xffff_ffff, + Precision::Double => cpu.v[usize::from(reg)][0], + } +} +/// Write a scalar result: the rest of the 128-bit register is zeroed. +fn write(cpu: &mut Cpu, reg: u8, bits: u64) { + cpu.v[usize::from(reg)] = [bits, 0]; +} + +/// Apply `f32` or `f64` code to the registers of precision `p`. +macro_rules! at { + ($p:expr, $t:ident => $body:expr) => { + match $p { + Precision::Single => { + type $t = f32; + $body + } + Precision::Double => { + type $t = f64; + $body + } + } + }; +} + +/// Run the floating-point instruction `word` on `cpu`. `word` must satisfy +/// [`is_supported`]. +pub fn run(cpu: &mut Cpu, word: u32) { + let op = parse(word).expect("an unsupported floating-point word reached the executor"); + match op { + Op::Unary { kind, p, rd, rn } => { + let a = read(cpu, p, rn); + let r = at!(p, T => unary(kind, T::from_u64(a)).to_u64()); + write(cpu, rd, r); + } + Op::Convert { p, rd, rn } => { + let a = read(cpu, p, rn); + let r = match p { + Precision::Double => { + let d = f64::from_bits(a); + // Rounding to single keeps a NaN's class; quiet it like Arm does. + // A NaN keeps its sign and payload (truncated) and is quietened. + let sign = ((a >> 63) as u32) << 31; + u64::from(if d.is_nan() { + sign | 0x7fc0_0000 | ((a >> 29) as u32 & 0x3f_ffff) + } else { + (d as f32).to_bits() + }) + } + Precision::Single => { + let s = f32::from_bits(a as u32); + if s.is_nan() { + f64::from_bits( + (((a >> 31) & 1) << 63) + | 0x7ff8_0000_0000_0000 + | ((a & 0x3f_ffff) << 29), + ) + .to_bits() + } else { + f64::from(s).to_bits() + } + } + }; + write(cpu, rd, r); + } + Op::Binary { + kind, + p, + rd, + rn, + rm, + } => { + let (a, b) = (read(cpu, p, rn), read(cpu, p, rm)); + let r = at!(p, T => binary(kind, T::from_u64(a), T::from_u64(b)).to_u64()); + write(cpu, rd, r); + } + Op::Fused { + kind, + p, + rd, + rn, + rm, + ra, + } => { + let (n, m, a) = (read(cpu, p, rn), read(cpu, p, rm), read(cpu, p, ra)); + let r = + at!(p, T => fused(kind, T::from_u64(n), T::from_u64(m), T::from_u64(a)).to_u64()); + write(cpu, rd, r); + } + Op::Compare { p, rn, rm } => { + let a = read(cpu, p, rn); + let b = rm.map_or(0, |rm| read(cpu, p, rm)); + let nzcv = at!(p, T => compare(T::from_u64(a), T::from_u64(b))); + cpu.flags = (cpu.flags & 0x0fff_ffff) | nzcv; + } + Op::CondCompare { + p, + rn, + rm, + cond, + nzcv, + } => { + let flags = if condition_holds(cond, cpu.flags) { + let (a, b) = (read(cpu, p, rn), read(cpu, p, rm)); + at!(p, T => compare(T::from_u64(a), T::from_u64(b))) + } else { + nzcv << 28 + }; + cpu.flags = (cpu.flags & 0x0fff_ffff) | flags; + } + Op::Select { + p, + rd, + rn, + rm, + cond, + } => { + let source = if condition_holds(cond, cpu.flags) { + rn + } else { + rm + }; + let bits = read(cpu, p, source); + write(cpu, rd, bits); + } + Op::Immediate { p, rd, imm8 } => { + write(cpu, rd, expand_immediate(imm8, p == Precision::Double)); + } + Op::IntToFloat { + signed, + p, + wide, + rd, + rn, + } => { + let raw = if rn == 31 { 0 } else { cpu.x[usize::from(rn)] }; + let r = int_to_float(raw, signed, wide, p); + write(cpu, rd, r); + } + Op::FloatToInt { + signed, + p, + wide, + rd, + rn, + } => { + let a = read(cpu, p, rn); + let r = float_to_int(a, signed, wide, p); + if rd != 31 { + cpu.x[usize::from(rd)] = r; + } + } + Op::ScalarIntToFloat { signed, p, rd, rn } => { + let raw = cpu.v[usize::from(rn)][0]; + let r = int_to_float(raw, signed, p == Precision::Double, p); + write(cpu, rd, r); + } + Op::ScalarFloatToInt { signed, p, rd, rn } => { + let a = read(cpu, p, rn); + let r = float_to_int(a, signed, p == Precision::Double, p); + write(cpu, rd, r); + } + Op::VectorSign { + negate, + double, + q, + rd, + rn, + } => { + let sign = if double { + 1 << 63 + } else { + 0x8000_0000_8000_0000 + }; + let source = cpu.v[usize::from(rn)]; + let apply = |half: u64| if negate { half ^ sign } else { half & !sign }; + cpu.v[usize::from(rd)] = [apply(source[0]), if q { apply(source[1]) } else { 0 }]; + } + } +} + +/// The float of precision `p` nearest to the `wide` (64-bit) or 32-bit integer in +/// `raw`, read as signed or unsigned. Integer-to-float casts round to nearest even. +fn int_to_float(raw: u64, signed: bool, wide: bool, p: Precision) -> u64 { + let value: i128 = match (signed, wide) { + (true, true) => i128::from(raw as i64), + (true, false) => i128::from(raw as u32 as i32), + (false, true) => i128::from(raw), + (false, false) => i128::from(raw as u32), + }; + match p { + Precision::Single => u64::from((value as f32).to_bits()), + Precision::Double => (value as f64).to_bits(), + } +} + +/// Truncate the float in `bits` toward zero into a 32- or 64-bit integer, saturating +/// out-of-range values and turning NaN into zero (which Rust's `as` already does). +fn float_to_int(bits: u64, signed: bool, wide: bool, p: Precision) -> u64 { + let value = match p { + Precision::Single => f64::from(f32::from_bits(bits as u32)), + Precision::Double => f64::from_bits(bits), + }; + match (signed, wide) { + (true, true) => (value as i64) as u64, + (true, false) => u64::from(value as i32 as u32), + (false, true) => value as u64, + (false, false) => u64::from(value as u32), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn cpu() -> Cpu { + Cpu::default() + } + fn set64(cpu: &mut Cpu, reg: usize, value: f64) { + cpu.v[reg] = [value.to_bits(), u64::MAX]; + } + fn set32(cpu: &mut Cpu, reg: usize, value: f32) { + cpu.v[reg] = [u64::from(value.to_bits()), u64::MAX]; + } + fn get64(cpu: &Cpu, reg: usize) -> f64 { + f64::from_bits(cpu.v[reg][0]) + } + fn get32(cpu: &Cpu, reg: usize) -> f32 { + f32::from_bits(cpu.v[reg][0] as u32) + } + + #[test] + fn arithmetic_rounds_in_the_register_precision_and_clears_the_upper_half() { + let mut c = cpu(); + set32(&mut c, 4, 16_777_216.0); + set32(&mut c, 5, 1.0); + // fadd s3, s4, s5: 2^24 + 1 is not representable in single precision. + run(&mut c, 0x1e252883); + assert_eq!(get32(&c, 3), 16_777_216.0); + assert_eq!(c.v[3][1], 0, "scalar writes clear bits 127:64"); + set64(&mut c, 1, 1.5); + set64(&mut c, 2, 4.0); + run(&mut c, 0x1e620820); // fmul d0, d1, d2 + assert_eq!(get64(&c, 0), 6.0); + set64(&mut c, 7, 1.0); + set64(&mut c, 8, 4.0); + run(&mut c, 0x1e6838e6); // fsub d6, d7, d8 + assert_eq!(get64(&c, 6), -3.0); + set32(&mut c, 10, 1.0); + set32(&mut c, 11, 4.0); + run(&mut c, 0x1e2b1949); // fdiv s9, s10, s11 + assert_eq!(get32(&c, 9), 0.25); + } + + #[test] + fn invalid_operations_give_the_default_nan_and_nan_operands_propagate_quietened() { + let mut c = cpu(); + set64(&mut c, 1, 0.0); + set64(&mut c, 2, 0.0); + run(&mut c, 0x1e621820); // fdiv d0, d1, d2 = 0/0 + assert_eq!(c.v[0][0], 0x7ff8_0000_0000_0000, "positive default NaN"); + // A signalling NaN operand comes back quietened. + c.v[1][0] = 0x7ff0_0000_0000_0001; + set64(&mut c, 2, 1.0); + run(&mut c, 0x1e620820); // fmul + assert_eq!(c.v[0][0], 0x7ff8_0000_0000_0001); + } + + #[test] + fn max_min_follow_the_zero_and_nan_rules() { + let mut c = cpu(); + let (zero, neg_zero) = (0.0f64, -0.0f64); + set64(&mut c, 1, zero); + set64(&mut c, 2, neg_zero); + run(&mut c, 0x1e624820); // fmax d0, d1, d2 + assert!(!get64(&c, 0).is_sign_negative(), "fmax(+0, -0) = +0"); + // fmin s0, s1, s2 on (+0, -0): -0. + set32(&mut c, 1, 0.0); + set32(&mut c, 2, -0.0); + run(&mut c, 0x1e225820); + assert!(get32(&c, 0).is_sign_negative()); + // fmax with a NaN is NaN; fmaxnm prefers the number. + set64(&mut c, 1, f64::NAN); + set64(&mut c, 2, 3.0); + run(&mut c, 0x1e624820); + assert!(get64(&c, 0).is_nan()); + run(&mut c, 0x1e626820); // fmaxnm d0, d1, d2 + assert_eq!(get64(&c, 0), 3.0); + // fminnm s0, s1, s2 with a NaN in the second operand. + set32(&mut c, 1, 2.0); + set32(&mut c, 2, f32::NAN); + run(&mut c, 0x1e227820); + assert_eq!(get32(&c, 0), 2.0); + } + + #[test] + fn fused_multiply_add_rounds_once() { + let mut c = cpu(); + let x = 1.0 + 2f64.powi(-30); + set64(&mut c, 1, x); + set64(&mut c, 2, x); + set64(&mut c, 3, -(1.0 + 2f64.powi(-29))); + run(&mut c, 0x1f420c20); // fmadd d0, d1, d2, d3 = d3 + d1*d2 + assert_eq!( + get64(&c, 0), + 2f64.powi(-60), + "an unfused multiply-add gives 0" + ); + // fmsub s0, s1, s2, s3 = s3 - s1*s2. + set32(&mut c, 1, 2.0); + set32(&mut c, 2, 3.0); + set32(&mut c, 3, 10.0); + run(&mut c, 0x1f028c20); + assert_eq!(get32(&c, 0), 4.0); + // fnmadd d0 = -d3 - d1*d2; fnmsub s0 = s1*s2 - s3. + set64(&mut c, 1, 2.0); + set64(&mut c, 2, 3.0); + set64(&mut c, 3, 1.0); + run(&mut c, 0x1f620c20); + assert_eq!(get64(&c, 0), -7.0); + set32(&mut c, 1, 2.0); + set32(&mut c, 2, 3.0); + set32(&mut c, 3, 1.0); + run(&mut c, 0x1f228c20); + assert_eq!(get32(&c, 0), 5.0); + } + + #[test] + fn compares_set_the_arm_flags_and_conditional_forms_select_by_them() { + let mut c = cpu(); + c.flags = 0x0abc_def0; + let cases = [ + (1.0, 2.0, 0x8000_0000), + (2.0, 2.0, 0x6000_0000), + (3.0, 2.0, 0x2000_0000), + (f64::NAN, 2.0, 0x3000_0000), + ]; + for (a, b, nzcv) in cases { + set64(&mut c, 0, a); + set64(&mut c, 1, b); + run(&mut c, 0x1e612000); // fcmp d0, d1 + assert_eq!(c.flags, 0x0abc_def0 & 0x0fff_ffff | nzcv, "{a} vs {b}"); + } + // fcmp s0, #0.0 ignores Rm. + set32(&mut c, 0, 0.0); + run(&mut c, 0x1e202008); + assert_eq!(c.flags >> 28, 0x6); + // fccmp d0, d1, #4, eq: with Z set it compares, otherwise it loads #4. + set64(&mut c, 0, 1.0); + set64(&mut c, 1, 2.0); + c.flags = 0x4000_0000; + run(&mut c, 0x1e610404); + assert_eq!(c.flags >> 28, 0x8); + c.flags = 0; + run(&mut c, 0x1e610404); + assert_eq!(c.flags >> 28, 0x4); + // fcsel d0, d1, d2, ne. + set64(&mut c, 1, 7.0); + set64(&mut c, 2, 9.0); + c.flags = 0; + run(&mut c, 0x1e621c20); + assert_eq!(get64(&c, 0), 7.0); + c.flags = 0x4000_0000; + run(&mut c, 0x1e621c20); + assert_eq!(get64(&c, 0), 9.0); + } + + #[test] + fn integer_conversions_saturate_truncate_and_round_to_nearest_even() { + let mut c = cpu(); + // fcvtzs x0, d1 truncates toward zero, saturates, and sends NaN to zero. + for (input, expected) in [ + (-2.9, (-2i64) as u64), + (1e30, i64::MAX as u64), + (-1e30, i64::MIN as u64), + (f64::NAN, 0), + ] { + set64(&mut c, 1, input); + run(&mut c, 0x9e780020); + assert_eq!(c.x[0], expected, "{input}"); + } + // fcvtzu w0, d1: negative inputs give 0; the result is zero-extended. + set64(&mut c, 1, -5.0); + run(&mut c, 0x1e790020); + assert_eq!(c.x[0], 0); + set64(&mut c, 1, 5e9); + run(&mut c, 0x1e790020); + assert_eq!(c.x[0], u64::from(u32::MAX)); + // fcvtzs w0, s1 of a negative value is a 32-bit two's complement, zero-extended. + set32(&mut c, 1, -1.5); + run(&mut c, 0x1e380020); + assert_eq!(c.x[0], 0xffff_ffff); + // ucvtf s0, x1: 2^24 + 1 rounds to even (2^24); scvtf d0, x1 of a negative. + c.x[1] = (1 << 24) + 1; + run(&mut c, 0x9e230020); + assert_eq!(get32(&c, 0), 16_777_216.0); + c.x[1] = (-3i64) as u64; + run(&mut c, 0x9e620020); + assert_eq!(get64(&c, 0), -3.0); + // ucvtf d0, w1 reads only the low 32 bits. + c.x[1] = 0xffff_ffff_0000_0007; + run(&mut c, 0x1e630020); + assert_eq!(get64(&c, 0), 7.0); + // The SIMD&FP-register forms: ucvtf s0, s1 and fcvtzs s0, s1. + c.v[1] = [0x8000_0000, 0]; + run(&mut c, 0x7e21d820); + assert_eq!(get32(&c, 0), 2_147_483_648.0); + set32(&mut c, 1, -7.9); + run(&mut c, 0x5ea1b820); + assert_eq!(c.v[0], [(-7i32) as u32 as u64, 0]); + } + + #[test] + fn precision_conversion_immediates_and_unary_forms() { + let mut c = cpu(); + set64(&mut c, 1, 0.1); + run(&mut c, 0x1e624020); // fcvt s0, d1 + assert_eq!(get32(&c, 0), 0.1f32); + set32(&mut c, 1, 0.5); + run(&mut c, 0x1e22c020); // fcvt d0, s1 + assert_eq!(get64(&c, 0), 0.5); + run(&mut c, 0x1e6e1000); // fmov d0, #1.0 + assert_eq!(get64(&c, 0), 1.0); + run(&mut c, 0x1e309001); // fmov s1, #-2.5 + assert_eq!(get32(&c, 1), -2.5); + set64(&mut c, 1, -16.0); + run(&mut c, 0x1e614020); // fneg d0, d1 + assert_eq!(get64(&c, 0), 16.0); + set64(&mut c, 1, 16.0); + run(&mut c, 0x1e61c020); // fsqrt d0, d1 + assert_eq!(get64(&c, 0), 4.0); + set32(&mut c, 1, -3.0); + run(&mut c, 0x1e20c020); // fabs s0, s1 + assert_eq!(get32(&c, 0), 3.0); + // sqrt of a negative is the default NaN. + set64(&mut c, 1, -1.0); + run(&mut c, 0x1e61c020); + assert_eq!(c.v[0][0], 0x7ff8_0000_0000_0000); + } + + #[test] + fn vector_fneg_and_fabs_flip_every_lane_and_q_clear_zeroes_the_upper_half() { + let mut c = cpu(); + c.v[1] = [0x3ff0_0000_0000_0000, 0xc000_0000_0000_0000]; + run(&mut c, 0x6ee0f820); // fneg v0.2d, v1.2d + assert_eq!(c.v[0], [0xbff0_0000_0000_0000, 0x4000_0000_0000_0000]); + c.v[1] = [0xbf80_0000_c000_0000, 0x3f80_0000_3f80_0000]; + run(&mut c, 0x4ea0f820); // fabs v0.4s, v1.4s + assert_eq!(c.v[0], [0x3f80_0000_4000_0000, 0x3f80_0000_3f80_0000]); + run(&mut c, 0x2ea0f820); // fneg v0.2s, v1.2s + assert_eq!(c.v[0], [0x3f80_0000_4000_0000, 0]); + } + + #[test] + fn unsupported_encodings_are_not_claimed() { + // fmov d0, x1 and fmov x0, d1 are `SimdFmov`, not conversions. + assert!(!is_supported(0x9e670020)); + assert!(!is_supported(0x9e660020)); + // Half precision (type 3) and reserved type 2. + assert!(!is_supported(0x1ee22820)); + assert!(!is_supported(0x1ea22820)); + // fneg v0.1d is reserved. + assert!(!is_supported(0x2ee0f820)); + assert!(is_supported(0x1e620820)); + } + + #[test] + fn the_immediate_expansion_matches_the_architectural_pattern() { + assert_eq!(expand_immediate(0x70, false), 0x3f80_0000); + assert_eq!(expand_immediate(0x70, true), 0x3ff0_0000_0000_0000); + assert_eq!(expand_immediate(0xf0, true), 0xbff0_0000_0000_0000); + } +} diff --git a/src/emulator/src/aarch64/host.rs b/src/emulator/src/aarch64/host.rs new file mode 100644 index 0000000..e884992 --- /dev/null +++ b/src/emulator/src/aarch64/host.rs @@ -0,0 +1,19 @@ +//! Instructions the JIT does not compile. It stores the raw word in `Cpu::value` +//! and traps; the machine (or user mode) then runs the word on the host here. +//! Floating point and the wider integer SIMD families live in [`fp`] and [`simd`]. + +use super::{Cpu, fp, simd}; + +/// Whether [`run`] implements `word`. +pub fn is_supported(word: u32) -> bool { + fp::is_supported(word) || simd::is_supported(word) +} + +/// Run `word` on `cpu`. `word` must satisfy [`is_supported`]. +pub fn run(cpu: &mut Cpu, word: u32) { + if fp::is_supported(word) { + fp::run(cpu, word) + } else { + simd::run(cpu, word) + } +} diff --git a/src/emulator/src/aarch64/identification.rs b/src/emulator/src/aarch64/identification.rs index e8b50b4..46a7ba2 100644 --- a/src/emulator/src/aarch64/identification.rs +++ b/src/emulator/src/aarch64/identification.rs @@ -1,6 +1,21 @@ use super::decode::SystemRegister; #[allow(non_upper_case_globals)] pub const counter_hz: u64 = 62_500_000; +/// Cache hierarchy `CLIDR_EL1` reports: split L1, unified L2, 64-byte lines. +/// LoC = 2, LoU = 1. +pub const CLIDR: u64 = 3 | 4 << 3 | 2 << 24 | 1 << 27; +/// `CCSIDR_EL1` (no CCIDX) for the cache `CSSELR_EL1` selects: cache type in +/// bits 31:28 (4 = write-back, 2 = read-allocate), sets - 1, ways - 1, and a +/// `LineSize` of 2 (64 bytes). +const fn ccsidr(cache_type: u64, sets: u64, ways: u64) -> u64 { + cache_type << 28 | (sets - 1) << 13 | (ways - 1) << 3 | 2 +} +/// L1 data: 32 KiB, 4-way. +pub const CCSIDR_L1D: u64 = ccsidr(4, 128, 4); +/// L1 instruction: 32 KiB, 4-way. +pub const CCSIDR_L1I: u64 = ccsidr(2, 128, 4); +/// L2 unified: 1 MiB, 8-way. +pub const CCSIDR_L2: u64 = ccsidr(4, 2048, 8); pub fn value(register: SystemRegister) -> Option { use SystemRegister::*; Some(match register { @@ -14,10 +29,12 @@ pub fn value(register: SystemRegister) -> Option { IdAa64pfr0El1 => (0xf << 20) | (0xf << 16) | (1 << 4) | 1, IdAa64dfr0El1 => 6, IdAa64mmfr0El1 => (0xf << 24) | 2, - RevidrEl1 | ClidrEl1 | IdAa64pfr1El1 | IdAa64pfr2El1 | IdAa64zfr0El1 | IdAa64smfr0El1 - | IdAa64fpfr0El1 | IdAa64isar3El1 | AidrEl1 | IdAa64dfr1El1 | IdAa64isar0El1 - | IdAa64isar1El1 | IdAa64isar2El1 | IdAa64mmfr1El1 | IdAa64mmfr2El1 | IdAa64mmfr3El1 - | IdAa64mmfr4El1 => 0, + IdAa64isar0El1 => 2 << 20, + IdAa64isar1El1 => 1 << 20, + ClidrEl1 => CLIDR, + RevidrEl1 | IdAa64pfr1El1 | IdAa64pfr2El1 | IdAa64zfr0El1 | IdAa64smfr0El1 + | IdAa64fpfr0El1 | IdAa64isar3El1 | AidrEl1 | IdAa64dfr1El1 | IdAa64isar2El1 + | IdAa64mmfr1El1 | IdAa64mmfr2El1 | IdAa64mmfr3El1 | IdAa64mmfr4El1 => 0, _ => return None, }) } @@ -38,8 +55,26 @@ mod tests { assert_eq!(mmfr0 & 15, 2); assert_eq!((mmfr0 >> 24) & 15, 15); assert_eq!((mmfr0 >> 20) & 15, 0); - assert_eq!(value(IdAa64isar0El1), Some(0)); + // FEAT_LSE (Atomics = 2) and FEAT_LRCPC (LDAPR) are implemented. + assert_eq!(value(IdAa64isar0El1), Some(2 << 20)); + assert_eq!(value(IdAa64isar1El1), Some(1 << 20)); assert_eq!(value(MpidrEl1), Some(1 << 31)); assert_eq!(value(Ttbr0El1), None); } + #[test] + fn cache_geometry_decodes_like_xnu_cpuid() { + // The size arithmetic of `osfmk/arm/cpuid.c`. + let size = |ccsidr: u64| { + let line = 4 * (1u64 << ((ccsidr & 7) + 2)); + (((ccsidr >> 13) & 0x7fff) + 1) * line * (((ccsidr >> 3) & 0x3ff) + 1) + }; + assert_eq!(size(CCSIDR_L1D), 32 << 10); + assert_eq!(size(CCSIDR_L1I), 32 << 10); + assert_eq!(size(CCSIDR_L2), 1 << 20); + // Separate L1 I/D (Ctype1 = 3), unified L2 (Ctype2 = 4), nothing beyond. + assert_eq!(CLIDR & 0x1ff, 3 | 4 << 3); + // XNU treats c_type 4 as write-back; the L1 data and L2 caches are. + assert_eq!(CCSIDR_L1D >> 28, 4); + assert_eq!(CCSIDR_L2 >> 28, 4); + } } diff --git a/src/emulator/src/aarch64/mod.rs b/src/emulator/src/aarch64/mod.rs index 9ad2fc4..db05489 100644 --- a/src/emulator/src/aarch64/mod.rs +++ b/src/emulator/src/aarch64/mod.rs @@ -4,7 +4,11 @@ pub mod compile; pub mod cpu; pub mod decode; pub mod exception; +pub mod fp; +pub mod host; pub mod identification; +pub mod simd; +pub mod simd_struct; pub mod translate; pub use cache::Cache; diff --git a/src/emulator/src/aarch64/simd.rs b/src/emulator/src/aarch64/simd.rs new file mode 100644 index 0000000..3cced27 --- /dev/null +++ b/src/emulator/src/aarch64/simd.rs @@ -0,0 +1,1163 @@ +//! Integer Advanced SIMD instructions executed on the host from the raw word: +//! the vector permutes (`uzp`, `zip`, `trn`), the two-register-miscellaneous +//! group (`rev`, `cnt`, `mvn`, compares against zero, `abs`/`neg`, the narrowing +//! `xtn` family and the pairwise-long adds), the three-registers-different long and +//! wide arithmetic (`uaddl`, `umull`, ...), and the 64-bit scalar forms. +//! +//! Each register is a `u128`, each lane a bit field of it, so one lane loop serves +//! every element size. Decode only classifies a word with [`is_supported`]; the +//! JIT traps and the machine (or user mode) calls [`run`]. + +use super::Cpu; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Permute { + Uzp1, + Uzp2, + Zip1, + Zip2, + Trn1, + Trn2, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Zero { + Gt, + Ge, + Le, + Lt, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Narrow { + Plain, + SignedSat, + UnsignedSat, + SignedToUnsignedSat, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Long { + Add, + AddWide, + Sub, + SubWide, + Abd, + Mul, + Mla, + Mls, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Scalar { + Add, + Sub, + Gt, + Hi, + Ge, + Hs, + Eq, + Tst, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Op { + Permute { + kind: Permute, + size: u32, + q: bool, + rd: u8, + rn: u8, + rm: u8, + }, + /// `rev16`/`rev32`/`rev64`: reverse the lanes inside each group of `group` bits. + Reverse { + group: u32, + size: u32, + q: bool, + rd: u8, + rn: u8, + }, + Count { + q: bool, + rd: u8, + rn: u8, + }, + Not { + q: bool, + rd: u8, + rn: u8, + }, + ReverseBits { + q: bool, + rd: u8, + rn: u8, + }, + CompareZero { + kind: Zero, + size: u32, + q: bool, + rd: u8, + rn: u8, + }, + Abs { + negate: bool, + size: u32, + q: bool, + rd: u8, + rn: u8, + }, + /// `xtn`/`sqxtn`/`uqxtn`/`sqxtun`; `upper` is the `2` form (keeps the low half). + Narrow { + kind: Narrow, + size: u32, + upper: bool, + rd: u8, + rn: u8, + }, + /// `saddlp`/`uaddlp` and, with `accumulate`, `sadalp`/`uadalp`. + PairLong { + signed: bool, + accumulate: bool, + size: u32, + q: bool, + rd: u8, + rn: u8, + }, + Long { + kind: Long, + signed: bool, + size: u32, + upper: bool, + rd: u8, + rn: u8, + rm: u8, + }, + ScalarAddp { + rd: u8, + rn: u8, + }, + ScalarDup { + size: u32, + index: u32, + rd: u8, + rn: u8, + }, + ScalarThree { + kind: Scalar, + rd: u8, + rn: u8, + rm: u8, + }, + /// `shl`, and `sshr`/`ushr`, by an immediate. + ShiftImm { + right: bool, + signed: bool, + size: u32, + q: bool, + shift: u32, + rd: u8, + rn: u8, + }, + /// `sshll`/`ushll` (`sxtl`/`uxtl` at shift 0); `upper` is the `2` form. + ShiftLong { + signed: bool, + size: u32, + shift: u32, + upper: bool, + rd: u8, + rn: u8, + }, + /// `shrn`/`shrn2`. + ShiftNarrow { + size: u32, + shift: u32, + upper: bool, + rd: u8, + rn: u8, + }, +} + +fn parse(word: u32) -> Option { + let rd = (word & 31) as u8; + let rn = ((word >> 5) & 31) as u8; + let rm = ((word >> 16) & 31) as u8; + let q = word & 0x4000_0000 != 0; + let u = word & 0x2000_0000 != 0; + let size = (word >> 22) & 3; + // Permute: bit 21 clear, bits 11:10 = 10, bits 14:12 pick the operation. + if word & 0xbf20_8c00 == 0x0e00_0800 { + let kind = match (word >> 12) & 7 { + 1 => Permute::Uzp1, + 2 => Permute::Trn1, + 3 => Permute::Zip1, + 5 => Permute::Uzp2, + 6 => Permute::Trn2, + 7 => Permute::Zip2, + _ => return None, + }; + // 64-bit lanes need the full vector. + if size == 3 && !q { + return None; + } + return Some(Op::Permute { + kind, + size, + q, + rd, + rn, + rm, + }); + } + // Two-register miscellaneous: bits 21:17 = 10000, bits 11:10 = 10. + if word & 0x9f3e_0c00 == 0x0e20_0800 { + let opcode = (word >> 12) & 0x1f; + let wide_needs_q = size != 3 || q; + return match (u, opcode) { + (false, 0) if size < 3 => Some(Op::Reverse { + group: 64, + size, + q, + rd, + rn, + }), + (true, 0) if size < 2 => Some(Op::Reverse { + group: 32, + size, + q, + rd, + rn, + }), + (false, 1) if size == 0 => Some(Op::Reverse { + group: 16, + size, + q, + rd, + rn, + }), + (false | true, 2) if size < 3 => Some(Op::PairLong { + signed: !u, + accumulate: false, + size, + q, + rd, + rn, + }), + (false | true, 6) if size < 3 => Some(Op::PairLong { + signed: !u, + accumulate: true, + size, + q, + rd, + rn, + }), + (false, 5) if size == 0 => Some(Op::Count { q, rd, rn }), + (true, 5) if size == 0 => Some(Op::Not { q, rd, rn }), + (true, 5) if size == 1 => Some(Op::ReverseBits { q, rd, rn }), + (false, 8) if wide_needs_q => Some(Op::CompareZero { + kind: Zero::Gt, + size, + q, + rd, + rn, + }), + (true, 8) if wide_needs_q => Some(Op::CompareZero { + kind: Zero::Ge, + size, + q, + rd, + rn, + }), + (true, 9) if wide_needs_q => Some(Op::CompareZero { + kind: Zero::Le, + size, + q, + rd, + rn, + }), + (false, 10) if wide_needs_q => Some(Op::CompareZero { + kind: Zero::Lt, + size, + q, + rd, + rn, + }), + (false, 11) if wide_needs_q => Some(Op::Abs { + negate: false, + size, + q, + rd, + rn, + }), + (true, 11) if wide_needs_q => Some(Op::Abs { + negate: true, + size, + q, + rd, + rn, + }), + (false, 0x12) if size < 3 => Some(Op::Narrow { + kind: Narrow::Plain, + size, + upper: q, + rd, + rn, + }), + (true, 0x12) if size < 3 => Some(Op::Narrow { + kind: Narrow::SignedToUnsignedSat, + size, + upper: q, + rd, + rn, + }), + (false, 0x14) if size < 3 => Some(Op::Narrow { + kind: Narrow::SignedSat, + size, + upper: q, + rd, + rn, + }), + (true, 0x14) if size < 3 => Some(Op::Narrow { + kind: Narrow::UnsignedSat, + size, + upper: q, + rd, + rn, + }), + // `cmeq #0` (U=0, opcode 9) is `SimdCompareZero`. + _ => None, + }; + } + // Three registers different: bit 21 set, bits 11:10 = 00. + if word & 0x9f20_0c00 == 0x0e20_0000 && size < 3 { + let kind = match (word >> 12) & 15 { + 0 => Long::Add, + 1 => Long::AddWide, + 2 => Long::Sub, + 3 => Long::SubWide, + 7 => Long::Abd, + 8 => Long::Mla, + 10 => Long::Mls, + 12 => Long::Mul, + _ => return None, + }; + return Some(Op::Long { + kind, + signed: !u, + size, + upper: q, + rd, + rn, + rm, + }); + } + // Shift by immediate: `immh` (bits 22:19) gives the element size and, with + // `immb`, the shift. `immh == 0` is the modified-immediate class instead. + if word & 0x9f80_0400 == 0x0f00_0400 && (word >> 19) & 15 != 0 { + let immh = (word >> 19) & 15; + let immhb = (word >> 16) & 0x7f; + // The position of the top set bit of immh: 0 for bytes up to 3 for doublewords. + let size = 31 - immh.leading_zeros(); + let esize = 8 << size; + return match (u, (word >> 11) & 0x1f) { + (false, 0b01010) if size < 3 || q => Some(Op::ShiftImm { + right: false, + signed: false, + size, + q, + shift: immhb - esize, + rd, + rn, + }), + (_, 0b00000) if size < 3 || q => Some(Op::ShiftImm { + right: true, + signed: !u, + size, + q, + shift: 2 * esize - immhb, + rd, + rn, + }), + (_, 0b10100) if size < 3 => Some(Op::ShiftLong { + signed: !u, + size, + shift: immhb - esize, + upper: q, + rd, + rn, + }), + (false, 0b10000) if size < 3 => Some(Op::ShiftNarrow { + size, + shift: 2 * esize - immhb, + upper: q, + rd, + rn, + }), + _ => None, + }; + } + // Scalar pairwise `addp Dd, Vn.2D`. + if word & 0xffff_fc00 == 0x5ef1_b800 { + return Some(Op::ScalarAddp { rd, rn }); + } + // Scalar `dup`: `mov Bd/Hd/Sd/Dd, Vn.T[index]`. + if word & 0xffe0_fc00 == 0x5e00_0400 { + let imm5 = (word >> 16) & 0x1f; + if imm5 == 0 || imm5.trailing_zeros() > 3 { + return None; + } + let size = imm5.trailing_zeros(); + return Some(Op::ScalarDup { + size, + index: imm5 >> (size + 1), + rd, + rn, + }); + } + // Scalar three-same, 64-bit only. + if word & 0xdf20_0400 == 0x5e20_0400 && size == 3 { + let kind = match (u, (word >> 11) & 0x1f) { + (false, 0x10) => Scalar::Add, + (true, 0x10) => Scalar::Sub, + (false, 0x06) => Scalar::Gt, + (true, 0x06) => Scalar::Hi, + (false, 0x07) => Scalar::Ge, + (true, 0x07) => Scalar::Hs, + (true, 0x11) => Scalar::Eq, + (false, 0x11) => Scalar::Tst, + _ => return None, + }; + return Some(Op::ScalarThree { kind, rd, rn, rm }); + } + None +} + +/// Whether [`run`] implements `word`. +pub fn is_supported(word: u32) -> bool { + parse(word).is_some() +} + +fn read(cpu: &Cpu, reg: u8) -> u128 { + let [lo, hi] = cpu.v[usize::from(reg)]; + u128::from(lo) | u128::from(hi) << 64 +} +/// Write `value`; without `full` the upper 64 bits are cleared. +fn write(cpu: &mut Cpu, reg: u8, value: u128, full: bool) { + cpu.v[usize::from(reg)] = [value as u64, if full { (value >> 64) as u64 } else { 0 }]; +} +fn mask(bits: u32) -> u128 { + if bits >= 128 { + u128::MAX + } else { + (1 << bits) - 1 + } +} +fn get(v: u128, bits: u32, index: u32) -> u128 { + (v >> (bits * index)) & mask(bits) +} +fn put(v: &mut u128, bits: u32, index: u32, value: u128) { + let field = mask(bits) << (bits * index); + *v = (*v & !field) | ((value << (bits * index)) & field); +} +/// The low `bits` bits of `x` as a signed number. +fn signed(x: u128, bits: u32) -> i128 { + let shift = 128 - bits; + ((x << shift) as i128) >> shift +} +fn extend(x: u128, bits: u32, signed_lane: bool) -> i128 { + if signed_lane { + signed(x, bits) + } else { + x as i128 + } +} +fn lanes(size: u32, q: bool) -> u32 { + (if q { 128 } else { 64 }) / (8 << size) +} + +/// Run the instruction `word` on `cpu`. `word` must satisfy [`is_supported`]. +pub fn run(cpu: &mut Cpu, word: u32) { + let op = parse(word).expect("an unsupported SIMD word reached the executor"); + match op { + Op::Permute { + kind, + size, + q, + rd, + rn, + rm, + } => { + let (a, b) = (read(cpu, rn), read(cpu, rm)); + let (bits, n) = (8 << size, lanes(size, q)); + let half = n / 2; + let mut out = 0; + for i in 0..n { + let value = match kind { + Permute::Uzp1 if i < half => get(a, bits, 2 * i), + Permute::Uzp1 => get(b, bits, 2 * (i - half)), + Permute::Uzp2 if i < half => get(a, bits, 2 * i + 1), + Permute::Uzp2 => get(b, bits, 2 * (i - half) + 1), + Permute::Zip1 => get(if i % 2 == 0 { a } else { b }, bits, i / 2), + Permute::Zip2 => get(if i % 2 == 0 { a } else { b }, bits, half + i / 2), + Permute::Trn1 => get(if i % 2 == 0 { a } else { b }, bits, i - i % 2), + Permute::Trn2 => get(if i % 2 == 0 { a } else { b }, bits, i | 1), + }; + put(&mut out, bits, i, value); + } + write(cpu, rd, out, q); + } + Op::Reverse { + group, + size, + q, + rd, + rn, + } => { + let a = read(cpu, rn); + let bits = 8 << size; + let per_group = group / bits; + let mut out = 0; + for i in 0..lanes(size, q) { + let (g, j) = (i / per_group, i % per_group); + put( + &mut out, + bits, + i, + get(a, bits, g * per_group + (per_group - 1 - j)), + ); + } + write(cpu, rd, out, q); + } + Op::Count { q, rd, rn } => { + let a = read(cpu, rn); + let mut out = 0; + for i in 0..lanes(0, q) { + put( + &mut out, + 8, + i, + u128::from((get(a, 8, i) as u8).count_ones()), + ); + } + write(cpu, rd, out, q); + } + Op::Not { q, rd, rn } => { + let a = !read(cpu, rn); + write(cpu, rd, a, q); + } + Op::ReverseBits { q, rd, rn } => { + let a = read(cpu, rn); + let mut out = 0; + for i in 0..lanes(0, q) { + put( + &mut out, + 8, + i, + u128::from((get(a, 8, i) as u8).reverse_bits()), + ); + } + write(cpu, rd, out, q); + } + Op::CompareZero { + kind, + size, + q, + rd, + rn, + } => { + let a = read(cpu, rn); + let bits = 8 << size; + let mut out = 0; + for i in 0..lanes(size, q) { + let x = signed(get(a, bits, i), bits); + let holds = match kind { + Zero::Gt => x > 0, + Zero::Ge => x >= 0, + Zero::Le => x <= 0, + Zero::Lt => x < 0, + }; + put(&mut out, bits, i, if holds { mask(bits) } else { 0 }); + } + write(cpu, rd, out, q); + } + Op::Abs { + negate, + size, + q, + rd, + rn, + } => { + let a = read(cpu, rn); + let bits = 8 << size; + let mut out = 0; + for i in 0..lanes(size, q) { + let x = signed(get(a, bits, i), bits); + // The most negative value keeps its pattern in both operations. + let r = if negate { + x.wrapping_neg() + } else { + x.wrapping_abs() + }; + put(&mut out, bits, i, r as u128 & mask(bits)); + } + write(cpu, rd, out, q); + } + Op::Narrow { + kind, + size, + upper, + rd, + rn, + } => { + let a = read(cpu, rn); + let (narrow, wide) = (8 << size, 16 << size); + let mut out = 0; + for i in 0..64 / narrow { + let x = get(a, wide, i); + let r = match kind { + Narrow::Plain => x, + Narrow::SignedSat => { + let s = signed(x, wide); + let (lo, hi) = (-(1i128 << (narrow - 1)), (1i128 << (narrow - 1)) - 1); + s.clamp(lo, hi) as u128 + } + Narrow::UnsignedSat => x.min(mask(narrow)), + Narrow::SignedToUnsignedSat => { + signed(x, wide).clamp(0, mask(narrow) as i128) as u128 + } + }; + put(&mut out, narrow, i, r & mask(narrow)); + } + let value = if upper { + (read(cpu, rd) & mask(64)) | out << 64 + } else { + out + }; + write(cpu, rd, value, true); + } + Op::PairLong { + signed: signed_lane, + accumulate, + size, + q, + rd, + rn, + } => { + let a = read(cpu, rn); + let (bits, wide) = (8 << size, 16 << size); + let old = read(cpu, rd); + let mut out = 0; + for i in 0..lanes(size, q) / 2 { + let pair = extend(get(a, bits, 2 * i), bits, signed_lane) + + extend(get(a, bits, 2 * i + 1), bits, signed_lane); + let sum = if accumulate { + pair.wrapping_add(get(old, wide, i) as i128) + } else { + pair + }; + put(&mut out, wide, i, sum as u128 & mask(wide)); + } + write(cpu, rd, out, q); + } + Op::Long { + kind, + signed: signed_lane, + size, + upper, + rd, + rn, + rm, + } => { + let (a, b, old) = (read(cpu, rn), read(cpu, rm), read(cpu, rd)); + let (bits, wide) = (8 << size, 16 << size); + // The narrow operand comes from the low or high half of the register. + let shift = if upper { 64 } else { 0 }; + let mut out = 0; + for i in 0..64 / bits { + let y = extend(get(b >> shift, bits, i), bits, signed_lane); + let x = match kind { + Long::AddWide | Long::SubWide => extend(get(a, wide, i), wide, signed_lane), + _ => extend(get(a >> shift, bits, i), bits, signed_lane), + }; + let accumulator = get(old, wide, i) as i128; + let r = match kind { + Long::Add | Long::AddWide => x + y, + Long::Sub | Long::SubWide => x - y, + Long::Abd => (x - y).abs(), + Long::Mul => x * y, + Long::Mla => accumulator.wrapping_add(x * y), + Long::Mls => accumulator.wrapping_sub(x * y), + }; + put(&mut out, wide, i, r as u128 & mask(wide)); + } + write(cpu, rd, out, true); + } + Op::ScalarAddp { rd, rn } => { + let [lo, hi] = cpu.v[usize::from(rn)]; + cpu.v[usize::from(rd)] = [lo.wrapping_add(hi), 0]; + } + Op::ScalarDup { + size, + index, + rd, + rn, + } => { + let value = get(read(cpu, rn), 8 << size, index); + write(cpu, rd, value, false); + } + Op::ShiftImm { + right, + signed: signed_lane, + size, + q, + shift, + rd, + rn, + } => { + let a = read(cpu, rn); + let bits = 8 << size; + let mut out = 0; + for i in 0..lanes(size, q) { + let x = get(a, bits, i); + let r = if !right { + x << shift + } else if signed_lane { + (signed(x, bits) >> shift) as u128 + } else { + x >> shift + }; + put(&mut out, bits, i, r & mask(bits)); + } + write(cpu, rd, out, q); + } + Op::ShiftLong { + signed: signed_lane, + size, + shift, + upper, + rd, + rn, + } => { + let a = read(cpu, rn) >> if upper { 64 } else { 0 }; + let (bits, wide) = (8 << size, 16 << size); + let mut out = 0; + for i in 0..64 / bits { + let x = extend(get(a, bits, i), bits, signed_lane); + put(&mut out, wide, i, (x << shift) as u128 & mask(wide)); + } + write(cpu, rd, out, true); + } + Op::ShiftNarrow { + size, + shift, + upper, + rd, + rn, + } => { + let a = read(cpu, rn); + let narrow = 8 << size; + let mut out = 0; + for i in 0..64 / narrow { + put( + &mut out, + narrow, + i, + (get(a, 2 * narrow, i) >> shift) & mask(narrow), + ); + } + let value = if upper { + (read(cpu, rd) & mask(64)) | out << 64 + } else { + out + }; + write(cpu, rd, value, true); + } + Op::ScalarThree { kind, rd, rn, rm } => { + let (a, b) = (cpu.v[usize::from(rn)][0], cpu.v[usize::from(rm)][0]); + let flag = |holds: bool| if holds { u64::MAX } else { 0 }; + let r = match kind { + Scalar::Add => a.wrapping_add(b), + Scalar::Sub => a.wrapping_sub(b), + Scalar::Gt => flag((a as i64) > (b as i64)), + Scalar::Hi => flag(a > b), + Scalar::Ge => flag((a as i64) >= (b as i64)), + Scalar::Hs => flag(a >= b), + Scalar::Eq => flag(a == b), + Scalar::Tst => flag(a & b != 0), + }; + cpu.v[usize::from(rd)] = [r, 0]; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A register whose lane `i` of `bits` bits is `f(i)`. + fn pack(bits: u32, count: u32, f: impl Fn(u32) -> u128) -> [u64; 2] { + let mut v = 0u128; + for i in 0..count { + put(&mut v, bits, i, f(i)); + } + [v as u64, (v >> 64) as u64] + } + fn unpack(r: [u64; 2], bits: u32, count: u32) -> Vec { + let v = u128::from(r[0]) | u128::from(r[1]) << 64; + (0..count).map(|i| get(v, bits, i)).collect() + } + /// A CPU whose v1 holds bytes 0..16 and v2 bytes 16..32. + fn permute_cpu() -> Cpu { + let mut cpu = Cpu::default(); + cpu.v[1] = pack(8, 16, |i| u128::from(i)); + cpu.v[2] = pack(8, 16, |i| u128::from(16 + i)); + cpu + } + fn run_lanes(word: u32, cpu: &mut Cpu, bits: u32, count: u32) -> Vec { + run(cpu, word); + unpack(cpu.v[0], bits, count) + } + + #[test] + fn permutes_pick_the_architectural_lanes_for_every_size_and_q() { + let mut c = permute_cpu(); + // Word lanes of v1 are w1_i = bytes 4i..4i+3; of v2 likewise from 16. + let a = |i: u128| 0x0302_0100 + 0x0404_0404 * i; + let b = |i: u128| 0x1312_1110 + 0x0404_0404 * i; + assert_eq!( + run_lanes(0x4e821820, &mut c, 32, 4), + [a(0), a(2), b(0), b(2)] + ); // uzp1.4s + assert_eq!( + run_lanes(0x4e825820, &mut c, 32, 4), + [a(1), a(3), b(1), b(3)] + ); // uzp2.4s + assert_eq!( + run_lanes(0x4e822820, &mut c, 32, 4), + [a(0), b(0), a(2), b(2)] + ); // trn1.4s + // zip1.8b interleaves the low bytes and clears the upper half. + let mut c = permute_cpu(); + run(&mut c, 0x0e023820); + assert_eq!(unpack(c.v[0], 8, 8), [0, 16, 1, 17, 2, 18, 3, 19]); + assert_eq!(c.v[0][1], 0); + // zip2.16b interleaves the high bytes. + let mut c = permute_cpu(); + run(&mut c, 0x4e027820); + let expected: Vec = (8..16).flat_map(|i| [i, 16 + i]).collect(); + assert_eq!(unpack(c.v[0], 8, 16), expected); + // trn2.8h takes the odd halfwords. + let mut c = permute_cpu(); + run(&mut c, 0x4e426820); + let half = |base: u128, i: u128| (base + 2 * i) | (base + 2 * i + 1) << 8; + assert_eq!( + unpack(c.v[0], 16, 8), + [ + half(0, 1), + half(16, 1), + half(0, 3), + half(16, 3), + half(0, 5), + half(16, 5), + half(0, 7), + half(16, 7) + ] + ); + // zip1.2d: the low doublewords. + let mut c = permute_cpu(); + run(&mut c, 0x4ec23820); + assert_eq!(c.v[0], [c.v[1][0], c.v[2][0]]); + } + + #[test] + fn reversals_count_and_bitwise_not() { + let mut c = Cpu::default(); + c.v[1] = pack(8, 16, |i| u128::from(i)); + // rev32.16b reverses the bytes inside each word. + run(&mut c, 0x6e200820); + assert_eq!( + unpack(c.v[0], 8, 16), + [3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12] + ); + // rev16.8b swaps byte pairs and clears the upper half. + c.v[0] = [u64::MAX; 2]; + run(&mut c, 0x0e201820); + assert_eq!(unpack(c.v[0], 8, 8), [1, 0, 3, 2, 5, 4, 7, 6]); + assert_eq!(c.v[0][1], 0); + // rev64.4s swaps the words of each doubleword. + c.v[1] = pack(32, 4, |i| u128::from(10 + i)); + run(&mut c, 0x4ea00820); + assert_eq!(unpack(c.v[0], 32, 4), [11, 10, 13, 12]); + // cnt counts the bits of every byte; mvn inverts; rbit reverses each byte. + c.v[1] = pack(8, 16, |i| { + [ + 0xff, 0, 0x0f, 1, 0x80, 0xaa, 0x55, 3, 0, 0, 0, 0, 0, 0, 0, 0xfe, + ][i as usize] + }); + run(&mut c, 0x4e205820); + assert_eq!(unpack(c.v[0], 8, 8), [8, 0, 4, 1, 1, 4, 4, 2]); + assert_eq!(unpack(c.v[0], 8, 16)[15], 7); + run(&mut c, 0x6e205820); + assert_eq!(unpack(c.v[0], 8, 4), [0, 0xff, 0xf0, 0xfe]); + c.v[0] = [u64::MAX; 2]; + run(&mut c, 0x2e605820); // rbit.8b + assert_eq!(unpack(c.v[0], 8, 4), [0xff, 0, 0xf0, 0x80]); + assert_eq!(c.v[0][1], 0); + } + + #[test] + fn compares_against_zero_abs_and_neg_are_signed_and_wrap() { + let mut c = Cpu::default(); + let words = [-1i32 as u32 as u128, 0, 1, 0x8000_0000]; + c.v[1] = pack(32, 4, |i| words[i as usize]); + let ones = u128::from(u32::MAX); + run(&mut c, 0x4ea08820); // cmgt.4s #0 + assert_eq!(unpack(c.v[0], 32, 4), [0, 0, ones, 0]); + // cmle.8h #0 over halfwords of the same register. + run(&mut c, 0x6e609820); + let h = unpack(c.v[0], 16, 8); + assert_eq!(h[0], 0xffff, "0xffff is -1"); + assert_eq!(h[1], 0xffff, "0xffff is -1 (upper half of the first word)"); + assert_eq!(h[4], 0, "1 is positive"); + // cmlt.2d #0 looks at the sign of 64-bit lanes. + c.v[1] = [u64::MAX, 1]; + run(&mut c, 0x4ee0a820); + assert_eq!(c.v[0], [u64::MAX, 0]); + // abs.4s keeps the most negative value; neg.2d wraps likewise. + c.v[1] = pack(32, 4, |i| { + [(-5i32) as u32 as u128, 5, 0x8000_0000, 0][i as usize] + }); + run(&mut c, 0x4ea0b820); + assert_eq!(unpack(c.v[0], 32, 4), [5, 5, 0x8000_0000, 0]); + c.v[1] = [1, 1 << 63]; + run(&mut c, 0x6ee0b820); + assert_eq!(c.v[0], [u64::MAX, 1 << 63]); + // cmge.16b #0 with both halves. + c.v[1] = pack(8, 16, |i| if i % 2 == 0 { 0x80 } else { 0x7f }); + run(&mut c, 0x6e208820); + assert_eq!(unpack(c.v[0], 8, 4), [0, 0xff, 0, 0xff]); + } + + #[test] + fn narrowing_truncates_or_saturates_and_the_2_form_keeps_the_low_half() { + let mut c = Cpu::default(); + // xtn.2s from two doublewords; the stale upper half is cleared. + c.v[1] = [0x1_0000_0002, 0x3_0000_0004]; + c.v[0] = [u64::MAX; 2]; + run(&mut c, 0x0ea12820); + assert_eq!(c.v[0], [0x0000_0004_0000_0002, 0]); + // xtn2.4s writes the upper half and keeps the lower. + c.v[0] = [0x1111, 0x2222]; + run(&mut c, 0x4ea12820); + assert_eq!(c.v[0], [0x1111, 0x0000_0004_0000_0002]); + // uqxtn.8b saturates unsigned. + c.v[1] = pack(16, 8, |i| [0x1ff, 5, 0xff, 0x100, 0, 1, 2, 3][i as usize]); + run(&mut c, 0x2e214820); + assert_eq!(unpack(c.v[0], 8, 8), [0xff, 5, 0xff, 0xff, 0, 1, 2, 3]); + // sqxtn.4h saturates signed; sqxtun.8b clamps negatives to zero. + c.v[1] = pack(32, 4, |i| { + [ + 70_000u128, + (-70_000i32) as u32 as u128, + 5, + (-5i32) as u32 as u128, + ][i as usize] + }); + run(&mut c, 0x0e614820); + assert_eq!(unpack(c.v[0], 16, 4), [0x7fff, 0x8000, 5, 0xfffb]); + c.v[1] = pack(16, 8, |i| { + [(-5i16) as u16 as u128, 300, 7, 255, 256, 0, 0, 0][i as usize] + }); + run(&mut c, 0x2e212820); + assert_eq!(unpack(c.v[0], 8, 5), [0, 255, 7, 255, 255]); + } + + #[test] + fn pairwise_long_adds_widen_and_accumulate() { + let mut c = Cpu::default(); + // uaddlp.4h from bytes. + c.v[1] = pack(8, 8, |i| [255, 255, 1, 2, 0, 0, 128, 128][i as usize]); + run(&mut c, 0x2e202820); + assert_eq!(unpack(c.v[0], 16, 4), [510, 3, 0, 256]); + // saddlp.2d from words: -1 + -1 = -2. + c.v[1] = pack(32, 4, |i| { + [u128::from(u32::MAX), u128::from(u32::MAX), 3, 4][i as usize] + }); + run(&mut c, 0x4ea02820); + assert_eq!(c.v[0], [(-2i64) as u64, 7]); + // uadalp.2d accumulates into the destination. + c.v[0] = [10, 20]; + run(&mut c, 0x6ea06820); + assert_eq!(c.v[0], [u64::from(u32::MAX) * 2 + 10, 27]); + // sadalp.8h sign-extends the bytes before adding. + c.v[1] = pack(8, 16, |i| if i % 2 == 0 { 0x80 } else { 1 }); + c.v[0] = pack(16, 8, |_| 100); + run(&mut c, 0x4e206820); + assert_eq!(unpack(c.v[0], 16, 8)[0], (100 - 128 + 1) as u16 as u128); + } + + #[test] + fn long_and_wide_arithmetic_uses_the_requested_half_and_extension() { + let mut c = Cpu::default(); + c.v[1] = pack(8, 16, |i| if i < 8 { 255 } else { 3 }); + c.v[2] = pack(8, 16, |i| if i < 8 { 255 } else { 5 }); + // uaddl.8h adds the low bytes of both operands as 16-bit lanes. + run(&mut c, 0x2e220020); + assert_eq!(unpack(c.v[0], 16, 8), [510; 8]); + // usubl.4s over halfwords: 3 - 5 wraps in 32 bits. + c.v[1] = pack(16, 8, |i| if i < 4 { 3 } else { 9 }); + c.v[2] = pack(16, 8, |i| if i < 4 { 5 } else { 4 }); + run(&mut c, 0x2e622020); + assert_eq!(unpack(c.v[0], 32, 4), [0xffff_fffe; 4]); + // uabdl2.2d takes the high words and the absolute difference. + c.v[1] = pack(32, 4, |i| [0, 0, 10, 100][i as usize]); + c.v[2] = pack(32, 4, |i| [0, 0, 40, 7][i as usize]); + run(&mut c, 0x6ea27020); + assert_eq!(c.v[0], [30, 93]); + // uaddw.2d adds the narrow low words to the wide Vn lanes (no extension of Vn). + c.v[1] = [1 << 40, 2]; + c.v[2] = pack(32, 4, |i| [7, 9, 0, 0][i as usize]); + run(&mut c, 0x2ea21020); + assert_eq!(c.v[0], [(1 << 40) + 7, 11]); + // saddw.8h sign-extends the bytes of Vm. + c.v[1] = pack(16, 8, |_| 1000); + c.v[2] = pack(8, 16, |i| if i == 0 { 0xff } else { 1 }); + run(&mut c, 0x0e221020); + assert_eq!(unpack(c.v[0], 16, 2), [999, 1001]); + // usubw2.8h: Vn wide minus the upper bytes of Vm. + c.v[1] = pack(16, 8, |_| 300); + c.v[2] = pack(8, 16, |i| if i >= 8 { 2 } else { 99 }); + run(&mut c, 0x6e223020); + assert_eq!(unpack(c.v[0], 16, 8), [298; 8]); + // umull.4s: 0xffff * 0xffff; smull.8h: -128 * -128. + c.v[1] = pack(16, 8, |_| 0xffff); + c.v[2] = pack(16, 8, |_| 0xffff); + run(&mut c, 0x2e62c020); + assert_eq!(unpack(c.v[0], 32, 4), [0xfffe_0001; 4]); + c.v[1] = pack(8, 16, |_| 0x80); + c.v[2] = pack(8, 16, |_| 0x80); + run(&mut c, 0x0e22c020); + assert_eq!(unpack(c.v[0], 16, 8), [16384; 8]); + // umull2.2d uses the upper words; umlal.4s accumulates; umlsl2.8h subtracts. + c.v[1] = pack(32, 4, |i| [1, 1, 100_000, 3][i as usize]); + c.v[2] = pack(32, 4, |i| [1, 1, 100_000, 5][i as usize]); + run(&mut c, 0x6ea2c020); + assert_eq!(c.v[0], [10_000_000_000, 15]); + c.v[1] = pack(16, 8, |_| 3); + c.v[2] = pack(16, 8, |_| 4); + c.v[0] = pack(32, 4, |_| 1); + run(&mut c, 0x2e628020); + assert_eq!(unpack(c.v[0], 32, 4), [13; 4]); + c.v[1] = pack(8, 16, |_| 2); + c.v[2] = pack(8, 16, |_| 3); + c.v[0] = pack(16, 8, |_| 10); + run(&mut c, 0x6e22a020); + assert_eq!(unpack(c.v[0], 16, 8), [4; 8]); + } + + #[test] + fn scalar_forms_work_on_64_bits_and_clear_the_rest_of_the_register() { + let mut c = Cpu::default(); + c.v[1] = [5, 7]; + c.v[0] = [u64::MAX; 2]; + run(&mut c, 0x5ef1b820); // addp d0, v1.2d + assert_eq!(c.v[0], [12, 0]); + c.v[0] = [u64::MAX; 2]; + run(&mut c, 0x5e180420); // mov d0, v1.d[1] + assert_eq!(c.v[0], [7, 0]); + c.v[1] = pack(8, 16, |i| u128::from(0x10 + i)); + run(&mut c, 0x5e070420); // mov b0, v1.b[3] + assert_eq!(c.v[0], [0x13, 0]); + c.v[1] = [2, 0]; + c.v[2] = [3, 0]; + run(&mut c, 0x5ee28420); // add d0, d1, d2 + assert_eq!(c.v[0][0], 5); + run(&mut c, 0x7ee28420); // sub d0, d1, d2 + assert_eq!(c.v[0][0], u64::MAX); + // cmhi d4, d2, d1 is unsigned: 3 > 2. + run(&mut c, 0x7ee13444); + assert_eq!(c.v[4][0], u64::MAX); + c.v[1] = [u64::MAX, 0]; + run(&mut c, 0x7ee13444); // d2 (3) > d1 (max)? no + assert_eq!(c.v[4][0], 0); + c.v[1] = [4, 0]; + c.v[2] = [4, 0]; + run(&mut c, 0x7ee28c20); // cmeq d0, d1, d2 + assert_eq!(c.v[0][0], u64::MAX); + } + + #[test] + fn shifts_by_immediate_widen_narrow_and_keep_lane_boundaries() { + let mut c = Cpu::default(); + // ushll.8h v1, v0.8b, #0 is `uxtl`: zero-extend the low eight bytes. + c.v[0] = pack(8, 16, |i| u128::from(0xf0 + i)); + run(&mut c, 0x2f08a401); + assert_eq!( + unpack(c.v[1], 16, 8), + (0..8).map(|i| 0xf0 + i).collect::>() + ); + // ushll2.4s #5 widens the upper four halfwords and shifts left. + c.v[1] = pack(16, 8, |i| u128::from(i + 1)); + run(&mut c, 0x6f15a420); + assert_eq!(unpack(c.v[0], 32, 4), [5 << 5, 6 << 5, 7 << 5, 8 << 5]); + // sshll.2d #31 sign-extends the low two words. + c.v[1] = pack(32, 4, |i| [u128::from(u32::MAX), 1, 9, 9][i as usize]); + run(&mut c, 0x0f3fa420); + assert_eq!(c.v[0], [0xffff_ffff_8000_0000, 1 << 31]); + // sshll2.8h #7 takes the high bytes. + c.v[1] = pack(8, 16, |i| if i >= 8 { 0x80 } else { 1 }); + run(&mut c, 0x4f0fa420); + assert_eq!( + unpack(c.v[0], 16, 8), + [0xc000; 8], + "-128 << 7 in sixteen bits" + ); + // shl.4s #5, shl.2d #63, and shl.8b #1 which clears the upper half. + c.v[1] = pack(32, 4, |i| u128::from(i + 1) | 0x8000_0000); + run(&mut c, 0x4f255420); + assert_eq!( + unpack(c.v[0], 32, 4), + [32, 64, 96, 128], + "bits shifted out are dropped" + ); + c.v[1] = [3, 2]; + run(&mut c, 0x4f7f5420); + assert_eq!(c.v[0], [1 << 63, 0]); + c.v[1] = [0x8040_2010_0804_0281, u64::MAX]; + c.v[0] = [u64::MAX; 2]; + run(&mut c, 0x0f095420); + assert_eq!(c.v[0], [0x0080_4020_1008_0402, 0]); + // ushr.2d #1, and ushr.4s #32 which shifts everything out. + c.v[1] = [u64::MAX, 2]; + run(&mut c, 0x6f7f0420); + assert_eq!(c.v[0], [u64::MAX >> 1, 1]); + run(&mut c, 0x6f200420); + assert_eq!(c.v[0], [0, 0]); + // sshr.16b #7 and sshr.8h #16 fill with the sign bit. + c.v[1] = pack(8, 16, |i| if i % 2 == 0 { 0x80 } else { 0x7f }); + run(&mut c, 0x4f090420); + assert_eq!(unpack(c.v[0], 8, 4), [0xff, 0, 0xff, 0]); + c.v[1] = pack(16, 8, |i| if i % 2 == 0 { 0x8000 } else { 0x7fff }); + run(&mut c, 0x4f100420); + assert_eq!(unpack(c.v[0], 16, 4), [0xffff, 0, 0xffff, 0]); + // shrn.8b #4 narrows halfwords; shrn2.4s #32 writes the upper half only. + c.v[1] = pack(16, 8, |i| u128::from(i + 1) << 4 | 0xf000); + c.v[0] = [u64::MAX; 2]; + run(&mut c, 0x0f0c8420); + assert_eq!(unpack(c.v[0], 8, 8), [1, 2, 3, 4, 5, 6, 7, 8]); + assert_eq!(c.v[0][1], 0); + c.v[1] = [0x1_0000_0000_u64 * 7 | 5, 0x1_0000_0000_u64 * 9]; + c.v[0] = [0x1111, 0x2222]; + run(&mut c, 0x4f208420); + assert_eq!(c.v[0], [0x1111, 0x0000_0009_0000_0007]); + // shrn.2s #1 from doublewords. + c.v[1] = [0x1_0000_0006, 0x2_0000_000a]; + run(&mut c, 0x0f3f8420); + assert_eq!(c.v[0], [0x0000_0005_8000_0003, 0]); + } + #[test] + fn classification_declines_what_it_does_not_implement() { + // cmeq #0 is `SimdCompareZero`; half-width lanes need Q for 64-bit lanes; + // `ins`, `umov` and the three-same group belong to other decoders. + assert!(!is_supported(0x0e209820)); + assert!(!is_supported(0x0ee0b820), "abs .1d is reserved"); + assert!(!is_supported(0x2ee21c20), "bif is three-same"); + assert!(!is_supported(0x4e083c20), "umov"); + assert!(is_supported(0x4e821820)); + } +} diff --git a/src/emulator/src/aarch64/simd_struct.rs b/src/emulator/src/aarch64/simd_struct.rs new file mode 100644 index 0000000..9d1c9ab --- /dev/null +++ b/src/emulator/src/aarch64/simd_struct.rs @@ -0,0 +1,310 @@ +//! SIMD structure loads and stores: `LD1`..`LD4`, `ST1`..`ST4`, their single-lane +//! forms and the replicating `LD1R`..`LD4R`. +//! +//! Decode describes one in a [`StructDesc`]. The JIT copies the descriptor into +//! [`Cpu::structure`](super::Cpu) and traps; the machine (or user mode) then runs +//! [`execute`] against its own memory, one element at a time, so each backend keeps +//! its own translation, device and fault handling. + +/// Which elements of the registers a structure access touches. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +#[repr(u8)] +pub enum Shape { + /// Whole vectors: `ld1 {v0.4s - v3.4s}` and the interleaved `ld2`..`ld4`. + #[default] + Multiple, + /// One element of each register: `ld1 {v0.s}[1]`. Other lanes are kept. + Lane, + /// One element broadcast to every lane: `ld1r`. Loads only. + Replicate, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +#[repr(C)] +pub struct StructDesc { + pub store: bool, + pub shape: Shape, + /// Registers touched, starting at `rt` and wrapping from v31 to v0. + pub registers: u8, + /// `ld2`..`ld4`: consecutive memory elements alternate between the registers. + /// Otherwise (`ld1`) each register is a contiguous block. + pub interleaved: bool, + /// Element size in bytes: 1, 2, 4 or 8. + pub esize: u8, + /// Element index of a [`Shape::Lane`] access. + pub lane: u8, + /// A 128-bit vector (otherwise 64-bit, with the upper half zeroed on loads). + pub q: bool, + pub rt: u8, +} + +impl StructDesc { + fn vector_bytes(&self) -> u8 { + if self.q { 16 } else { 8 } + } + /// Bytes of memory the access spans; the post-index immediate advances by this. + pub fn bytes(&self) -> u64 { + let registers = u64::from(self.registers); + match self.shape { + Shape::Multiple => registers * u64::from(self.vector_bytes()), + Shape::Lane | Shape::Replicate => registers * u64::from(self.esize), + } + } +} + +fn mask(esize: u8) -> u64 { + if esize >= 8 { + u64::MAX + } else { + (1 << (u32::from(esize) * 8)) - 1 + } +} +fn extract(register: &[u64; 2], esize: u8, lane: u8) -> u64 { + let bit = u32::from(lane) * u32::from(esize) * 8; + (register[(bit / 64) as usize] >> (bit % 64)) & mask(esize) +} +fn insert(register: &mut [u64; 2], esize: u8, lane: u8, value: u64) { + let bit = u32::from(lane) * u32::from(esize) * 8; + let (word, shift) = ((bit / 64) as usize, bit % 64); + register[word] = (register[word] & !(mask(esize) << shift)) | ((value & mask(esize)) << shift); +} + +/// Run `desc` at `base` over the register file `v`. `access(address, width, store)` +/// moves one element: it writes `store` when given, otherwise it reads and returns +/// the element, zero-extended. The first error stops the access; `v` may then be +/// partly updated, so callers work on a copy and keep it only on success. +pub fn execute( + desc: &StructDesc, + v: &mut [[u64; 2]; 32], + base: u64, + mut access: impl FnMut(u64, u8, Option) -> Result, +) -> Result<(), E> { + let esize = desc.esize; + let registers = usize::from(desc.registers); + let register = |r: usize| usize::from(desc.rt) + r & 31; + let mut element = |v: &mut [[u64; 2]; 32], r: usize, lane: u8, address: u64| { + let reg = register(r); + if desc.store { + access(address, esize, Some(extract(&v[reg], esize, lane))).map(drop) + } else { + access(address, esize, None).map(|value| insert(&mut v[reg], esize, lane, value)) + } + }; + match desc.shape { + Shape::Multiple => { + let lanes = usize::from(desc.vector_bytes() / esize); + if !desc.store { + for r in 0..registers { + v[register(r)] = [0; 2]; + } + } + for slot in 0..registers * lanes { + let (r, lane) = if desc.interleaved { + (slot % registers, slot / registers) + } else { + (slot / lanes, slot % lanes) + }; + let address = base.wrapping_add(slot as u64 * u64::from(esize)); + element(v, r, lane as u8, address)?; + } + } + Shape::Lane => { + for r in 0..registers { + let address = base.wrapping_add(r as u64 * u64::from(esize)); + element(v, r, desc.lane, address)?; + } + } + Shape::Replicate => { + let lanes = desc.vector_bytes() / esize; + for r in 0..registers { + let address = base.wrapping_add(r as u64 * u64::from(esize)); + let value = access(address, esize, None)?; + let mut broadcast = [0; 2]; + for lane in 0..lanes { + insert(&mut broadcast, esize, lane, value); + } + v[register(r)] = broadcast; + } + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use core::convert::Infallible; + + fn memory(len: u8) -> Vec { + (0..len).collect() + } + fn run(desc: &StructDesc, v: &mut [[u64; 2]; 32], memory: &mut [u8]) { + let result: Result<(), Infallible> = execute(desc, v, 0, |address, width, store| { + let at = address as usize..address as usize + usize::from(width); + Ok(match store { + Some(value) => { + memory[at].copy_from_slice(&value.to_le_bytes()[..usize::from(width)]); + 0 + } + None => { + let mut bytes = [0; 8]; + bytes[..usize::from(width)].copy_from_slice(&memory[at]); + u64::from_le_bytes(bytes) + } + }) + }); + result.unwrap(); + } + fn bytes(register: [u64; 2]) -> Vec { + register.iter().flat_map(|w| w.to_le_bytes()).collect() + } + + #[test] + fn ld2_deinterleaves_elements() { + // ld2 {v0.4h, v1.4h}: v0 gets halfwords 0, 2, 4, 6; v1 gets 1, 3, 5, 7. + let desc = StructDesc { + registers: 2, + interleaved: true, + esize: 2, + ..StructDesc::default() + }; + let mut v = [[u64::MAX; 2]; 32]; + run(&desc, &mut v, &mut memory(16)); + assert_eq!( + bytes(v[0]), + [0, 1, 4, 5, 8, 9, 12, 13, 0, 0, 0, 0, 0, 0, 0, 0] + ); + assert_eq!( + bytes(v[1]), + [2, 3, 6, 7, 10, 11, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0] + ); + } + #[test] + fn st3_interleaves_and_wraps_registers() { + // st3 {v31.16b, v0.16b, v1.16b}: register numbers wrap, memory alternates. + let desc = StructDesc { + store: true, + registers: 3, + interleaved: true, + esize: 1, + q: true, + rt: 31, + ..StructDesc::default() + }; + let mut v = [[0; 2]; 32]; + v[31] = [u64::from_le_bytes([1; 8]); 2]; + v[0] = [u64::from_le_bytes([2; 8]); 2]; + v[1] = [u64::from_le_bytes([3; 8]); 2]; + let mut out = vec![0; 48]; + run(&desc, &mut v, &mut out); + assert!(out.chunks(3).all(|c| c == [1, 2, 3])); + } + #[test] + fn ld1_registers_are_contiguous_blocks() { + let desc = StructDesc { + registers: 2, + esize: 4, + q: true, + ..StructDesc::default() + }; + let mut v = [[0; 2]; 32]; + run(&desc, &mut v, &mut memory(32)); + assert_eq!(bytes(v[0]), (0..16).collect::>()); + assert_eq!(bytes(v[1]), (16..32).collect::>()); + } + #[test] + fn lane_load_keeps_other_lanes_and_store_reads_the_lane() { + // ld1 {v3.s}[1] replaces bits 63:32 only. + let desc = StructDesc { + shape: Shape::Lane, + registers: 1, + esize: 4, + lane: 1, + rt: 3, + ..StructDesc::default() + }; + let mut v = [[0; 2]; 32]; + v[3] = [0x1111_1111_2222_2222, 0x3333_3333_4444_4444]; + run(&desc, &mut v, &mut [0xaa, 0xbb, 0xcc, 0xdd]); + assert_eq!(v[3], [0xddcc_bbaa_2222_2222, 0x3333_3333_4444_4444]); + // st1 {v3.d}[1] writes the upper doubleword. + let store = StructDesc { + store: true, + esize: 8, + lane: 1, + ..desc + }; + let mut out = [0; 8]; + run(&store, &mut v, &mut out); + assert_eq!(u64::from_le_bytes(out), 0x3333_3333_4444_4444); + } + #[test] + fn lane_form_spans_consecutive_elements_per_register() { + // ld3 {v0.s, v1.s, v2.s}[3]: elements are adjacent in memory, lane 3 of each. + let desc = StructDesc { + shape: Shape::Lane, + registers: 3, + esize: 4, + lane: 3, + q: true, + ..StructDesc::default() + }; + let mut v = [[0; 2]; 32]; + run(&desc, &mut v, &mut memory(12)); + assert_eq!(v[0][1] >> 32, 0x0302_0100); + assert_eq!(v[1][1] >> 32, 0x0706_0504); + assert_eq!(v[2][1] >> 32, 0x0b0a_0908); + assert_eq!(desc.bytes(), 12); + } + #[test] + fn replicate_fills_every_lane_and_zeroes_the_upper_half() { + // ld1r {v0.8b}: one byte everywhere in the low half, upper half cleared. + let desc = StructDesc { + shape: Shape::Replicate, + registers: 1, + esize: 1, + ..StructDesc::default() + }; + let mut v = [[u64::MAX; 2]; 32]; + run(&desc, &mut v, &mut [0x5a]); + assert_eq!(v[0], [0x5a5a_5a5a_5a5a_5a5a, 0]); + // ld1r {v0.2d}: both doublewords. + let wide = StructDesc { + esize: 8, + q: true, + ..desc + }; + run(&wide, &mut v, &mut 7u64.to_le_bytes()); + assert_eq!(v[0], [7, 7]); + } + #[test] + fn bytes_is_the_post_index_advance() { + let ld4 = StructDesc { + registers: 4, + interleaved: true, + esize: 4, + q: true, + ..StructDesc::default() + }; + assert_eq!(ld4.bytes(), 64); + assert_eq!(StructDesc { q: false, ..ld4 }.bytes(), 32); + } + #[test] + fn first_error_stops_the_access() { + let desc = StructDesc { + registers: 2, + interleaved: true, + esize: 8, + q: true, + ..StructDesc::default() + }; + let mut v = [[0; 2]; 32]; + let mut calls = 0; + let result = execute(&desc, &mut v, 0, |_, _, _| { + calls += 1; + if calls == 2 { Err("fault") } else { Ok(1) } + }); + assert_eq!(result, Err("fault")); + assert_eq!(calls, 2); + } +} diff --git a/src/emulator/src/devices/gicv3.rs b/src/emulator/src/devices/gicv3.rs new file mode 100644 index 0000000..8611ca6 --- /dev/null +++ b/src/emulator/src/devices/gicv3.rs @@ -0,0 +1,253 @@ +//! GICv3 interrupt controller for one processing element: the part XNU's QEMU +//! platform (`pexpert/arm/pe_fiq.c`) drives. +//! +//! It brings the redistributor out of sleep, puts the virtual timer's PPI 27 in +//! group 0 (delivered as a FIQ) and enables it. The CPU interface is the `ICC_*` +//! system registers, which live in the CPU's system state; this device holds the +//! memory-mapped distributor and redistributor and decides whether PPI 27 is +//! forwarded to the CPU. +//! +//! Everything else (SPIs, SGIs, LPIs, group 1, affinity routing) reads as zero and +//! ignores writes: nothing in the XNU boot path uses it. + +pub const DISTRIBUTOR_SIZE: u64 = 0x1_0000; +/// The redistributor window of QEMU `virt`: 0x20000 (two frames) per PE. +pub const REDISTRIBUTOR_SIZE: u64 = 0xf6_0000; +const FRAME: u64 = 0x1_0000; + +const GICD_CTLR: u64 = 0x0; +const GICD_TYPER: u64 = 0x4; +const GICR_CTLR: u64 = 0x0; +const GICR_TYPER: u64 = 0x8; +const GICR_WAKER: u64 = 0x14; +/// `PIDR2`: the architecture revision (3 = GICv3) in bits 7:4. +const PIDR2: u64 = 0xffe8; +const PIDR2_GICV3: u64 = 0x3b; +// SGI frame, relative to its base. +const IGROUPR0: u64 = 0x80; +const ISENABLER0: u64 = 0x100; +const ICENABLER0: u64 = 0x180; +const ISPENDR0: u64 = 0x200; +const ICPENDR0: u64 = 0x280; +const IPRIORITYR: u64 = 0x400; + +/// `GICD_CTLR.EnableGrp0`, `EnableGrp1NS` and `ARE_NS`: the writable bits. +const GICD_CTLR_WRITABLE: u32 = 0x1 | 0x2 | 0x10; +const GICD_CTLR_ENABLE_GRP0: u32 = 0x1; +/// `GICR_TYPER.Last`: this is the final redistributor in the window. +const TYPER_LAST: u64 = 1 << 4; +const WAKER_PROCESSOR_SLEEP: u32 = 1 << 1; +const WAKER_CHILDREN_ASLEEP: u32 = 1 << 2; +/// The virtual timer's private peripheral interrupt. +pub const VIRTUAL_TIMER: u32 = 27; + +#[derive(Debug, Clone)] +pub struct Gicv3 { + distributor_ctlr: u32, + /// Only `ProcessorSleep` is stored; `ChildrenAsleep` follows it at once. + processor_sleep: bool, + group: u32, + enabled: u32, + pending: u32, + priority: [u8; 32], +} + +impl Default for Gicv3 { + fn default() -> Self { + Self { + distributor_ctlr: 0, + // A redistributor comes out of reset asleep. + processor_sleep: true, + group: 0, + enabled: 0, + pending: 0, + priority: [0; 32], + } + } +} + +/// The bytes of `value` an access of `size` bytes at `offset` within a naturally +/// aligned 64-bit register sees. +fn window(value: u64, offset: u64, size: u8) -> u64 { + let shifted = value >> ((offset & 7) * 8); + if size >= 8 { + shifted + } else { + shifted & ((1 << (u32::from(size) * 8)) - 1) + } +} + +impl Gicv3 { + pub fn read_distributor(&self, offset: u64, size: u8) -> u64 { + // All of these are 32-bit registers. + let register = match offset & !3 { + GICD_CTLR => u64::from(self.distributor_ctlr), + // `ITLinesNumber` 0 (32 INTIDs), one PE, 16 ID bits. + GICD_TYPER => 15 << 19, + PIDR2 => PIDR2_GICV3, + _ => 0, + }; + window(register, offset & 3, size) + } + pub fn write_distributor(&mut self, offset: u64, _size: u8, value: u64) { + if offset == GICD_CTLR { + self.distributor_ctlr = value as u32 & GICD_CTLR_WRITABLE; + } + } + + fn waker(&self) -> u32 { + if self.processor_sleep { + WAKER_PROCESSOR_SLEEP | WAKER_CHILDREN_ASLEEP + } else { + 0 + } + } + pub fn read_redistributor(&self, offset: u64, size: u8) -> u64 { + let frame = offset / FRAME; + let at = offset % FRAME; + // Only PE 0's frames exist; the rest of the window is reserved. + if frame >= 2 { + return 0; + } + if frame == 0 { + return match at & !7 { + GICR_CTLR => 0, + // Affinity 0.0.0.0, processor number 0, and the last in the window. + GICR_TYPER => window(TYPER_LAST, at, size), + _ if at & !3 == GICR_WAKER => window(u64::from(self.waker()), at & 3, size), + _ if at & !3 == PIDR2 & !3 => { + window(PIDR2_GICV3 << ((PIDR2 & 3) * 8), at & 3, size) + } + _ => 0, + }; + } + let word = |value: u32| window(u64::from(value), at & 3, size); + match at { + IGROUPR0 => word(self.group), + ISENABLER0 | ICENABLER0 => word(self.enabled), + ISPENDR0 | ICPENDR0 => word(self.pending), + IPRIORITYR..=0x41f => { + let start = (at - IPRIORITYR) as usize; + (0..usize::from(size).min(4)) + .map(|i| u64::from(self.priority[start + i]) << (8 * i)) + .fold(0, |a, b| a | b) + } + _ => 0, + } + } + pub fn write_redistributor(&mut self, offset: u64, size: u8, value: u64) { + let frame = offset / FRAME; + let at = offset % FRAME; + if frame == 0 { + if at == GICR_WAKER { + self.processor_sleep = value as u32 & WAKER_PROCESSOR_SLEEP != 0; + } + return; + } + if frame != 1 { + return; + } + let bits = value as u32; + match at { + IGROUPR0 => self.group = bits, + ISENABLER0 => self.enabled |= bits, + ICENABLER0 => self.enabled &= !bits, + ISPENDR0 => self.pending |= bits, + ICPENDR0 => self.pending &= !bits, + IPRIORITYR..=0x41f => { + let start = (at - IPRIORITYR) as usize; + for i in 0..usize::from(size).min(4).min(32 - start) { + self.priority[start + i] = (value >> (8 * i)) as u8; + } + } + _ => {} + } + } + + /// The priority the virtual timer's interrupt is forwarded with, or `None` if + /// the controller would not deliver it as a FIQ: the redistributor must be + /// awake, group 0 enabled at the distributor, and PPI 27 enabled in group 0 + /// (a group 1 interrupt is an IRQ, not a FIQ). + pub fn vtimer_fiq_priority(&self) -> Option { + let bit = 1 << VIRTUAL_TIMER; + let forwarded = !self.processor_sleep + && self.distributor_ctlr & GICD_CTLR_ENABLE_GRP0 != 0 + && self.enabled & bit != 0 + && self.group & bit == 0; + forwarded.then(|| self.priority[VIRTUAL_TIMER as usize]) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The sequence of `pe_init_fiq`: find this PE's redistributor, wake it, + /// make the timer a group 0 interrupt, enable it and the distributor's group 0. + fn init_like_xnu(gic: &mut Gicv3) { + let typer = gic.read_redistributor(GICR_TYPER, 8); + assert_eq!(typer >> 32 & 0xffff, 0, "affinity 0.0 is this PE"); + assert_ne!(typer & TYPER_LAST, 0, "the scan stops at the first frame"); + let waker = gic.read_redistributor(GICR_WAKER, 4) as u32; + assert_ne!(waker & WAKER_CHILDREN_ASLEEP, 0); + gic.write_redistributor(GICR_WAKER, 4, u64::from(waker & !WAKER_PROCESSOR_SLEEP)); + assert_eq!( + gic.read_redistributor(GICR_WAKER, 4) as u32 & WAKER_CHILDREN_ASLEEP, + 0 + ); + gic.write_redistributor(FRAME + IGROUPR0, 4, 0x81ff_ffff); + gic.write_redistributor(FRAME + ISENABLER0, 4, 1 << 27); + let ctlr = gic.read_distributor(GICD_CTLR, 4) as u32; + gic.write_distributor(GICD_CTLR, 4, u64::from(ctlr | GICD_CTLR_ENABLE_GRP0)); + } + + #[test] + fn the_xnu_init_sequence_forwards_the_virtual_timer_as_a_fiq() { + let mut gic = Gicv3::default(); + assert_eq!(gic.vtimer_fiq_priority(), None); + init_like_xnu(&mut gic); + assert_eq!(gic.vtimer_fiq_priority(), Some(0)); + } + + #[test] + fn each_gate_on_the_path_to_the_cpu_blocks_the_timer() { + let ready = |edit: &dyn Fn(&mut Gicv3)| { + let mut gic = Gicv3::default(); + init_like_xnu(&mut gic); + edit(&mut gic); + gic.vtimer_fiq_priority() + }; + // The redistributor asleep again. + assert_eq!( + ready(&|g| g.write_redistributor(GICR_WAKER, 4, u64::from(WAKER_PROCESSOR_SLEEP))), + None + ); + // Distributor group 0 disabled. + assert_eq!(ready(&|g| g.write_distributor(GICD_CTLR, 4, 0)), None); + // PPI 27 disabled (ICENABLER clears it). + assert_eq!( + ready(&|g| g.write_redistributor(FRAME + ICENABLER0, 4, 1 << 27)), + None + ); + // PPI 27 in group 1: an IRQ, never a FIQ. + assert_eq!( + ready(&|g| g.write_redistributor(FRAME + IGROUPR0, 4, 1 << 27)), + None + ); + // The priority the guest set is reported for the CPU interface's mask check. + assert_eq!( + ready(&|g| g.write_redistributor(FRAME + IPRIORITYR + 27, 1, 0x80)), + Some(0x80) + ); + } + + #[test] + fn typer_is_readable_as_two_words_and_other_frames_are_reserved() { + let gic = Gicv3::default(); + assert_eq!(gic.read_redistributor(GICR_TYPER, 4), TYPER_LAST); + assert_eq!(gic.read_redistributor(GICR_TYPER + 4, 4), 0); + // PE 1 does not exist: the second PE's frames read as zero. + assert_eq!(gic.read_redistributor(2 * FRAME + GICR_TYPER, 8), 0); + assert_eq!(gic.read_distributor(PIDR2, 4) & 0xf0, 0x30, "GICv3"); + } +} diff --git a/src/emulator/src/devices/mod.rs b/src/emulator/src/devices/mod.rs index 1ce362b..72ce939 100644 --- a/src/emulator/src/devices/mod.rs +++ b/src/emulator/src/devices/mod.rs @@ -1,9 +1,11 @@ pub mod bus; pub mod gicv2; +pub mod gicv3; pub mod pl011; pub mod virtio_blk; pub use bus::{Bus, BusDevice, Device}; pub use gicv2::Gicv2; +pub use gicv3::Gicv3; pub use pl011::{LEN as PL011_LEN, Pl011}; pub use virtio_blk::{BlockBackend, VirtioBlock, VirtioIo}; diff --git a/src/emulator/src/system/afdt.rs b/src/emulator/src/system/afdt.rs new file mode 100644 index 0000000..0764784 --- /dev/null +++ b/src/emulator/src/system/afdt.rs @@ -0,0 +1,322 @@ +//! Apple flattened device tree for the XNU QEMU `virt` board, the tree +//! `pexpert` looks nodes up in. Port of the AFDT builder the XNU fork's +//! `qemu/mkafdt.py` ships; each node here exists because XNU panics or loses +//! its console without it: +//! +//! * `/chosen` `dram-base`/`dram-size`: `arm_init` panics without them. +//! * `/defaults` `serial-device` plus `/arm-io/uart0` `AAPL,phandle`: the +//! serial console lookup in `pe_serial.c`. +//! * `/arm-io` `ranges`: its second value becomes the SoC base address that +//! every `reg` below it is relative to. +//! * `/arm-io/gic` `reg`: GICD and GICR windows for `pe_fiq.c`. +//! * `/options`: `IODTPlatformExpert` creates the NVRAM node from it. +//! +//! Encoding (little-endian): `u32 nProperties, u32 nChildren`, then per property +//! a 32-byte NUL-padded name, `u32 length` and the value padded to four bytes, +//! then the children recursively. +use super::fdt::{GICD_BASE, UART_BASE}; +use crate::aarch64::identification::counter_hz; + +pub const GICD_SIZE: u64 = 0x1_0000; +/// The redistributor window of QEMU `virt`: 0x20000 per PE. +pub const GICR_BASE: u64 = 0x080a_0000; +pub const GICR_SIZE: u64 = 0x00f6_0000; +pub const RTC_BASE: u64 = 0x0901_0000; +const UART_PHANDLE: u32 = 0x100; +/// Size of the panic log the tree places in the last of RAM (`chosen/pram`). +pub const PANIC_LOG_SIZE: u32 = 0x8_0000; +const NVRAM_BANK_SIZE: usize = 0x2000; +const RANDOM_SEED_SIZE: usize = 256; + +fn property(name: &str, value: &[u8]) -> Vec { + assert!( + name.len() < 32, + "device tree property name {name:?} is too long" + ); + let mut bytes = Vec::with_capacity(36 + value.len() + 3); + bytes.extend_from_slice(name.as_bytes()); + bytes.resize(32, 0); + bytes.extend_from_slice(&(value.len() as u32).to_le_bytes()); + bytes.extend_from_slice(value); + bytes.resize(bytes.len().next_multiple_of(4), 0); + bytes +} +fn string(name: &str, value: &str) -> Vec { + let mut bytes = Vec::with_capacity(value.len() + 1); + bytes.extend_from_slice(value.as_bytes()); + bytes.push(0); + property(name, &bytes) +} +fn word(name: &str, value: u32) -> Vec { + property(name, &value.to_le_bytes()) +} +fn words64(name: &str, values: &[u64]) -> Vec { + let bytes: Vec = values.iter().flat_map(|v| v.to_le_bytes()).collect(); + property(name, &bytes) +} +fn node(properties: &[Vec], children: &[Vec]) -> Vec { + let mut bytes = Vec::new(); + bytes.extend_from_slice(&(properties.len() as u32).to_le_bytes()); + bytes.extend_from_slice(&(children.len() as u32).to_le_bytes()); + for part in properties.iter().chain(children) { + bytes.extend_from_slice(part); + } + bytes +} + +fn adler32(bytes: &[u8]) -> u32 { + let (mut a, mut b) = (1u32, 0u32); + for &byte in bytes { + a = (a + u32::from(byte)) % 65521; + b = (b + a) % 65521; + } + (b << 16) | a +} + +/// An empty, valid CHRP NVRAM image: the bank header and one `common` partition. +fn nvram_image() -> Vec { + let mut image = vec![0u8; NVRAM_BANK_SIZE]; + let mut header = |offset: usize, length: usize, name: &[u8]| { + image[offset] = 0x70; + image[offset + 2..offset + 4].copy_from_slice(&((length / 16) as u16).to_le_bytes()); + image[offset + 4..offset + 4 + name.len()].copy_from_slice(name); + let mut checksum = u32::from(image[offset]) + + image[offset + 2..offset + 16] + .iter() + .map(|&b| u32::from(b)) + .sum::(); + while checksum > 0xff { + checksum = (checksum & 0xff) + (checksum >> 8); + } + image[offset + 1] = checksum as u8; + }; + header(0, 32, b"nvram"); + header(32, NVRAM_BANK_SIZE - 32, b"common"); + let sum = adler32(&image[20..]); + image[16..20].copy_from_slice(&sum.to_le_bytes()); + image +} + +/// Deterministic non-zero bytes. XNU rejects an all-zero seed; a fixed stream +/// keeps boots reproducible, which matters more here than entropy. +fn random_seed() -> Vec { + let mut state = 0x9e37_79b9_7f4a_7c15u64; + let mut seed = Vec::with_capacity(RANDOM_SEED_SIZE); + while seed.len() < RANDOM_SEED_SIZE { + state = state.wrapping_add(0x9e37_79b9_7f4a_7c15); + let mut z = state; + z = (z ^ (z >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + z = (z ^ (z >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + seed.extend_from_slice(&(z ^ (z >> 31)).to_le_bytes()); + } + seed +} + +fn chosen(ram_base: u64, ram_size: u64, boot_args: &str) -> Vec { + node( + &[ + string("name", "chosen"), + property("dram-base", &ram_base.to_le_bytes()), + property("dram-size", &ram_size.to_le_bytes()), + string("firmware-version", "volt"), + string("system-firmware-version", "volt"), + string("boot-args", boot_args), + property("unique-chip-id", &[1, 2, 3, 4, 5, 6, 7, 8]), + word("embedded-panic-log-size", PANIC_LOG_SIZE), + word("kernel-ctrr-to-be-enabled", 0), + word("debug-enabled", 1), + property("random-seed", &random_seed()), + word("nvram-bank-size", NVRAM_BANK_SIZE as u32), + word("nvram-bank-count", 2), + word("nvram-current-bank", 0), + property("nvram-proxy-data", &nvram_image()), + ], + &[node(&[string("name", "memory-map")], &[])], + ) +} + +fn arm_io() -> Vec { + let uart = node( + &[ + string("name", "uart0"), + string("compatible", "arm,pl011"), + words64("reg", &[UART_BASE - GICD_BASE, 0x1000]), + word("AAPL,phandle", UART_PHANDLE), + ], + &[], + ); + let gic = node( + &[ + string("name", "gic"), + string("interrupt-controller", "master"), + string("compatible", "arm,gic-v3"), + words64("reg", &[0, GICD_SIZE, GICR_BASE - GICD_BASE, GICR_SIZE]), + ], + &[], + ); + let timer = node( + &[ + string("name", "timer"), + string("device_type", "timer"), + words64("reg", &[RTC_BASE - GICD_BASE, 0x1000]), + ], + &[], + ); + node( + &[ + string("name", "arm-io"), + string("device_type", "arm-io"), + words64("ranges", &[0, GICD_BASE]), + ], + &[uart, gic, timer], + ) +} + +fn cpus() -> Vec { + let cpu0 = node( + &[ + string("name", "cpu0"), + string("state", "running"), + word("reg", 0), + word("timebase-frequency", counter_hz as u32), + word("clock-frequency", 1_000_000_000), + ], + &[], + ); + node(&[string("name", "cpus")], &[cpu0]) +} + +/// The device tree for one CPU, RAM at `[ram_base, ram_base + ram_size)` and +/// `boot_args` as the kernel's command line. +pub fn build(ram_base: u64, ram_size: u64, boot_args: &str) -> Vec { + let panic_log = ram_base + ram_size - u64::from(PANIC_LOG_SIZE); + node( + &[ + string("name", "device-tree"), + string("compatible", "qemu,virt"), + string("model", "QEMU,virt"), + ], + &[ + chosen(ram_base, ram_size, boot_args), + node( + &[ + string("name", "defaults"), + word("serial-device", UART_PHANDLE), + ], + &[], + ), + node(&[string("name", "options")], &[]), + arm_io(), + cpus(), + node( + &[ + string("name", "pram"), + words64("reg", &[panic_log, u64::from(PANIC_LOG_SIZE)]), + ], + &[], + ), + ], + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A parsed node: its properties by name and its children in order. + struct Parsed { + properties: Vec<(String, Vec)>, + children: Vec, + } + fn u32_at(bytes: &[u8], at: usize) -> usize { + u32::from_le_bytes(bytes[at..at + 4].try_into().unwrap()) as usize + } + /// Walk the tree the way `pexpert/gen/device_tree.c` does. + fn parse(bytes: &[u8], at: &mut usize) -> Parsed { + let (count, children) = (u32_at(bytes, *at), u32_at(bytes, *at + 4)); + *at += 8; + let mut properties = Vec::new(); + for _ in 0..count { + let name = &bytes[*at..*at + 32]; + let end = name + .iter() + .position(|&b| b == 0) + .expect("name is NUL-terminated"); + let length = u32_at(bytes, *at + 32); + let value = bytes[*at + 36..*at + 36 + length].to_vec(); + properties.push((String::from_utf8(name[..end].to_vec()).unwrap(), value)); + *at += 36 + length.next_multiple_of(4); + } + let children = (0..children).map(|_| parse(bytes, at)).collect(); + Parsed { + properties, + children, + } + } + impl Parsed { + fn get(&self, name: &str) -> &[u8] { + &self + .properties + .iter() + .find(|(n, _)| n == name) + .unwrap_or_else(|| panic!("no property {name}")) + .1 + } + fn child(&self, name: &str) -> &Parsed { + self.children + .iter() + .find(|c| { + c.properties.iter().any(|(n, v)| { + n == "name" && v.starts_with(name.as_bytes()) && v[name.len()] == 0 + }) + }) + .unwrap_or_else(|| panic!("no node {name}")) + } + } + + #[test] + fn the_tree_parses_to_its_exact_length_with_the_nodes_xnu_requires() { + let tree = build(0x4000_0000, 1 << 30, "-v serial=3"); + let mut at = 0; + let root = parse(&tree, &mut at); + assert_eq!(at, tree.len(), "properties and children cover the blob"); + let chosen = root.child("chosen"); + assert_eq!(chosen.get("dram-base"), 0x4000_0000u64.to_le_bytes()); + assert_eq!(chosen.get("dram-size"), (1u64 << 30).to_le_bytes()); + assert_eq!(chosen.get("boot-args"), b"-v serial=3\0"); + assert_eq!( + root.child("defaults").get("serial-device"), + root.child("arm-io").child("uart0").get("AAPL,phandle") + ); + let gic = root.child("arm-io").child("gic"); + assert_eq!(gic.get("compatible"), b"arm,gic-v3\0"); + assert_eq!(gic.get("reg").len(), 32); + root.child("options"); + root.child("cpus").child("cpu0"); + } + + #[test] + fn unaligned_values_are_padded_so_the_next_property_stays_aligned() { + let mut bytes = property("odd", &[1, 2, 3]); + assert_eq!(bytes.len(), 32 + 4 + 4); + assert_eq!(&bytes[36..], &[1, 2, 3, 0]); + bytes.extend(property("next", &[])); + assert_eq!(bytes.len() % 4, 0); + } + + #[test] + fn the_random_seed_is_not_all_zero() { + let seed = random_seed(); + assert_eq!(seed.len(), RANDOM_SEED_SIZE); + assert!(seed.iter().any(|&b| b != 0)); + } + + #[test] + fn the_nvram_image_carries_a_valid_adler_checksum() { + let image = nvram_image(); + assert_eq!(u32_at(&image, 16) as u32, adler32(&image[20..])); + assert_eq!(image[0], 0x70); + assert_eq!(&image[4..9], b"nvram"); + assert_eq!(&image[36..42], b"common"); + } +} diff --git a/src/emulator/src/system/boot.rs b/src/emulator/src/system/boot.rs index 3a47dae..1dd9a04 100644 --- a/src/emulator/src/system/boot.rs +++ b/src/emulator/src/system/boot.rs @@ -38,6 +38,10 @@ pub enum Error { NotAnImage, BigEndianKernel, NoRoom, + /// A Mach-O kernel that is truncated or inconsistent. + BadMachO(&'static str), + /// A boot option the selected kernel format cannot honour. + Unsupported(&'static str), Memory(MemoryError), } impl core::fmt::Display for Error { @@ -82,7 +86,9 @@ pub fn load_address(ram_base: u64, header: Header) -> Result { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct Layout { pub entry: u64, - pub device_tree: u64, + /// Guest-physical address handed to the kernel in `x0`: the device tree + /// blob for a Linux Image, the `boot_args` block for XNU. + pub boot_info: u64, pub initrd: Option<(u64, u64)>, } pub fn prepare( @@ -152,7 +158,7 @@ pub fn prepare_boot( memory.write(device_tree, &tree).map_err(Error::Memory)?; Ok(Layout { entry, - device_tree, + boot_info: device_tree, initrd: initrd_info, }) } @@ -203,7 +209,7 @@ mod tests { let (initrd_start, initrd_end) = layout.initrd.unwrap(); assert!(initrd_start >= layout.entry + 0x200000); assert_eq!(initrd_end, initrd_start + initrd_data.len() as u64); - assert!(layout.device_tree >= initrd_end); + assert!(layout.boot_info >= initrd_end); let mut read_back = vec![0u8; initrd_data.len()]; mem.read(initrd_start, &mut read_back).unwrap(); diff --git a/src/emulator/src/system/machine.rs b/src/emulator/src/system/machine.rs index a6060bf..5abff51 100644 --- a/src/emulator/src/system/machine.rs +++ b/src/emulator/src/system/machine.rs @@ -1,7 +1,9 @@ //! One-vCPU machine over native translated blocks (Mirage aarch64/Machine.zig). +use super::afdt; use super::fdt; use crate::aarch64::cpu::{Exclusive, Trap}; use crate::aarch64::decode::SignExtend; +use crate::aarch64::simd_struct; use crate::aarch64::{Cache, Cpu}; use crate::aarch64::{exception, translate}; use crate::devices::{bus::Device, gicv2::Gicv2, pl011::Pl011}; @@ -12,6 +14,8 @@ pub enum Error { Compile(crate::aarch64::compile::Error), Memory(MemoryError), InvalidAccessWidth(u8), + /// An LSE atomic on memory that is not host RAM, or not aligned to its size. + UnsupportedAtomic(u64), } impl core::fmt::Display for Error { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { @@ -26,6 +30,13 @@ pub trait DeviceIo { fn write_gic_distributor(&mut self, cpu_id: u32, offset: u64, size: u8, value: u64); fn read_gic_cpu(&mut self, cpu_id: u32, offset: u64, size: u8) -> u64; fn write_gic_cpu(&mut self, cpu_id: u32, offset: u64, size: u8, value: u64); + fn read_gic_redistributor(&mut self, offset: u64, size: u8) -> u64 { + let _ = (offset, size); + 0 + } + fn write_gic_redistributor(&mut self, offset: u64, size: u8, value: u64) { + let _ = (offset, size, value); + } fn read_virtio(&mut self, offset: u64, size: u8) -> u64 { let _ = (offset, size); 0 @@ -153,6 +164,9 @@ pub struct Machine { /// `CodeTracker::epoch` the inline data TLB's write permissions reflect. watch_epoch: u64, pub irq_line: bool, + /// Set when a GICv3 forwards PPI 27 (the virtual timer) as a group 0 FIQ: the + /// priority it was given. `None` leaves the timer to the GICv2 path. + pub vtimer_fiq: Option, timer_fired: bool, pub stalled_at: u64, pub stalled: Option, @@ -186,6 +200,7 @@ impl Machine { dtlb_generation: 0, watch_epoch: 0, irq_line: false, + vtimer_fiq: None, timer_fired: false, stalled_at: 0, stalled: None, @@ -232,6 +247,19 @@ impl Machine { self.timer_fired = true; true } + /// Whether the interrupt controller presents the virtual timer to this CPU as + /// a FIQ right now: the GIC forwards it (`vtimer_fiq`), the CPU interface has + /// group 0 enabled and its priority mask lets the interrupt through, and the + /// timer's condition holds. Evaluated from live state, so the line drops as soon + /// as the guest masks the timer or moves its compare value. + fn fiq_asserted(&self) -> bool { + let system = &self.cpu.system; + self.vtimer_fiq.is_some_and(|priority| { + system.icc_igrpen0_el1 & 1 != 0 + && u64::from(priority) < system.icc_pmr_el1 & 0xff + && self.timer_due() + }) + } pub fn run(&mut self, serial: &mut Pl011, gic: &mut Gicv2) -> Result { self.run_with_devices(&mut (serial, gic)) } @@ -241,12 +269,21 @@ impl Machine { if self.timer_edge() { return Ok(Exit::Timer); } - if self.irq_line && self.cpu.system.daif & 2 == 0 { + let fiq = self.fiq_asserted(); + self.cpu.system.isr_el1 = u64::from(self.irq_line) << 7 | u64::from(fiq) << 6; + let kind = if fiq && self.cpu.system.daif & 1 == 0 { + Some(exception::Kind::Fiq) + } else if self.irq_line && self.cpu.system.daif & 2 == 0 { + Some(exception::Kind::Irq) + } else { + None + }; + if let Some(kind) = kind { let pc = self.cpu.pc; exception::take( &mut self.cpu, exception::Reason { - kind: exception::Kind::Irq, + kind, ..Default::default() }, pc, @@ -274,6 +311,23 @@ impl Machine { self.abort(address, permission, true, false); continue; } + if let crate::aarch64::compile::Error::Decode { word, pc } = &error + && crate::aarch64::decode::is_undefined(*word) + { + // An Undefined Instruction exception (EC 0, 32-bit instruction). + self.cpu.pc = *pc; + exception::take( + &mut self.cpu, + exception::Reason { + ec: 0, + instruction: true, + status: 0, + ..Default::default() + }, + 0, + ); + continue; + } self.stalled_at = self.cpu.pc; self.stalled = Some(format!("{error:?}")); return Err(Error::Compile(error)); @@ -365,6 +419,17 @@ impl Machine { self.tlb.flush(); self.cpu.trap = Trap::None; } + Trap::Host => { + let word = self.cpu.value as u32; + crate::aarch64::host::run(&mut self.cpu, word); + self.cpu.trap = Trap::None; + } + Trap::AddressTranslate => { + let (address, flags) = (self.cpu.address, self.cpu.value); + self.cpu.system.par_el1 = + self.address_translate(address, flags & 1 != 0, flags & 2 != 0); + self.cpu.trap = Trap::None; + } Trap::Translation => { let pc = self.cpu.pc; self.abort(pc, false, true, false); @@ -456,6 +521,44 @@ impl Machine { return Ok(Exit::Interrupted); } } + Trap::Atomic => { + self.faulted = false; + let Some(old) = self.atomic()? else { + continue; + }; + // The words read go to the registers the instruction named; + // register 31 is XZR and discards them. + for (slot, value) in old + .into_iter() + .enumerate() + .take(self.cpu.atomic_kind.words()) + { + let dest = usize::from(self.cpu.atomic_dests[slot]); + if dest < 31 { + self.cpu.x[dest] = value; + } + } + } + Trap::SimdStruct => { + self.faulted = false; + self.device_access = false; + self.simd_struct(devices)?; + if self.faulted { + self.cpu.writeback = false; + continue; + } + if self.cpu.writeback { + if self.cpu.writeback_dest == 31 { + self.cpu.sp = self.cpu.writeback_value; + } else { + self.cpu.x[self.cpu.writeback_dest as usize] = self.cpu.writeback_value; + } + } + self.cpu.writeback = false; + if self.device_access { + return Ok(Exit::Interrupted); + } + } } } Ok(Exit::Interrupted) @@ -611,6 +714,167 @@ impl Machine { self.access(address, width, 31, value, devices)?; Ok(!self.faulted) } + /// `AT S1E{1,0}{R,W}`: the `PAR_EL1` value of a stage-1 walk of `address`. Success + /// holds the page's physical address; failure sets `F` with a fault status code + /// (level 3 translation or permission fault). + fn address_translate(&mut self, address: u64, write: bool, user: bool) -> u64 { + let access = if write { + translate::Access::Write + } else { + translate::Access::Read + }; + let result = if user { + // Walk with EL0 permissions on a scratch TLB: switching the context of + // the real one would flush it. + let mut cpu = self.cpu.clone(); + cpu.system.el = 0; + translate::translate( + &mut translate::Tlb::new(), + &cpu, + &mut self.memory, + address, + access, + ) + } else { + translate::translate(&mut self.tlb, &self.cpu, &mut self.memory, address, access) + }; + match result { + Ok(physical) => (physical & 0x0000_ffff_ffff_f000) | 1 << 11, + Err(translate::TranslateError::PermissionFault) => 1 | 0b001111 << 1, + Err(_) => 1 | 0b000111 << 1, + } + } + /// SIMD structure load/store: `simd_struct::execute` over the vector registers, + /// one element per `access`. The registers change only if every element succeeded. + fn simd_struct(&mut self, devices: &mut impl DeviceIo) -> Result<(), Error> { + let desc = self.cpu.structure; + let saved = self.cpu.v; + let mut registers = saved; + // `access` takes its direction from the trap kind. + self.cpu.trap = if desc.store { Trap::Store } else { Trap::Load }; + // `Err(None)` is a guest fault `access` already recorded; `Some` is fatal. + let result = simd_struct::execute( + &desc, + &mut registers, + self.cpu.address, + |address, width, store| { + if let Some(value) = store { + self.cpu.vector_dest = false; + self.access(address, width, 31, value, devices) + .map_err(Some)?; + } else { + // `access` loads whole registers: use v0 as scratch and merge the + // element into `registers` (v0 is restored below). + self.cpu.vector_dest = true; + self.access(address, width, 0, 0, devices).map_err(Some)?; + } + if self.faulted { + return Err(None); + } + Ok(self.cpu.v[0][0]) + }, + ); + self.cpu.vector_dest = false; + self.cpu.trap = Trap::None; + self.cpu.v = saved; + match result { + Ok(()) => { + self.cpu.v = registers; + Ok(()) + } + Err(None) => Ok(()), + Err(Some(error)) => Err(error), + } + } + /// LSE atomic (`ldadd`, `swp`, `cas`, `casp`, ...) on RAM. Every LSE operation + /// holds one global lock while it reads, combines and writes back, so a CASP + /// pair stays consistent against every other LSE operation on any vCPU; each + /// word is still accessed through a host atomic. Plain stores that bypass this + /// path (inline JIT stores, `stxr`) are not ordered against the lock. + /// Returns the words read, or `None` if translation faulted. + fn atomic(&mut self) -> Result, Error> { + static LSE: parking_lot::Mutex<()> = parking_lot::Mutex::new(()); + let (kind, address, width) = (self.cpu.atomic_kind, self.cpu.address, self.cpu.width); + if !matches!(width, 1 | 2 | 4 | 8) { + return Err(Error::InvalidAccessWidth(width)); + } + let words = kind.words(); + let span = u64::from(width) * words as u64; + let physical = match translate::translate( + &mut self.tlb, + &self.cpu, + &mut self.memory, + address, + translate::Access::Write, + ) { + Ok(physical) => physical, + Err(error) => { + self.abort( + address, + matches!(error, translate::TranslateError::PermissionFault), + false, + true, + ); + return Ok(None); + } + }; + // Aligned to the whole span, so every word lies in this one page. + let page = (physical % span == 0) + .then(|| self.memory.host_page(physical & !0xfff)) + .flatten() + .ok_or(Error::UnsupportedAtomic(address))?; + let width_bytes = usize::from(width); + // SAFETY: `page` is a live RAM page and `physical` is aligned to `span`, so + // each word's address is inside the page and aligned to `width`. + let word = + |slot: usize| unsafe { page.add((physical & 0xfff) as usize + slot * width_bytes) }; + let guard = LSE.lock(); + let mut old = [0u64; 2]; + for (slot, value) in old.iter_mut().enumerate().take(words) { + *value = Self::load_word(word(slot), width); + } + let new = if kind.is_exclusive_pair() { + self.cpu.exclusive_pair(kind, old) + } else { + kind.apply(old, self.cpu.atomic_operands, width) + }; + let mut changed = false; + for slot in 0..words { + if new[slot] != old[slot] { + Self::store_word(word(slot), width, new[slot]); + changed = true; + } + } + drop(guard); + if changed && let Some(tracker) = self.memory.code_tracker() { + tracker.note_write(physical, span as usize); + } + Ok(Some(old)) + } + fn load_word(at: *mut u8, width: u8) -> u64 { + use core::sync::atomic::{AtomicU8, AtomicU16, AtomicU32, AtomicU64, Ordering::SeqCst}; + // SAFETY: callers pass an address inside a live RAM page, aligned to `width`. + unsafe { + match width { + 1 => u64::from(AtomicU8::from_ptr(at).load(SeqCst)), + 2 => u64::from(AtomicU16::from_ptr(at.cast()).load(SeqCst)), + 4 => u64::from(AtomicU32::from_ptr(at.cast()).load(SeqCst)), + _ => AtomicU64::from_ptr(at.cast()).load(SeqCst), + } + } + } + fn store_word(at: *mut u8, width: u8, value: u64) { + use core::sync::atomic::{AtomicU8, AtomicU16, AtomicU32, AtomicU64, Ordering::SeqCst}; + // SAFETY: callers pass an address inside a live RAM page, aligned to `width`. + unsafe { + match width { + 1 => AtomicU8::from_ptr(at).store(value as u8, SeqCst), + 2 => AtomicU16::from_ptr(at.cast()).store(value as u16, SeqCst), + 4 => AtomicU32::from_ptr(at.cast()).store(value as u32, SeqCst), + _ => AtomicU64::from_ptr(at.cast()).store(value, SeqCst), + } + } + } fn access( &mut self, address: u64, @@ -718,7 +982,7 @@ impl Machine { devices.write_uart(physical - fdt::UART_BASE, width, value); None } - } else if window(fdt::GICD_BASE, crate::devices::gicv2::DISTRIBUTOR_SIZE) { + } else if window(fdt::GICD_BASE, crate::devices::gicv3::DISTRIBUTOR_SIZE) { if load { Some(devices.read_gic_distributor( self.cpu_id, @@ -746,6 +1010,13 @@ impl Machine { ); None } + } else if window(afdt::GICR_BASE, crate::devices::gicv3::REDISTRIBUTOR_SIZE) { + if load { + Some(devices.read_gic_redistributor(physical - afdt::GICR_BASE, width)) + } else { + devices.write_gic_redistributor(physical - afdt::GICR_BASE, width, value); + None + } } else if window(fdt::VIRTIO_MMIO_BASE, fdt::VIRTIO_MMIO_SIZE) { if load { Some(devices.read_virtio(physical - fdt::VIRTIO_MMIO_BASE, width)) @@ -895,6 +1166,97 @@ mod tests { assert_eq!(machine.cpu.system.cntvct_el0, 2); } + #[test] + fn an_architecturally_undefined_word_raises_undef_after_the_decoded_prefix_ran() { + let mut machine = machine(); + machine.cpu.pc = 0x40000000; + machine.cpu.system.vbar_el1 = 0x40000000; + // movz x0, #1; XNU's TRAP_DEBUGGER word; the guest's synchronous vector. + for (at, word) in [ + (0x40000000, 0xd2800020u32), + (0x40000004, 0xe7ffdeff), + (0x40000200, 0xd4000002), + ] { + machine.memory.write(at, &word.to_le_bytes()).unwrap(); + } + let mut serial = Pl011::new(|_| {}); + let mut gic = Gicv2::default(); + let exit = machine.run(&mut serial, &mut gic).unwrap(); + assert!(matches!(exit, Exit::Psci { function: 1, .. })); + assert_eq!(machine.cpu.x[0], 1, "the instruction before it ran"); + assert_eq!( + machine.cpu.system.elr_el1, 0x40000004, + "the exception is taken at the word" + ); + assert_eq!(machine.cpu.system.esr_el1 >> 26, 0, "unknown reason"); + assert_ne!( + machine.cpu.system.esr_el1 & (1 << 25), + 0, + "32-bit instruction" + ); + } + + #[test] + fn the_virtual_timer_is_a_fiq_only_while_every_gate_is_open() { + // (F masked, forwarded by the GIC, ICC_IGRPEN0, ICC_PMR, timer CTL, expect FIQ) + let cases = [ + (false, true, 1, 0xff, 1, true), + (true, true, 1, 0xff, 1, false), + (false, false, 1, 0xff, 1, false), + (false, true, 0, 0xff, 1, false), + (false, true, 1, 0x00, 1, false), + (false, true, 1, 0xff, 3, false), + ]; + for (masked, forwarded, group0, pmr, ctl, expect) in cases { + let mut machine = machine(); + machine.cpu.pc = 0x40000000; + machine.cpu.system.vbar_el1 = 0x40000000; + machine.cpu.system.daif = u64::from(masked); + machine.vtimer_fiq = forwarded.then_some(0); + machine.cpu.system.icc_igrpen0_el1 = group0; + machine.cpu.system.icc_pmr_el1 = pmr; + machine.cpu.system.cntv_ctl_el0 = ctl; + // The mainline code and the FIQ vector (current EL, SPx: +0x200 + 2 * 0x80) + // both end in `hvc`; only a taken FIQ sets ELR. + for at in [0x40000000, 0x40000300] { + machine + .memory + .write(at, &0xd4000002u32.to_le_bytes()) + .unwrap(); + } + let mut serial = Pl011::new(|_| {}); + let mut gic = Gicv2::default(); + let exit = loop { + match machine.run(&mut serial, &mut gic).unwrap() { + Exit::Timer => continue, + other => break other, + } + }; + assert!(matches!(exit, Exit::Psci { .. })); + assert_eq!( + machine.cpu.system.elr_el1 == 0x40000000, + expect, + "masked={masked} forwarded={forwarded} group0={group0} pmr={pmr:#x} ctl={ctl}" + ); + // `ISR_EL1.F` reports the asserted line whether or not the CPU masks it. + let asserted = forwarded && group0 == 1 && pmr > 0 && ctl == 1; + assert_eq!(machine.cpu.system.isr_el1 & (1 << 6) != 0, asserted); + } + } + + #[test] + fn address_translation_reports_the_physical_page_in_par() { + // `at s1e1r, x0` with the MMU off translates to the same address. + let mut machine = machine(); + machine.cpu.x[0] = 0x40000123; + run_to_hvc(&mut machine, &[0xd5087800, HVC]); + assert_eq!(machine.cpu.system.par_el1 & 1, 0, "no fault"); + assert_eq!( + machine.cpu.system.par_el1 & 0x0000_ffff_ffff_f000, + 0x40000000 + ); + } + /// Run guest code at `0x40000000` until it hits the `hvc` that ends it. fn run_to_hvc(machine: &mut Machine, words: &[u32]) { for (i, word) in words.iter().enumerate() { @@ -914,6 +1276,71 @@ mod tests { const LDXR_X1_X0: u32 = 0xc85f7c01; const STXR_W2_X3_X0: u32 = 0xc8027c03; const HVC: u32 = 0xd4000002; + #[test] + fn ld1_of_several_registers_loads_every_register_and_post_indexes() { + // `ld1.2d {v16, v17, v18, v19}, [x10]`, then the same with `, #64`. + for (word, advance) in [(0x4c402d50, 0), (0x4cdf2d50, 64)] { + let mut machine = machine(); + machine.cpu.x[10] = 0x40000800; + for i in 0..8u64 { + machine + .memory + .write(0x40000800 + 8 * i, &(11 + i).to_le_bytes()) + .unwrap(); + } + run_to_hvc(&mut machine, &[word, HVC]); + assert_eq!(machine.cpu.v[16], [11, 12]); + assert_eq!(machine.cpu.v[17], [13, 14]); + assert_eq!(machine.cpu.v[18], [15, 16]); + assert_eq!(machine.cpu.v[19], [17, 18]); + assert_eq!(machine.cpu.x[10], 0x40000800 + advance); + } + } + + const LDXP_X9_X21_X8: u32 = 0xc87f5509; + /// `stxp w11, x19, x10, [x8]`. + const STXP_W11_X19_X10_X8: u32 = 0xc82b2913; + + #[test] + fn exclusive_pair_stores_only_while_both_words_are_unchanged() { + let read_pair = |machine: &mut Machine| { + let mut bytes = [0u8; 16]; + machine.memory.read(0x40000800, &mut bytes).unwrap(); + [ + u64::from_le_bytes(bytes[..8].try_into().unwrap()), + u64::from_le_bytes(bytes[8..].try_into().unwrap()), + ] + }; + for (interfere, status, stored) in [(false, 0, [30, 40]), (true, 1, [1, 99])] { + let mut machine = machine(); + machine.cpu.x[8] = 0x40000800; + machine.cpu.x[19] = 30; + machine.cpu.x[10] = 40; + let mut init = [0u8; 16]; + init[..8].copy_from_slice(&1u64.to_le_bytes()); + init[8..].copy_from_slice(&2u64.to_le_bytes()); + machine.memory.write(0x40000800, &init).unwrap(); + run_to_hvc(&mut machine, &[LDXP_X9_X21_X8, HVC]); + assert_eq!( + (machine.cpu.x[9], machine.cpu.x[21]), + (1, 2), + "ldxp loads both words" + ); + if interfere { + // Another agent changes the second word before the store. + machine + .memory + .write(0x40000808, &99u64.to_le_bytes()) + .unwrap(); + } + run_to_hvc(&mut machine, &[STXP_W11_X19_X10_X8, HVC]); + assert_eq!(machine.cpu.x[11], status, "interfere = {interfere}"); + assert_eq!(read_pair(&mut machine), stored); + // The monitor is spent: a second stxp without a new ldxp fails. + run_to_hvc(&mut machine, &[STXP_W11_X19_X10_X8, HVC]); + assert_eq!(machine.cpu.x[11], 1); + } + } #[test] fn store_exclusive_succeeds_when_memory_is_unchanged() { diff --git a/src/emulator/src/system/mod.rs b/src/emulator/src/system/mod.rs index 47d73d7..2b4740a 100644 --- a/src/emulator/src/system/mod.rs +++ b/src/emulator/src/system/mod.rs @@ -1,16 +1,19 @@ -//! AArch64 Linux full-system execution on the Volt native JIT. -//! Ports Mirage's JIT Machine, raw Image launch and one-vCPU serial/GIC runner. +//! AArch64 Linux and XNU full-system execution on the Volt native JIT. +//! Ports Mirage's JIT Machine, raw Image launch and one-vCPU serial/GIC runner; a +//! Mach-O kernel boots through [`xnu`] on a GICv3 instead of the Linux GICv2. +pub mod afdt; pub mod boot; pub mod esr; pub mod fdt; pub mod machine; pub mod psci; pub mod smp; +pub mod xnu; use crate::aarch64::{Cpu, decode, translate}; use crate::devices::bus::Device; -use crate::devices::{gicv2::Gicv2, pl011::Pl011}; +use crate::devices::{gicv2::Gicv2, gicv3::Gicv3, pl011::Pl011}; use crate::memory::{GuestMemory, PhysicalMemory, Region, SharedMemory}; use alloc::sync::Arc; use core::{ @@ -23,6 +26,11 @@ use std::{io::Write, time::Instant}; pub const RAM_BASE: u64 = 0x4000_0000; pub const RAM_SIZE: usize = 128 << 20; pub const DEFAULT_CMDLINE: &str = "console=ttyAMA0 earlycon=pl011,0x9000000 nokaslr loglevel=8"; +/// Boot arguments for an XNU kernel: verbose boot, console and debug output on the +/// PL011 that the device tree describes as `uart0`. +pub const DEFAULT_XNU_CMDLINE: &str = "-v serial=3 debug=0x14e keepsyms=1 serial-device-name=uart0"; +/// XNU sizes its zones from RAM; 128 MiB leaves it little to work with. +pub const XNU_RAM_SIZE: usize = 1 << 30; #[derive(Debug)] pub enum Error { @@ -95,8 +103,8 @@ impl core::fmt::Display for BootReport { writeln!(f, "--- stopped: {:?}", self.reason)?; writeln!( f, - "entry {:x}, device tree {:x}", - self.layout.entry, self.layout.device_tree + "entry {:x}, boot info {:x}", + self.layout.entry, self.layout.boot_info )?; if let Some(at) = self.stalled_at { writeln!(f, "refused at {at:x}")?; @@ -196,9 +204,48 @@ impl Default for SystemConfig { } } +/// The interrupt controller a boot configures: the GICv2 a Linux device tree +/// describes, or the GICv3 XNU's platform expects. Shared peripherals (UART, virtio) +/// are wired to the GICv2 only; the GICv3 carries the virtual timer, which the +/// machine evaluates live from the timer registers. +enum Intc { + V2(Gicv2), + V3(Gicv3), +} +impl Intc { + fn raise(&mut self, intid: u32) { + if let Self::V2(gic) = self { + gic.raise(intid); + } + } + fn lower(&mut self, intid: u32) { + if let Self::V2(gic) = self { + gic.lower(intid); + } + } + fn raise_on(&mut self, cpu: u32, intid: u32) { + if let Self::V2(gic) = self { + gic.raise_on(cpu, intid); + } + } + fn signalled(&self, cpu: u32) -> bool { + match self { + Self::V2(gic) => gic.signalled(cpu), + Self::V3(_) => false, + } + } + /// See [`machine::Machine::vtimer_fiq`]. + fn vtimer_fiq(&self) -> Option { + match self { + Self::V2(_) => None, + Self::V3(gic) => gic.vtimer_fiq_priority(), + } + } +} + struct SystemDevices<'a, V> { serial: &'a mut Pl011, - gic: &'a mut Gicv2, + gic: &'a mut Intc, virtio: Option<&'a mut V>, } @@ -210,16 +257,41 @@ impl machine::DeviceIo for SystemDevices<'_, V> { self.serial.write(offset, size, value); } fn read_gic_distributor(&mut self, cpu_id: u32, offset: u64, size: u8) -> u64 { - self.gic.read_distributor_for(cpu_id, offset, size) + match self.gic { + Intc::V2(gic) => gic.read_distributor_for(cpu_id, offset, size), + Intc::V3(gic) => gic.read_distributor(offset, size), + } } fn write_gic_distributor(&mut self, cpu_id: u32, offset: u64, size: u8, value: u64) { - self.gic.write_distributor_for(cpu_id, offset, size, value); + match self.gic { + Intc::V2(gic) => { + gic.write_distributor_for(cpu_id, offset, size, value); + } + Intc::V3(gic) => gic.write_distributor(offset, size, value), + } } fn read_gic_cpu(&mut self, cpu_id: u32, offset: u64, size: u8) -> u64 { - self.gic.read_cpu_for(cpu_id, offset, size) + match self.gic { + Intc::V2(gic) => gic.read_cpu_for(cpu_id, offset, size), + // The GICv3 CPU interface is the `ICC_*` system registers. + Intc::V3(_) => 0, + } } fn write_gic_cpu(&mut self, cpu_id: u32, offset: u64, size: u8, value: u64) { - self.gic.write_cpu_for(cpu_id, offset, size, value); + if let Intc::V2(gic) = self.gic { + gic.write_cpu_for(cpu_id, offset, size, value); + } + } + fn read_gic_redistributor(&mut self, offset: u64, size: u8) -> u64 { + match self.gic { + Intc::V3(gic) => gic.read_redistributor(offset, size), + Intc::V2(_) => 0, + } + } + fn write_gic_redistributor(&mut self, offset: u64, size: u8, value: u64) { + if let Intc::V3(gic) = self.gic { + gic.write_redistributor(offset, size, value); + } } fn read_virtio(&mut self, offset: u64, size: u8) -> u64 { if let Some(v) = &mut self.virtio { @@ -307,16 +379,32 @@ pub fn boot_system( let memory = PhysicalMemory::new(vec![region]).map_err(Error::Memory)?; let mut shared_mem = SharedMemory::new(memory); let has_virtio = config.disk.is_some(); - let layout = boot::prepare_boot( - &mut shared_mem, - image, - config.initrd.as_deref(), - &config.cmdline, - RAM_BASE, - ram_size as u64, - 1, - has_virtio, - ) + let is_xnu = xnu::is_macho(image); + let layout = if is_xnu { + if config.initrd.is_some() || has_virtio { + return Err(Error::Boot(boot::Error::Unsupported( + "XNU boot has no initrd or virtio disk support", + ))); + } + xnu::prepare_boot( + &mut shared_mem, + image, + &config.cmdline, + RAM_BASE, + ram_size as u64, + ) + } else { + boot::prepare_boot( + &mut shared_mem, + image, + config.initrd.as_deref(), + &config.cmdline, + RAM_BASE, + ram_size as u64, + 1, + has_virtio, + ) + } .map_err(Error::Boot)?; let console = Arc::new(Mutex::new(Vec::new())); let serial_console = console.clone(); @@ -327,7 +415,11 @@ pub fn boot_system( sink(byte); }) .with_line(move |level| serial_line.store(level, Ordering::Release)); - let mut gic = Gicv2::default(); + let mut gic = if is_xnu { + Intc::V3(Gicv3::default()) + } else { + Intc::V2(Gicv2::default()) + }; let virtio_line = Arc::new(AtomicBool::new(false)); let virtio_notify = virtio_line.clone(); let mut virtio = config.disk.map(|backend| { @@ -336,7 +428,7 @@ pub fn boot_system( }); let mut machine = machine::Machine::new(shared_mem); machine.cpu.pc = layout.entry; - machine.cpu.x[0] = layout.device_tree; + machine.cpu.x[0] = layout.boot_info; machine.cpu.x[1..4].fill(0); let started = Instant::now(); let mut exits = 0; @@ -356,6 +448,7 @@ pub fn boot_system( gic.lower(fdt::VIRTIO_INTID); } machine.irq_line = gic.signalled(0); + machine.vtimer_fiq = gic.vtimer_fiq(); let mut dev = SystemDevices { serial: &mut serial, gic: &mut gic, @@ -425,7 +518,9 @@ pub fn boot_system( } exits += 1; let sp = machine.cpu.sp; - if machine.cpu.system.sctlr_el1 & 1 == 0 + // XNU points `sp` at a kernel virtual address before it enables the MMU. + if !is_xnu + && machine.cpu.system.sctlr_el1 & 1 == 0 && sp != 0 && !(RAM_BASE..=RAM_BASE + ram_size as u64).contains(&sp) { diff --git a/src/emulator/src/system/smp.rs b/src/emulator/src/system/smp.rs index 05666ba..ade790e 100644 --- a/src/emulator/src/system/smp.rs +++ b/src/emulator/src/system/smp.rs @@ -273,7 +273,7 @@ impl fmt::Display for SmpReport { f, "entry {:x}, device tree {:x}, cpus: {}", self.layout.entry, - self.layout.device_tree, + self.layout.boot_info, self.cpus.len() )?; for (idx, c) in self.cpus.iter().enumerate() { @@ -300,6 +300,11 @@ pub fn boot_smp( config: SmpConfig, mut sink: impl FnMut(u8) + Send + 'static, ) -> Result { + if super::xnu::is_macho(image) { + return Err(Error::Boot(boot::Error::Unsupported( + "XNU boot is single-CPU: omit --cpus", + ))); + } let cpu_count = config.cpus.clamp(1, gicv2::MAX_CPUS as u32); let ram_size = if config.ram_size > 0 { config.ram_size @@ -381,7 +386,7 @@ pub fn boot_smp( let idle_mode = config.idle_mode; let timeout = config.timeout; let entry = layout.entry; - let dtb = layout.device_tree; + let dtb = layout.boot_info; let handle = thread::Builder::new() .name(alloc::format!("vcpu-{cpu_id}")) diff --git a/src/emulator/src/system/xnu.rs b/src/emulator/src/system/xnu.rs new file mode 100644 index 0000000..f1a7115 --- /dev/null +++ b/src/emulator/src/system/xnu.rs @@ -0,0 +1,446 @@ +//! XNU boot: a Mach-O kernel, a `boot_args` block and an Apple device tree, +//! handed over the way iBoot does on a cold boot (`osfmk/arm64/start.s` +//! `start_first_cpu`): MMU off, `x0` = physical address of `boot_args`. +//! +//! XNU derives its slide from the PC-relative address of `_mh_execute_header`, +//! so the Mach-O header must sit at `physBase` and every segment at +//! `physBase + (vmaddr - virtBase)`. The kernel then builds its own bootstrap +//! page tables and enables the MMU. +//! +//! Memory, low to high: kernel segments, device tree, `boot_args`, up to +//! `topOfKernelData`; everything above is the kernel's to manage. +use super::boot::{Error, Layout}; +use crate::memory::GuestMemory; + +const MH_MAGIC_64: u32 = 0xfeed_facf; +const CPU_TYPE_ARM64: u32 = 0x0100_000c; +const MH_EXECUTE: u32 = 2; +const LC_UNIXTHREAD: u32 = 0x5; +const LC_SEGMENT_64: u32 = 0x19; +const LC_MAIN: u32 = 0x8000_0028; +const ARM_THREAD_STATE64: u32 = 6; +const HEADER_SIZE: usize = 32; +const SEGMENT_COMMAND_SIZE: usize = 72; +/// `cmd, cmdsize, flavor, count`, then x0-x28, fp, lr, sp, pc. +const THREAD_PC_OFFSET: usize = 16 + 32 * 8; + +/// The kernel's page size (`__ARM_16K_PG__`); `boot_args` regions are padded to it. +const PAGE: u64 = 0x4000; +/// One L2 block of the 16 KiB granule. `start_first_cpu` maps whole L2 blocks +/// and keeps only the VA offset within a block, so physical and virtual +/// addresses must agree modulo this size. +const L2_BLOCK: u64 = 32 << 20; + +const BOOT_ARGS_REVISION: u16 = 2; +const BOOT_ARGS_VERSION: u16 = 2; +pub const BOOT_LINE_LENGTH: usize = 1024; +// `struct boot_args` in `pexpert/pexpert/arm64/boot.h`. +const BA_VIRT_BASE: usize = 8; +const BA_PHYS_BASE: usize = 16; +const BA_MEM_SIZE: usize = 24; +const BA_TOP_OF_KERNEL_DATA: usize = 32; +const BA_DEVICE_TREE: usize = 96; +const BA_DEVICE_TREE_LENGTH: usize = 104; +const BA_COMMAND_LINE: usize = 108; +const BOOT_ARGS_SIZE: usize = 1152; + +/// One `LC_SEGMENT_64`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Segment { + pub vmaddr: u64, + pub vmsize: u64, + pub fileoff: u64, + pub filesize: u64, +} + +/// What the loader needs from a Mach-O kernel. +#[derive(Debug, PartialEq, Eq)] +pub struct Kernel { + /// Link-time address of the Mach-O header (`virtBase` in `boot_args`). + pub virt_base: u64, + /// Link-time address of the entry point. + pub entry: u64, + pub segments: Vec, +} + +pub fn is_macho(image: &[u8]) -> bool { + image.len() >= 4 && u32::from_le_bytes(image[..4].try_into().unwrap()) == MH_MAGIC_64 +} + +fn bad(reason: &'static str) -> Error { + Error::BadMachO(reason) +} + +/// Parse the load commands of an arm64 `MH_EXECUTE` image. The entry point is +/// `LC_MAIN` (relative to the header) or `LC_UNIXTHREAD`'s `pc`. +pub fn parse(image: &[u8]) -> Result { + if image.len() < HEADER_SIZE { + return Err(Error::TooSmall); + } + if !is_macho(image) { + return Err(bad("not a 64-bit little-endian Mach-O")); + } + let word = |at: usize| u32::from_le_bytes(image[at..at + 4].try_into().unwrap()); + let double = |at: usize| u64::from_le_bytes(image[at..at + 8].try_into().unwrap()); + if word(4) != CPU_TYPE_ARM64 { + return Err(bad("not an arm64 Mach-O")); + } + if word(12) != MH_EXECUTE { + return Err(bad("not an executable Mach-O")); + } + let commands_end = HEADER_SIZE + .checked_add(word(20) as usize) + .filter(|&end| end <= image.len()) + .ok_or_else(|| bad("load commands run past the end of the file"))?; + let mut segments = Vec::new(); + let mut main_offset = None; + let mut thread_pc = None; + let mut at = HEADER_SIZE; + for _ in 0..word(16) { + if at + 8 > commands_end { + return Err(bad("load command header runs past the load commands")); + } + let (command, size) = (word(at), word(at + 4) as usize); + if size < 8 || size % 4 != 0 || at + size > commands_end { + return Err(bad("load command size is out of range")); + } + match command { + LC_SEGMENT_64 => { + if size < SEGMENT_COMMAND_SIZE { + return Err(bad("segment command is too short")); + } + let segment = Segment { + vmaddr: double(at + 24), + vmsize: double(at + 32), + fileoff: double(at + 40), + filesize: double(at + 48), + }; + let in_file = segment + .fileoff + .checked_add(segment.filesize) + .is_some_and(|end| end <= image.len() as u64); + if !in_file { + return Err(bad("segment data runs past the end of the file")); + } + if segment.filesize > segment.vmsize { + return Err(bad("segment has more file data than memory")); + } + // Prelink placeholders are empty and linked at unrelated addresses. + if segment.vmsize != 0 { + segments.push(segment); + } + } + LC_MAIN if size >= 16 => main_offset = Some(double(at + 8)), + LC_UNIXTHREAD => { + if size < THREAD_PC_OFFSET + 8 || word(at + 8) != ARM_THREAD_STATE64 { + return Err(bad("LC_UNIXTHREAD is not an ARM_THREAD_STATE64")); + } + thread_pc = Some(double(at + THREAD_PC_OFFSET)); + } + _ => {} + } + at += size; + } + let virt_base = segments + .iter() + .find(|s| s.fileoff == 0 && s.filesize > 0) + .map(|s| s.vmaddr) + .ok_or_else(|| bad("no segment maps the Mach-O header"))?; + let entry = match (main_offset, thread_pc) { + (Some(offset), _) => virt_base.checked_add(offset), + (None, Some(pc)) => Some(pc), + (None, None) => return Err(bad("no LC_MAIN or LC_UNIXTHREAD entry point")), + } + .ok_or_else(|| bad("entry point overflows"))?; + Ok(Kernel { + virt_base, + entry, + segments, + }) +} + +fn align(value: u64, to: u64) -> Result { + value + .checked_add(to - 1) + .map(|v| v & !(to - 1)) + .ok_or(Error::NoRoom) +} + +fn boot_args( + virt_base: u64, + phys_base: u64, + mem_size: u64, + top_of_kernel_data: u64, + device_tree: u64, + device_tree_length: u32, + cmdline: &str, +) -> Result<[u8; BOOT_ARGS_SIZE], Error> { + // The kernel reads a NUL-terminated string out of a fixed field; cutting a + // command line short would silently change what the kernel is told. + if cmdline.len() >= BOOT_LINE_LENGTH { + return Err(Error::Unsupported("XNU boot-args exceed 1023 bytes")); + } + let mut args = [0u8; BOOT_ARGS_SIZE]; + args[0..2].copy_from_slice(&BOOT_ARGS_REVISION.to_le_bytes()); + args[2..4].copy_from_slice(&BOOT_ARGS_VERSION.to_le_bytes()); + let mut put = |at: usize, value: u64| args[at..at + 8].copy_from_slice(&value.to_le_bytes()); + put(BA_VIRT_BASE, virt_base); + put(BA_PHYS_BASE, phys_base); + put(BA_MEM_SIZE, mem_size); + put(BA_TOP_OF_KERNEL_DATA, top_of_kernel_data); + put(BA_DEVICE_TREE, device_tree); + args[BA_DEVICE_TREE_LENGTH..BA_DEVICE_TREE_LENGTH + 4] + .copy_from_slice(&device_tree_length.to_le_bytes()); + args[BA_COMMAND_LINE..BA_COMMAND_LINE + cmdline.len()].copy_from_slice(cmdline.as_bytes()); + Ok(args) +} + +/// Load `image` in the first L2 block of RAM, at the same offset within the +/// block as its link-time base. `boot_args` and the device tree follow it. +/// `Layout::entry` is the physical address of the entry point and +/// `Layout::boot_info` the physical address of `boot_args`. +pub fn prepare_boot( + memory: &mut impl GuestMemory, + image: &[u8], + cmdline: &str, + ram_base: u64, + ram_size: u64, +) -> Result { + let kernel = parse(image)?; + let ram_end = ram_base.checked_add(ram_size).ok_or(Error::NoRoom)?; + // The device tree reserves the last of RAM for the panic log (`chosen/pram`), + // so the kernel's allocator must stop short of it, as `bootxnu` does. + let managed_end = ram_end + .checked_sub(u64::from(super::afdt::PANIC_LOG_SIZE)) + .ok_or(Error::NoRoom)?; + // `physBase` must agree with `virtBase` modulo an L2 block (see `L2_BLOCK`). + let phys_base = align(ram_base, L2_BLOCK)? + .checked_add(kernel.virt_base % L2_BLOCK) + .ok_or(Error::NoRoom)?; + let physical = |virt: u64| { + virt.checked_sub(kernel.virt_base) + .and_then(|offset| phys_base.checked_add(offset)) + .ok_or_else(|| bad("segment lies below the Mach-O header")) + }; + let mut kernel_end = phys_base; + for segment in &kernel.segments { + let end = physical(segment.vmaddr)? + .checked_add(segment.vmsize) + .ok_or(Error::NoRoom)?; + kernel_end = kernel_end.max(end); + } + let entry = physical(kernel.entry)?; + if entry >= kernel_end { + return Err(bad("entry point lies outside the kernel")); + } + + let device_tree = align(kernel_end, PAGE)?; + let tree = super::afdt::build(ram_base, ram_size, cmdline); + let args_at = align( + device_tree + .checked_add(tree.len() as u64) + .ok_or(Error::NoRoom)?, + PAGE, + )?; + // XNU lays its physical aperture out from `topOfKernelData` and, in this + // configuration, lets `physmap_end` leave no room for the random slide of the + // aperture base: the first segment is shifted by up to one L2 block to match + // `physBase`'s offset within a block, and only the region below `topOfKernelData` + // absorbs that shift. Keep it at least one block above `physBase`, whatever the + // seed makes of the slide. + let top_of_kernel_data = align(args_at + BOOT_ARGS_SIZE as u64, PAGE)? + .max(phys_base.checked_add(L2_BLOCK).ok_or(Error::NoRoom)?); + if top_of_kernel_data > managed_end { + return Err(Error::NoRoom); + } + + let args = boot_args( + kernel.virt_base, + phys_base, + managed_end - phys_base, + top_of_kernel_data, + // The kernel reads this pointer after it enables the MMU, through the + // kernel virtual mapping of RAM. + kernel.virt_base + (device_tree - phys_base), + tree.len() as u32, + cmdline, + )?; + // RAM starts zeroed, so only the file-backed part of each segment is copied. + for segment in &kernel.segments { + let data = &image[segment.fileoff as usize..(segment.fileoff + segment.filesize) as usize]; + memory + .write(physical(segment.vmaddr)?, data) + .map_err(Error::Memory)?; + } + memory.write(device_tree, &tree).map_err(Error::Memory)?; + memory.write(args_at, &args).map_err(Error::Memory)?; + Ok(Layout { + entry, + boot_info: args_at, + initrd: None, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::memory::{PhysicalMemory, Region}; + + /// A Mach-O with `__TEXT` (header and code) and `__DATA` (bss only). + fn image(entry: Entry) -> Vec { + let mut image = vec![0u8; 0x1000]; + let put32 = |image: &mut Vec, at: usize, v: u32| { + image[at..at + 4].copy_from_slice(&v.to_le_bytes()) + }; + let put64 = |image: &mut Vec, at: usize, v: u64| { + image[at..at + 8].copy_from_slice(&v.to_le_bytes()) + }; + put32(&mut image, 0, MH_MAGIC_64); + put32(&mut image, 4, CPU_TYPE_ARM64); + put32(&mut image, 12, MH_EXECUTE); + let mut at = HEADER_SIZE; + let mut commands = 0; + let mut segment = |image: &mut Vec, vmaddr, vmsize, fileoff, filesize| { + put32(image, at, LC_SEGMENT_64); + put32(image, at + 4, SEGMENT_COMMAND_SIZE as u32); + put64(image, at + 24, vmaddr); + put64(image, at + 32, vmsize); + put64(image, at + 40, fileoff); + put64(image, at + 48, filesize); + at += SEGMENT_COMMAND_SIZE; + commands += 1; + }; + segment(&mut image, 0xffff_fe00_0700_4000, 0x4000, 0, 0x1000); + segment(&mut image, 0xffff_fe00_0700_8000, 0x8000, 0, 0); + segment(&mut image, 0xffff_fe00_0400_4000, 0, 0, 0); + match entry { + Entry::Main(offset) => { + put32(&mut image, at, LC_MAIN); + put32(&mut image, at + 4, 24); + put64(&mut image, at + 8, offset); + at += 24; + } + Entry::Thread(pc) => { + put32(&mut image, at, LC_UNIXTHREAD); + put32(&mut image, at + 4, (THREAD_PC_OFFSET + 8) as u32); + put32(&mut image, at + 8, ARM_THREAD_STATE64); + put64(&mut image, at + THREAD_PC_OFFSET, pc); + at += THREAD_PC_OFFSET + 8; + } + } + commands += 1; + put32(&mut image, 16, commands); + put32(&mut image, 20, (at - HEADER_SIZE) as u32); + image[0x800..0x804].copy_from_slice(&[0xde, 0xad, 0xbe, 0xef]); + image + } + #[derive(Clone, Copy)] + enum Entry { + Main(u64), + Thread(u64), + } + + #[test] + fn lc_main_is_relative_to_the_header_and_lc_unixthread_is_absolute() { + let main = parse(&image(Entry::Main(0x800))).unwrap(); + let thread = parse(&image(Entry::Thread(0xffff_fe00_0700_4800))).unwrap(); + assert_eq!(main.virt_base, 0xffff_fe00_0700_4000); + assert_eq!(main.entry, 0xffff_fe00_0700_4800); + assert_eq!(main, thread); + assert_eq!( + main.segments.len(), + 2, + "the empty prelink placeholder is dropped" + ); + } + + #[test] + fn truncated_or_inconsistent_images_are_refused() { + let good = image(Entry::Main(0x800)); + assert!(matches!(parse(&good[..20]), Err(Error::TooSmall))); + // The load commands claim more bytes than the file has. + assert!(matches!(parse(&good[..100]), Err(Error::BadMachO(_)))); + let mut data_past_eof = good.clone(); + data_past_eof[HEADER_SIZE + 48..HEADER_SIZE + 56].copy_from_slice(&0x2000u64.to_le_bytes()); + assert!(matches!(parse(&data_past_eof), Err(Error::BadMachO(_)))); + let mut x86 = good.clone(); + x86[4..8].copy_from_slice(&0x0100_0007u32.to_le_bytes()); + assert!(matches!(parse(&x86), Err(Error::BadMachO(_)))); + let mut zero_size_command = good; + zero_size_command[HEADER_SIZE + 4..HEADER_SIZE + 8].copy_from_slice(&0u32.to_le_bytes()); + assert!(matches!(parse(&zero_size_command), Err(Error::BadMachO(_)))); + } + + #[test] + fn segments_land_at_phys_base_plus_their_link_offset_and_boot_args_describe_them() { + const RAM: u64 = 0x4000_0000; + // `virtBase` sits 0x1004000 into its 32 MiB block, so `physBase` does too. + const PHYS: u64 = RAM + 0x1004000; + let region = Region::ram(RAM, vec![0; 128 << 20]).unwrap(); + let mut memory = PhysicalMemory::new(vec![region]).unwrap(); + let layout = prepare_boot( + &mut memory, + &image(Entry::Main(0x800)), + "-v", + RAM, + 128 << 20, + ) + .unwrap(); + assert_eq!(layout.entry, PHYS + 0x800); + let mut code = [0u8; 4]; + memory.read(layout.entry, &mut code).unwrap(); + assert_eq!(code, [0xde, 0xad, 0xbe, 0xef]); + + let mut args = [0u8; BOOT_ARGS_SIZE]; + memory.read(layout.boot_info, &mut args).unwrap(); + let field = |at: usize| u64::from_le_bytes(args[at..at + 8].try_into().unwrap()); + assert_eq!(&args[0..4], &[2, 0, 2, 0], "Revision 2, Version 2"); + assert_eq!(field(BA_VIRT_BASE), 0xffff_fe00_0700_4000); + assert_eq!(field(BA_PHYS_BASE), PHYS); + assert_eq!( + field(BA_PHYS_BASE) % L2_BLOCK, + field(BA_VIRT_BASE) % L2_BLOCK + ); + // The panic log in the last 512 KiB of RAM (`chosen/pram`) is not the kernel's. + assert_eq!( + field(BA_MEM_SIZE), + (RAM + (128 << 20) - u64::from(crate::system::afdt::PANIC_LOG_SIZE)) - PHYS + ); + // `__DATA` ends 0xc000 above the header; the tree and boot_args follow. + let device_tree_pa = field(BA_DEVICE_TREE) - field(BA_VIRT_BASE) + PHYS; + assert_eq!(device_tree_pa, PHYS + 0xc000); + assert!(layout.boot_info > device_tree_pa); + assert_eq!(layout.boot_info % PAGE, 0); + assert_eq!(field(BA_TOP_OF_KERNEL_DATA) % PAGE, 0); + assert!(field(BA_TOP_OF_KERNEL_DATA) >= layout.boot_info + BOOT_ARGS_SIZE as u64); + // XNU's physical aperture has no slack for a random slide in this + // configuration: its first segment is shifted by up to one L2 block to match + // `physBase`'s offset, and that shift is absorbed by the kernel region. + assert!(field(BA_TOP_OF_KERNEL_DATA) - PHYS >= L2_BLOCK); + assert_eq!(&args[BA_COMMAND_LINE..BA_COMMAND_LINE + 3], b"-v\0"); + // The tree the kernel is told about is the tree in memory. + let length = u32::from_le_bytes(args[104..108].try_into().unwrap()) as usize; + let mut tree = vec![0u8; length]; + memory.read(device_tree_pa, &mut tree).unwrap(); + assert_eq!(tree, super::super::afdt::build(RAM, 128 << 20, "-v")); + } + + #[test] + fn a_kernel_that_does_not_fit_and_an_overlong_command_line_are_refused() { + const RAM: u64 = 0x4000_0000; + let mut memory = + PhysicalMemory::new(vec![Region::ram(RAM, vec![0; 0x8000]).unwrap()]).unwrap(); + let good = image(Entry::Main(0x800)); + assert!(matches!( + prepare_boot(&mut memory, &good, "", RAM, 0x8000), + Err(Error::NoRoom) + )); + let mut memory = + PhysicalMemory::new(vec![Region::ram(RAM, vec![0; 64 << 20]).unwrap()]).unwrap(); + let long = "x".repeat(BOOT_LINE_LENGTH); + assert!(matches!( + prepare_boot(&mut memory, &good, &long, RAM, 64 << 20), + Err(Error::Unsupported(_)) + )); + } +} diff --git a/src/emulator/src/user/mod.rs b/src/emulator/src/user/mod.rs index 9603641..7dae319 100644 --- a/src/emulator/src/user/mod.rs +++ b/src/emulator/src/user/mod.rs @@ -105,6 +105,13 @@ pub fn run(path: &Path, args: &[OsString], env: &[(OsString, OsString)]) -> Resu match cpu.trap { Trap::None => {} Trap::Load | Trap::Store => service(&mut space, &mut cpu)?, + Trap::Atomic => atomic(&mut space, &mut cpu)?, + Trap::SimdStruct => simd_struct(&mut space, &mut cpu)?, + Trap::Host => { + let word = cpu.value as u32; + crate::aarch64::host::run(&mut cpu, word); + cpu.trap = Trap::None; + } Trap::Svc => { cpu.trap = Trap::None; cpu.monitor_valid = false; @@ -178,7 +185,11 @@ fn user_instruction(instruction: &Instruction) -> bool { | SystemRegister::CntfrqEl0 => operand.read, _ => false, }, - Instruction::Eret | Instruction::Psci | Instruction::Tlbi | Instruction::Wfi => false, + Instruction::Eret + | Instruction::Psci + | Instruction::Tlbi + | Instruction::Wfi + | Instruction::AddressTranslate(_) => false, _ => true, } } @@ -260,6 +271,80 @@ pub fn service(space: &mut Space, cpu: &mut Cpu) -> Result<(), Error> { cpu.trap = Trap::None; Ok(()) } +/// Retire a SIMD structure load/store through the address space, then apply the +/// post-index writeback. The registers change only if every element succeeded. +fn simd_struct(space: &mut Space, cpu: &mut Cpu) -> Result<(), Error> { + let desc = cpu.structure; + let mut registers = cpu.v; + crate::aarch64::simd_struct::execute( + &desc, + &mut registers, + cpu.address, + |address, width, store| -> Result { + let step = usize::from(width); + if let Some(value) = store { + space.check(address, step, 2)?; + space.write(address, &value.to_le_bytes()[..step])?; + return Ok(0); + } + space.check(address, step, 1)?; + let mut bytes = [0; 8]; + space.read(address, &mut bytes[..step])?; + Ok(u64::from_le_bytes(bytes)) + }, + )?; + cpu.v = registers; + if cpu.writeback { + if cpu.writeback_dest == 31 { + cpu.sp = cpu.writeback_value; + } else { + cpu.x[cpu.writeback_dest as usize] = cpu.writeback_value; + } + cpu.writeback = false; + } + cpu.trap = Trap::None; + Ok(()) +} +/// Retire an LSE atomic. The user-mode machine runs a single vCPU, so a plain +/// read-modify-write through the address space is atomic. +fn atomic(space: &mut Space, cpu: &mut Cpu) -> Result<(), Error> { + let (kind, address, width) = (cpu.atomic_kind, cpu.address, cpu.width); + if !matches!(width, 1 | 2 | 4 | 8) { + return Err(Error::InvalidAccessWidth); + } + let (words, step) = (kind.words(), usize::from(width)); + let span = words * step; + // Read and write permission: the operation does both. + space.check(address, span, 1 | 2)?; + let mut bytes = [0u8; 16]; + space.read(address, &mut bytes[..span])?; + let mut old = [0u64; 2]; + for (slot, value) in old.iter_mut().enumerate().take(words) { + let mut word = [0u8; 8]; + word[..step].copy_from_slice(&bytes[slot * step..][..step]); + *value = u64::from_le_bytes(word); + } + let new = if kind.is_exclusive_pair() { + cpu.exclusive_pair(kind, old) + } else { + kind.apply(old, cpu.atomic_operands, width) + }; + if new != old { + for (slot, value) in new.iter().enumerate().take(words) { + bytes[slot * step..][..step].copy_from_slice(&value.to_le_bytes()[..step]); + } + space.write(address, &bytes[..span])?; + cpu.monitor_valid = false; + } + for (slot, value) in old.into_iter().enumerate().take(words) { + let dest = usize::from(cpu.atomic_dests[slot]); + if dest < 31 { + cpu.x[dest] = value; + } + } + cpu.trap = Trap::None; + Ok(()) +} fn access( space: &mut Space, cpu: &mut Cpu,