candela_vm/
native.rs

1//! Running the machine code a `.cdlb` carries.
2//!
3//! `candela build` compiles the functions it can to machine code and puts
4//! the code in the artifact beside the bytecode, one section per kind of
5//! machine ([`NativeImage`]). Loading a program picks the section this
6//! machine can run, copies its code into memory of its own, makes that memory
7//! executable (never writable and executable at once), and rewrites the calls
8//! the section covers so they call the machine code. Everything else runs on
9//! the interpreter, and any section that does not fit this machine, or memory
10//! the operating system refuses to make executable, leaves the whole program
11//! on bytecode, which is complete on its own.
12//!
13//! The code reaches the VM only through a [`NativeCtx`]: the register file,
14//! the object pool, a few status words and the helpers it calls for anything
15//! the VM does itself. The contract between the two is versioned by
16//! [`abi::VERSION`].
17
18use crate::artifact::NativeArch;
19use crate::artifact::NativeImage;
20use crate::artifact::NativeOs;
21use crate::data::Data;
22use crate::instr::Instr;
23
24/// The contract between generated code and the VM: the layout of the
25/// context, the status codes, and the layout of an object in the pool.
26pub mod abi {
27    use super::NativeCtx;
28    use core::mem::offset_of;
29
30    /// Bumped whenever anything below, the helpers' signatures, or the layout
31    /// of a value changes.
32    pub const VERSION: u32 = 1;
33
34    pub const CTX_REGS: i32 = offset_of!(NativeCtx, regs) as i32;
35    pub const CTX_OBJS: i32 = offset_of!(NativeCtx, objs) as i32;
36    pub const CTX_STATUS: i32 = offset_of!(NativeCtx, status) as i32;
37    pub const CTX_ERR_AT: i32 = offset_of!(NativeCtx, err_at) as i32;
38    pub const CTX_ERR_A: i32 = offset_of!(NativeCtx, err_a) as i32;
39    pub const CTX_ERR_B: i32 = offset_of!(NativeCtx, err_b) as i32;
40    pub const CTX_MARKING: i32 = offset_of!(NativeCtx, marking) as i32;
41    pub const CTX_RET: i32 = offset_of!(NativeCtx, ret) as i32;
42    pub const CTX_DYLIBS: i32 = offset_of!(NativeCtx, dylibs) as i32;
43    pub const CTX_SHADE: i32 = offset_of!(NativeCtx, shade) as i32;
44    pub const CTX_PRINT: i32 = offset_of!(NativeCtx, print) as i32;
45    pub const CTX_CALL_VM: i32 = offset_of!(NativeCtx, call_vm) as i32;
46    pub const CTX_PUSH_ROOTS: i32 = offset_of!(NativeCtx, push_roots) as i32;
47    pub const CTX_POP_ROOTS: i32 = offset_of!(NativeCtx, pop_roots) as i32;
48
49    /// The call went through.
50    pub const STATUS_OK: u32 = 0;
51    /// A list index was out of range: `err_a` is the length, `err_b` the index.
52    pub const STATUS_INDEX: u32 = 1;
53    /// A call would have stood past the call depth limit.
54    pub const STATUS_DEPTH: u32 = 2;
55    /// `main` ran to its end.
56    pub const STATUS_HALT: u32 = 3;
57    /// The interpreter, entered from the code, stopped with an error it
58    /// recorded itself.
59    pub const STATUS_VM: u32 = 4;
60    /// A shift count was outside 0..64: `err_a` is the count.
61    pub const STATUS_SHIFT: u32 = 5;
62    /// A panic in a host function is on its way out; the VM holds it.
63    pub const STATUS_PANIC: u32 = 6;
64
65    /// How an object in the pool is laid out: the size of one entry, and the
66    /// offsets of its buffer pointer and its length. The pool is a vector of
67    /// vectors the code indexes directly, and Rust promises no layout for a
68    /// vector, so the build measures it and the VM checks the build's answer
69    /// against its own before it runs any of the code.
70    #[must_use]
71    pub fn object_layout() -> [u8; 3] {
72        let mut probe: Vec<crate::data::Data> = Vec::with_capacity(7);
73        probe.push(crate::data::NULL);
74        probe.push(crate::data::NULL);
75        probe.push(crate::data::NULL);
76        let words: [usize; 3] = unsafe { core::mem::transmute_copy(&probe) };
77        let offset = |value: usize| {
78            words
79                .iter()
80                .position(|word| *word == value)
81                .map_or(u8::MAX, |i| (i * size_of::<usize>()) as u8)
82        };
83        [
84            size_of::<Vec<crate::data::Data>>() as u8,
85            offset(probe.as_ptr() as usize),
86            offset(probe.len()),
87        ]
88    }
89}
90
91/// What a native function and the VM share while it runs. One lives for each
92/// run of the interpreter, and every call into native code during that run
93/// reuses it.
94#[repr(C)]
95pub struct NativeCtx {
96    /// The register file.
97    pub regs: *mut Data,
98    /// The object pool's first entry. The pool can move when it grows, so
99    /// the VM refreshes this whenever control comes back from it.
100    pub objs: *const u8,
101    /// Why the code stopped, one of the `abi::STATUS_*` codes.
102    pub status: u32,
103    /// The instruction a failure happened at.
104    pub err_at: u32,
105    pub err_a: i64,
106    pub err_b: i64,
107    /// Whether the collector is marking, when a store has to shade what it
108    /// overwrites.
109    pub marking: u64,
110    /// The two words of the value an entry point returns.
111    pub ret: [u64; 2],
112    /// The call depth native code may recurse to before a call carries on in
113    /// the interpreter; each native function is handed how far below it it
114    /// stands.
115    pub ceiling: i64,
116    /// The call depth below the first function of the native code running.
117    pub segment_base: i64,
118    /// The resolved address of each `dylib` function, in binding order.
119    pub dylibs: *const *const u8,
120    /// The interpreter the helpers work on.
121    pub machine: *mut core::ffi::c_void,
122    /// Shades a value a store overwrites while the collector marks.
123    pub shade: extern "C" fn(*mut Self, u64, u64),
124    /// Prints a value.
125    pub print: extern "C" fn(*mut Self, u64, u64),
126    /// Runs a function on the interpreter: its first instruction, the
127    /// register its result lands in, the call instruction, how far below the
128    /// ceiling the caller stands, and the values to keep alive meanwhile.
129    pub call_vm: extern "C" fn(*mut Self, u64, u64, u64, i64, *const u64, u64) -> u32,
130    /// Keeps values the code holds alive across a call that can collect.
131    pub push_roots: extern "C" fn(*mut Self, *const u64, u64),
132    /// Lets go of values [`Self::push_roots`] kept.
133    pub pop_roots: extern "C" fn(*mut Self, u64),
134}
135
136/// The signature of every entry point: the context, and how far below the
137/// ceiling the function stands.
138pub type Entry = unsafe extern "C" fn(*mut NativeCtx, i64) -> u32;
139
140/// One function of the loaded code.
141pub struct NativeFunction {
142    pub entry: Entry,
143    /// The instruction its bytecode starts at.
144    pub loc: u32,
145    pub returns: bool,
146}
147
148/// The machine code of a loaded program.
149pub struct NativeCode {
150    _memory: CodeMemory,
151    pub functions: Vec<NativeFunction>,
152    /// The function that stands in for `main`.
153    pub main: Option<usize>,
154    /// The calls the load rewrote, with the instruction each was, in
155    /// instruction order. Errors name the instruction a program was compiled
156    /// with.
157    originals: Vec<(u32, Instr)>,
158    /// Where a run the code starts on the interpreter returns to.
159    pub exit: u32,
160    /// The most stack one native call takes.
161    pub frame_bytes: u32,
162    /// The resolved `dylib` functions, for the code to call.
163    pub dylibs: Vec<*const u8>,
164}
165
166impl NativeCode {
167    /// The instruction the program was compiled with at `at`, which is
168    /// `current` unless the load rewrote it.
169    #[must_use]
170    pub fn original(&self, at: usize, current: Instr) -> Instr {
171        if !matches!(current, Instr::CallNative(..) | Instr::CallNativeRec(..)) {
172            return current;
173        }
174        self.originals
175            .binary_search_by_key(&(at as u32), |(i, _)| *i)
176            .map_or(current, |k| self.originals[k].1)
177    }
178}
179
180/// This machine, as a section names what it was built for.
181const HOST: Option<(NativeArch, NativeOs)> = {
182    let arch = if cfg!(target_arch = "x86_64") {
183        Some(NativeArch::X86_64)
184    } else if cfg!(target_arch = "aarch64") {
185        Some(NativeArch::Aarch64)
186    } else {
187        None
188    };
189    let os = if cfg!(target_os = "linux") {
190        Some(NativeOs::Linux)
191    } else if cfg!(target_os = "macos") {
192        Some(NativeOs::MacOs)
193    } else if cfg!(target_os = "windows") {
194        Some(NativeOs::Windows)
195    } else {
196        None
197    };
198    match (arch, os) {
199        (Some(arch), Some(os)) => Some((arch, os)),
200        _ => None,
201    }
202};
203
204/// Whether this machine has the instruction set extension `feature`, by the
205/// name Rust's feature detection uses. A name it does not know is a feature
206/// it does not have.
207fn cpu_has(feature: &str) -> bool {
208    #[cfg(target_arch = "x86_64")]
209    {
210        match feature {
211            "sse3" => std::arch::is_x86_feature_detected!("sse3"),
212            "ssse3" => std::arch::is_x86_feature_detected!("ssse3"),
213            "sse4.1" => std::arch::is_x86_feature_detected!("sse4.1"),
214            "sse4.2" => std::arch::is_x86_feature_detected!("sse4.2"),
215            "popcnt" => std::arch::is_x86_feature_detected!("popcnt"),
216            "avx" => std::arch::is_x86_feature_detected!("avx"),
217            "avx2" => std::arch::is_x86_feature_detected!("avx2"),
218            "bmi1" => std::arch::is_x86_feature_detected!("bmi1"),
219            "bmi2" => std::arch::is_x86_feature_detected!("bmi2"),
220            "lzcnt" => std::arch::is_x86_feature_detected!("lzcnt"),
221            "fma" => std::arch::is_x86_feature_detected!("fma"),
222            _ => false,
223        }
224    }
225    #[cfg(all(target_arch = "aarch64", not(target_os = "windows")))]
226    {
227        match feature {
228            "lse" => std::arch::is_aarch64_feature_detected!("lse"),
229            "fp16" => std::arch::is_aarch64_feature_detected!("fp16"),
230            "paca" => std::arch::is_aarch64_feature_detected!("paca"),
231            _ => false,
232        }
233    }
234    #[cfg(not(any(
235        target_arch = "x86_64",
236        all(target_arch = "aarch64", not(target_os = "windows"))
237    )))]
238    {
239        let _ = feature;
240        false
241    }
242}
243
244/// Whether this VM can run `section`: built for this architecture and
245/// operating system, against this contract and object layout, for a
246/// processor this one is at least.
247fn fits(section: &NativeImage) -> bool {
248    HOST == Some((section.arch, section.os))
249        && section.abi == abi::VERSION
250        && section.object_layout == abi::object_layout()
251        && section.cpu_features.iter().all(|f| cpu_has(f))
252}
253
254/// Whether `section` describes code that lines up with `instructions`: every
255/// entry inside the code, every rewritten call a call of the function it
256/// names. A section that fails this was damaged or built for another
257/// program, and runs nothing.
258fn consistent(section: &NativeImage, instructions: &[Instr]) -> bool {
259    let functions = &section.functions;
260    // The run a call into the interpreter starts returns to one instruction
261    // past the end, and a return address is 16 bits.
262    instructions.len() <= usize::from(u16::MAX)
263        && !functions.is_empty()
264        && functions.len() <= usize::from(u16::MAX)
265        && section.frame_bytes > 0
266        && functions.iter().all(|f| {
267            (f.offset as usize) < section.code.len() && (f.loc as usize) < instructions.len()
268        })
269        && section
270            .main
271            .is_none_or(|main| (main as usize) < functions.len())
272        && section.sites.iter().all(|site| {
273            functions.get(site.function as usize).is_some_and(|f| {
274                matches!(
275                    instructions.get(site.at as usize),
276                    Some(Instr::CallFunc(loc, _) | Instr::CallFuncRecursive(loc, _))
277                        if u32::from(*loc) == f.loc
278                )
279            })
280        })
281}
282
283/// Loads the first of `sections` this machine can run: maps its code, and
284/// rewrites the calls it covers in `instructions`. Answers `None`, leaving
285/// `instructions` as they were, when no section fits or the code cannot be
286/// made executable; the program then runs on bytecode alone.
287pub fn load(
288    sections: &[NativeImage],
289    instructions: &mut Vec<Instr>,
290    dylibs: Vec<*const u8>,
291) -> Option<NativeCode> {
292    let section = sections.iter().find(|s| fits(s))?;
293    if !consistent(section, instructions) {
294        return None;
295    }
296    let memory = CodeMemory::new(&section.code)?;
297
298    let functions = section
299        .functions
300        .iter()
301        .map(|f| NativeFunction {
302            // SAFETY: `consistent` checked the offset lies inside the code,
303            // and the build put an entry point there.
304            entry: unsafe { std::mem::transmute::<*const u8, Entry>(memory.at(f.offset as usize)) },
305            loc: f.loc,
306            returns: f.returns,
307        })
308        .collect();
309
310    let mut originals = Vec::with_capacity(section.sites.len());
311    for site in &section.sites {
312        let at = site.at as usize;
313        let before = instructions[at];
314        instructions[at] = match before {
315            Instr::CallFunc(_, ret) => Instr::CallNative(site.function as u16, ret),
316            Instr::CallFuncRecursive(_, ret) => Instr::CallNativeRec(site.function as u16, ret),
317            other => other,
318        };
319        originals.push((site.at, before));
320    }
321    originals.sort_unstable_by_key(|(at, _)| *at);
322    originals.dedup_by_key(|(at, _)| *at);
323
324    let exit = instructions.len() as u32;
325    instructions.push(Instr::NativeExit);
326
327    Some(NativeCode {
328        _memory: memory,
329        functions,
330        main: section.main.map(|m| m as usize),
331        originals,
332        exit,
333        frame_bytes: section.frame_bytes,
334        dylibs,
335    })
336}
337
338/// Memory holding machine code, readable and executable and never writable
339/// once the code is in it. Given back to the operating system on drop.
340struct CodeMemory {
341    ptr: *mut u8,
342    len: usize,
343}
344
345impl CodeMemory {
346    /// The address `offset` bytes into the code.
347    fn at(&self, offset: usize) -> *const u8 {
348        // SAFETY: the caller keeps `offset` inside the code.
349        unsafe { self.ptr.add(offset) }
350    }
351}
352
353#[cfg(unix)]
354mod os {
355    use core::ffi::c_int;
356    use core::ffi::c_void;
357
358    unsafe extern "C" {
359        fn mmap(
360            addr: *mut c_void,
361            len: usize,
362            prot: c_int,
363            flags: c_int,
364            fd: c_int,
365            off: i64,
366        ) -> *mut c_void;
367        fn mprotect(addr: *mut c_void, len: usize, prot: c_int) -> c_int;
368        fn munmap(addr: *mut c_void, len: usize) -> c_int;
369    }
370    #[cfg(target_os = "macos")]
371    unsafe extern "C" {
372        fn sys_icache_invalidate(start: *mut c_void, len: usize);
373    }
374
375    const PROT_READ: c_int = 1;
376    const PROT_WRITE: c_int = 2;
377    const PROT_EXEC: c_int = 4;
378    const MAP_PRIVATE: c_int = 2;
379    #[cfg(target_os = "macos")]
380    const MAP_ANONYMOUS: c_int = 0x1000;
381    #[cfg(not(target_os = "macos"))]
382    const MAP_ANONYMOUS: c_int = 0x20;
383
384    /// Maps fresh memory, copies `code` in, and turns the memory from
385    /// writable into executable. `None` when any step is refused.
386    pub fn map_code(code: &[u8]) -> Option<*mut u8> {
387        let len = code.len();
388        unsafe {
389            let ptr = mmap(
390                core::ptr::null_mut(),
391                len,
392                PROT_READ | PROT_WRITE,
393                MAP_PRIVATE | MAP_ANONYMOUS,
394                -1,
395                0,
396            );
397            if ptr as isize == -1 || ptr.is_null() {
398                return None;
399            }
400            core::ptr::copy_nonoverlapping(code.as_ptr(), ptr.cast::<u8>(), len);
401            flush(ptr, len);
402            if mprotect(ptr, len, PROT_READ | PROT_EXEC) != 0 {
403                munmap(ptr, len);
404                return None;
405            }
406            Some(ptr.cast())
407        }
408    }
409
410    pub fn unmap(ptr: *mut u8, len: usize) {
411        unsafe {
412            munmap(ptr.cast(), len);
413        }
414    }
415
416    /// Makes the instruction cache see freshly written code. Only needed
417    /// where the caches are not kept coherent for code, which is aarch64.
418    #[allow(unused_variables)]
419    unsafe fn flush(ptr: *mut c_void, len: usize) {
420        #[cfg(all(target_arch = "aarch64", target_os = "macos"))]
421        unsafe {
422            sys_icache_invalidate(ptr, len);
423        }
424        #[cfg(all(target_arch = "aarch64", not(target_os = "macos")))]
425        unsafe {
426            super::flush_aarch64(ptr.cast(), len);
427        }
428    }
429}
430
431#[cfg(windows)]
432mod os {
433    use core::ffi::c_void;
434
435    #[link(name = "kernel32")]
436    unsafe extern "system" {
437        fn VirtualAlloc(addr: *mut c_void, size: usize, kind: u32, protect: u32) -> *mut c_void;
438        fn VirtualProtect(addr: *mut c_void, size: usize, protect: u32, old: *mut u32) -> i32;
439        fn VirtualFree(addr: *mut c_void, size: usize, kind: u32) -> i32;
440        fn FlushInstructionCache(process: *mut c_void, addr: *const c_void, size: usize) -> i32;
441        fn GetCurrentProcess() -> *mut c_void;
442    }
443
444    const MEM_COMMIT: u32 = 0x1000;
445    const MEM_RESERVE: u32 = 0x2000;
446    const MEM_RELEASE: u32 = 0x8000;
447    const PAGE_READWRITE: u32 = 0x04;
448    const PAGE_EXECUTE_READ: u32 = 0x20;
449
450    /// Allocates fresh memory, copies `code` in, and turns the memory from
451    /// writable into executable. `None` when any step is refused.
452    pub fn map_code(code: &[u8]) -> Option<*mut u8> {
453        let len = code.len();
454        unsafe {
455            let ptr = VirtualAlloc(
456                core::ptr::null_mut(),
457                len,
458                MEM_COMMIT | MEM_RESERVE,
459                PAGE_READWRITE,
460            );
461            if ptr.is_null() {
462                return None;
463            }
464            core::ptr::copy_nonoverlapping(code.as_ptr(), ptr.cast::<u8>(), len);
465            let mut old = 0;
466            if VirtualProtect(ptr, len, PAGE_EXECUTE_READ, &raw mut old) == 0 {
467                VirtualFree(ptr, 0, MEM_RELEASE);
468                return None;
469            }
470            FlushInstructionCache(GetCurrentProcess(), ptr, len);
471            Some(ptr.cast())
472        }
473    }
474
475    pub fn unmap(ptr: *mut u8, _len: usize) {
476        unsafe {
477            VirtualFree(ptr.cast(), 0, MEM_RELEASE);
478        }
479    }
480}
481
482#[cfg(not(any(unix, windows)))]
483mod os {
484    /// No executable memory on this target: programs run on bytecode.
485    pub fn map_code(_code: &[u8]) -> Option<*mut u8> {
486        None
487    }
488
489    pub fn unmap(_ptr: *mut u8, _len: usize) {}
490}
491
492/// Cleans the data cache and invalidates the instruction cache over freshly
493/// written code, line by line, the way the architecture asks for before new
494/// code runs.
495#[cfg(all(target_arch = "aarch64", unix, not(target_os = "macos")))]
496unsafe fn flush_aarch64(ptr: *const u8, len: usize) {
497    let ctr: u64;
498    unsafe {
499        core::arch::asm!("mrs {}, ctr_el0", out(reg) ctr, options(nomem, nostack));
500    }
501    let dline = 4usize << ((ctr >> 16) & 0xf);
502    let iline = 4usize << (ctr & 0xf);
503    let start = ptr as usize;
504    let end = start + len;
505    let mut addr = start & !(dline - 1);
506    while addr < end {
507        unsafe {
508            core::arch::asm!("dc cvau, {}", in(reg) addr, options(nostack));
509        }
510        addr += dline;
511    }
512    unsafe {
513        core::arch::asm!("dsb ish", options(nostack));
514    }
515    let mut addr = start & !(iline - 1);
516    while addr < end {
517        unsafe {
518            core::arch::asm!("ic ivau, {}", in(reg) addr, options(nostack));
519        }
520        addr += iline;
521    }
522    unsafe {
523        core::arch::asm!("dsb ish", "isb", options(nostack));
524    }
525}
526
527impl CodeMemory {
528    fn new(code: &[u8]) -> Option<Self> {
529        if code.is_empty() {
530            return None;
531        }
532        let ptr = os::map_code(code)?;
533        Some(Self {
534            ptr,
535            len: code.len(),
536        })
537    }
538}
539
540impl Drop for CodeMemory {
541    fn drop(&mut self) {
542        os::unmap(self.ptr, self.len);
543    }
544}
candela_vm/native.rs

candela_vm/
native.rs
