From ef213d60bb29c9acee1eb47211cb341d89f89a6d Mon Sep 17 00:00:00 2001 From: koumingyang <1761674434@qq.com> Date: Thu, 15 Nov 2018 14:22:57 +0800 Subject: [PATCH 01/22] mmu --- crate/aarch64/Cargo.lock | 68 +++ crate/aarch64/Cargo.toml | 16 + crate/aarch64/src/addr.rs | 418 ++++++++++++++ crate/aarch64/src/asm.rs | 162 ++++++ crate/aarch64/src/barrier.rs | 87 +++ crate/aarch64/src/lib.rs | 29 + crate/aarch64/src/paging/frame_alloc.rs | 15 + crate/aarch64/src/paging/mod.rs | 528 +++++++++++++++++ crate/aarch64/src/paging/page_table.rs | 185 ++++++ crate/aarch64/src/paging/recursive.rs | 623 +++++++++++++++++++++ crate/aarch64/src/regs/cntfrq_el0.rs | 31 + crate/aarch64/src/regs/cnthctl_el2.rs | 75 +++ crate/aarch64/src/regs/cntp_ctl_el0.rs | 62 ++ crate/aarch64/src/regs/cntp_tval_el0.rs | 30 + crate/aarch64/src/regs/cntpct_el0.rs | 29 + crate/aarch64/src/regs/cntvoff_el2.rs | 32 ++ crate/aarch64/src/regs/currentel.rs | 52 ++ crate/aarch64/src/regs/daif.rs | 90 +++ crate/aarch64/src/regs/elr_el2.rs | 30 + crate/aarch64/src/regs/hcr_el2.rs | 123 ++++ crate/aarch64/src/regs/id_aa64mmfr0_el1.rs | 82 +++ crate/aarch64/src/regs/macros.rs | 85 +++ crate/aarch64/src/regs/mair_el1.rs | 105 ++++ crate/aarch64/src/regs/mod.rs | 51 ++ crate/aarch64/src/regs/mpidr_el1.rs | 30 + crate/aarch64/src/regs/sctlr_el1.rs | 103 ++++ crate/aarch64/src/regs/sp.rs | 28 + crate/aarch64/src/regs/sp_el0.rs | 31 + crate/aarch64/src/regs/sp_el1.rs | 36 ++ crate/aarch64/src/regs/spsel.rs | 48 ++ crate/aarch64/src/regs/spsr_el2.rs | 106 ++++ crate/aarch64/src/regs/tcr_el1.rs | 178 ++++++ crate/aarch64/src/regs/ttbr0_el1.rs | 56 ++ kernel/Cargo.lock | 14 + kernel/Cargo.toml | 1 + kernel/src/arch/aarch64/memory.rs | 65 ++- kernel/src/arch/aarch64/paging.rs | 339 ++++++----- kernel/src/lib.rs | 2 + 38 files changed, 3896 insertions(+), 149 deletions(-) create mode 100644 crate/aarch64/Cargo.lock create mode 100644 crate/aarch64/Cargo.toml create mode 100644 crate/aarch64/src/addr.rs create mode 100644 crate/aarch64/src/asm.rs create mode 100644 crate/aarch64/src/barrier.rs create mode 100644 crate/aarch64/src/lib.rs create mode 100644 crate/aarch64/src/paging/frame_alloc.rs create mode 100644 crate/aarch64/src/paging/mod.rs create mode 100644 crate/aarch64/src/paging/page_table.rs create mode 100644 crate/aarch64/src/paging/recursive.rs create mode 100644 crate/aarch64/src/regs/cntfrq_el0.rs create mode 100644 crate/aarch64/src/regs/cnthctl_el2.rs create mode 100644 crate/aarch64/src/regs/cntp_ctl_el0.rs create mode 100644 crate/aarch64/src/regs/cntp_tval_el0.rs create mode 100644 crate/aarch64/src/regs/cntpct_el0.rs create mode 100644 crate/aarch64/src/regs/cntvoff_el2.rs create mode 100644 crate/aarch64/src/regs/currentel.rs create mode 100644 crate/aarch64/src/regs/daif.rs create mode 100644 crate/aarch64/src/regs/elr_el2.rs create mode 100644 crate/aarch64/src/regs/hcr_el2.rs create mode 100644 crate/aarch64/src/regs/id_aa64mmfr0_el1.rs create mode 100644 crate/aarch64/src/regs/macros.rs create mode 100644 crate/aarch64/src/regs/mair_el1.rs create mode 100644 crate/aarch64/src/regs/mod.rs create mode 100644 crate/aarch64/src/regs/mpidr_el1.rs create mode 100644 crate/aarch64/src/regs/sctlr_el1.rs create mode 100644 crate/aarch64/src/regs/sp.rs create mode 100644 crate/aarch64/src/regs/sp_el0.rs create mode 100644 crate/aarch64/src/regs/sp_el1.rs create mode 100644 crate/aarch64/src/regs/spsel.rs create mode 100644 crate/aarch64/src/regs/spsr_el2.rs create mode 100644 crate/aarch64/src/regs/tcr_el1.rs create mode 100644 crate/aarch64/src/regs/ttbr0_el1.rs diff --git a/crate/aarch64/Cargo.lock b/crate/aarch64/Cargo.lock new file mode 100644 index 0000000..2d0a329 --- /dev/null +++ b/crate/aarch64/Cargo.lock @@ -0,0 +1,68 @@ +[[package]] +name = "aarch64" +version = "0.1.0" +dependencies = [ + "bit_field 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", + "cortex-a 2.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "os_bootinfo 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "register 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "usize_conversions 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "ux 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "bit_field" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "bitflags" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "cortex-a" +version = "2.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "register 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "os_bootinfo" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "register" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "tock-registers 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tock-registers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "usize_conversions" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "ux" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[metadata] +"checksum bit_field 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ed8765909f9009617974ab6b7d332625b320b33c326b1e9321382ef1999b5d56" +"checksum bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "228047a76f468627ca71776ecdebd732a3423081fcf5125585bcd7c49886ce12" +"checksum cortex-a 2.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "2b187d0d728b4a99ba1d79f9671b976bcdd71a8a2c719585218fd2dc14a4d08c" +"checksum os_bootinfo 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "66481dbeb5e773e7bd85b63cd6042c30786f834338288c5ec4f3742673db360a" +"checksum register 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e10f31b6d2299e5620986ad9fcdd66463e125ad72af4f403f9aedf7592d5ccdb" +"checksum tock-registers 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3a385d94f3f62e60445a0adb9ff8d9621faa272234530d4c0f848ec98f88e316" +"checksum usize_conversions 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f70329e2cbe45d6c97a5112daad40c34cd9a4e18edb5a2a18fefeb584d8d25e5" +"checksum ux 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "53d8df5dd8d07fedccd202de1887d94481fadaea3db70479f459e8163a1fab41" diff --git a/crate/aarch64/Cargo.toml b/crate/aarch64/Cargo.toml new file mode 100644 index 0000000..7514dd0 --- /dev/null +++ b/crate/aarch64/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "aarch64" +version = "0.1.0" +authors = ["koumingyang <1761674434@qq.com>"] + +[dependencies] +register = "0.2.0" +bit_field = "0.9.0" +bitflags = "1.0.1" +usize_conversions = "0.2.0" +os_bootinfo = "0.2.0" +bare-metal = "0.2.0" + +[dependencies.ux] +default-features = false +version = "0.1.0" \ No newline at end of file diff --git a/crate/aarch64/src/addr.rs b/crate/aarch64/src/addr.rs new file mode 100644 index 0000000..e347567 --- /dev/null +++ b/crate/aarch64/src/addr.rs @@ -0,0 +1,418 @@ +use core::convert::{Into, TryInto}; +use core::fmt; +use core::ops::{Add, AddAssign, Sub, SubAssign}; + +use bit_field::BitField; +use usize_conversions::FromUsize; +use ux::*; + +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +#[repr(transparent)] +pub struct VirtAddr(u64); + +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +#[repr(transparent)] +pub struct PhysAddr(u64); + +#[derive(Debug)] +pub struct VirtAddrNotValid(u64); + +impl VirtAddr { + /// Creates a new canonical virtual address. + /// + /// This function performs sign extension of bit 47 to make the address canonical. Panics + /// if the bits in the range 48 to 64 contain data (i.e. are not null and no sign extension). + pub fn new(addr: u64) -> VirtAddr { + Self::try_new(addr).expect( + "invalid virtual address", + ) + } + + /// Tries to create a new canonical virtual address. + /// in aarch64, valid virtual address starts with 0x0000 or 0xffff. + pub fn try_new(addr: u64) -> Result { + match addr.get_bits(48..64) { + 0 | 0xffff => Ok(VirtAddr(addr)), // address is canonical + other => Err(VirtAddrNotValid(other)), + } + } + + pub fn new_unchecked(addr: u64) -> VirtAddr { + VirtAddr(addr) + } + + /// Creates a virtual address that points to `0`. + pub const fn zero() -> VirtAddr { + VirtAddr(0) + } + + /// Converts the address to an `u64`. + pub fn as_u64(self) -> u64 { + self.0 + } + + /// Creates a virtual address from the given pointer + pub fn from_ptr(ptr: *const T) -> Self { + use usize_conversions::FromUsize; + + Self::new(u64::from_usize(ptr as usize)) + } + + /// Converts the address to a raw pointer. + #[cfg(target_pointer_width = "64")] + pub fn as_ptr(self) -> *const T { + use usize_conversions::usize_from; + + usize_from(self.as_u64()) as *const T + } + + /// Converts the address to a mutable raw pointer. + #[cfg(target_pointer_width = "64")] + pub fn as_mut_ptr(self) -> *mut T { + self.as_ptr::() as *mut T + } + + /// Aligns the virtual address upwards to the given alignment. + /// + /// See the `align_up` function for more information. + pub fn align_up(self, align: U) -> Self + where + U: Into, + { + VirtAddr(align_up(self.0, align.into())) + } + + /// Aligns the virtual address downwards to the given alignment. + /// + /// See the `align_down` function for more information. + pub fn align_down(self, align: U) -> Self + where + U: Into, + { + VirtAddr(align_down(self.0, align.into())) + } + + /// Checks whether the virtual address has the demanded alignment. + pub fn is_aligned(self, align: U) -> bool + where + U: Into, + { + self.align_down(align) == self + } + + /// Returns the 12-bit page offset of this virtual address. + pub fn page_offset(&self) -> u12 { + u12::new((self.0 & 0xfff).try_into().unwrap()) + } + + /// Returns the 9-bit level 1 page table index. + pub fn p1_index(&self) -> u9 { + u9::new(((self.0 >> 12) & 0o777).try_into().unwrap()) + } + + /// Returns the 9-bit level 2 page table index. + pub fn p2_index(&self) -> u9 { + u9::new(((self.0 >> 12 >> 9) & 0o777).try_into().unwrap()) + } + + /// Returns the 9-bit level 3 page table index. + pub fn p3_index(&self) -> u9 { + u9::new(((self.0 >> 12 >> 9 >> 9) & 0o777).try_into().unwrap()) + } + + /// Returns the 9-bit level 4 page table index. + pub fn p4_index(&self) -> u9 { + u9::new(((self.0 >> 12 >> 9 >> 9 >> 9) & 0o777).try_into().unwrap()) + } +} + +impl fmt::Debug for VirtAddr { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "VirtAddr({:#x})", self.0) + } +} + +impl Add for VirtAddr { + type Output = Self; + fn add(self, rhs: u64) -> Self::Output { + VirtAddr::new(self.0 + rhs) + } +} + +impl AddAssign for VirtAddr { + fn add_assign(&mut self, rhs: u64) { + *self = *self + rhs; + } +} + +impl Add for VirtAddr +where + u64: FromUsize, +{ + type Output = Self; + fn add(self, rhs: usize) -> Self::Output { + self + u64::from_usize(rhs) + } +} + +impl AddAssign for VirtAddr +where + u64: FromUsize, +{ + fn add_assign(&mut self, rhs: usize) { + self.add_assign(u64::from_usize(rhs)) + } +} + +impl Sub for VirtAddr { + type Output = Self; + fn sub(self, rhs: u64) -> Self::Output { + VirtAddr::new(self.0.checked_sub(rhs).unwrap()) + } +} + +impl SubAssign for VirtAddr { + fn sub_assign(&mut self, rhs: u64) { + *self = *self - rhs; + } +} + +impl Sub for VirtAddr +where + u64: FromUsize, +{ + type Output = Self; + fn sub(self, rhs: usize) -> Self::Output { + self - u64::from_usize(rhs) + } +} + +impl SubAssign for VirtAddr +where + u64: FromUsize, +{ + fn sub_assign(&mut self, rhs: usize) { + self.sub_assign(u64::from_usize(rhs)) + } +} + +impl Sub for VirtAddr { + type Output = u64; + fn sub(self, rhs: VirtAddr) -> Self::Output { + self.as_u64().checked_sub(rhs.as_u64()).unwrap() + } +} + +/// A passed `u64` was not a valid physical address. +/// +/// This means that bits 52 to 64 are not were not all null. +#[derive(Debug)] +pub struct PhysAddrNotValid(u64); + +impl PhysAddr { + /// Creates a new physical address. + /// + /// Panics if a bit in the range 52 to 64 is set. + pub fn new(addr: u64) -> PhysAddr { + assert_eq!( + addr.get_bits(52..64), + 0, + "physical addresses must not have any bits in the range 52 to 64 set" + ); + PhysAddr(addr) + } + + /// Tries to create a new physical address. + /// + /// Fails if any bits in the range 52 to 64 are set. + pub fn try_new(addr: u64) -> Result { + match addr.get_bits(52..64) { + 0 => Ok(PhysAddr(addr)), // address is valid + other => Err(PhysAddrNotValid(other)), + } + } + + /// Converts the address to an `u64`. + pub fn as_u64(self) -> u64 { + self.0 + } + + /// Convenience method for checking if a physical address is null. + pub fn is_null(&self) -> bool { + self.0 == 0 + } + + /// Aligns the physical address upwards to the given alignment. + /// + /// See the `align_up` function for more information. + pub fn align_up(self, align: U) -> Self + where + U: Into, + { + PhysAddr(align_up(self.0, align.into())) + } + + /// Aligns the physical address downwards to the given alignment. + /// + /// See the `align_down` function for more information. + pub fn align_down(self, align: U) -> Self + where + U: Into, + { + PhysAddr(align_down(self.0, align.into())) + } + + /// Checks whether the physical address has the demanded alignment. + pub fn is_aligned(self, align: U) -> bool + where + U: Into, + { + self.align_down(align) == self + } +} + +impl fmt::Debug for PhysAddr { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "PhysAddr({:#x})", self.0) + } +} + +impl fmt::Binary for PhysAddr { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + self.0.fmt(f) + } +} + +impl fmt::LowerHex for PhysAddr { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + self.0.fmt(f) + } +} + +impl fmt::Octal for PhysAddr { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + self.0.fmt(f) + } +} + +impl fmt::UpperHex for PhysAddr { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + self.0.fmt(f) + } +} + +impl Add for PhysAddr { + type Output = Self; + fn add(self, rhs: u64) -> Self::Output { + PhysAddr::new(self.0 + rhs) + } +} + +impl AddAssign for PhysAddr { + fn add_assign(&mut self, rhs: u64) { + *self = *self + rhs; + } +} + +impl Add for PhysAddr +where + u64: FromUsize, +{ + type Output = Self; + fn add(self, rhs: usize) -> Self::Output { + self + u64::from_usize(rhs) + } +} + +impl AddAssign for PhysAddr +where + u64: FromUsize, +{ + fn add_assign(&mut self, rhs: usize) { + self.add_assign(u64::from_usize(rhs)) + } +} + +impl Sub for PhysAddr { + type Output = Self; + fn sub(self, rhs: u64) -> Self::Output { + PhysAddr::new(self.0.checked_sub(rhs).unwrap()) + } +} + +impl SubAssign for PhysAddr { + fn sub_assign(&mut self, rhs: u64) { + *self = *self - rhs; + } +} + +impl Sub for PhysAddr +where + u64: FromUsize, +{ + type Output = Self; + fn sub(self, rhs: usize) -> Self::Output { + self - u64::from_usize(rhs) + } +} + +impl SubAssign for PhysAddr +where + u64: FromUsize, +{ + fn sub_assign(&mut self, rhs: usize) { + self.sub_assign(u64::from_usize(rhs)) + } +} + +impl Sub for PhysAddr { + type Output = u64; + fn sub(self, rhs: PhysAddr) -> Self::Output { + self.as_u64().checked_sub(rhs.as_u64()).unwrap() + } +} + +/// Align address downwards. +/// +/// Returns the greatest x with alignment `align` so that x <= addr. The alignment must be +/// a power of 2. +pub fn align_down(addr: u64, align: u64) -> u64 { + assert!(align.is_power_of_two(), "`align` must be a power of two"); + addr & !(align - 1) +} + +/// Align address upwards. +/// +/// Returns the smallest x with alignment `align` so that x >= addr. The alignment must be +/// a power of 2. +pub fn align_up(addr: u64, align: u64) -> u64 { + assert!(align.is_power_of_two(), "`align` must be a power of two"); + let align_mask = align - 1; + if addr & align_mask == 0 { + addr // already aligned + } else { + (addr | align_mask) + 1 + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + pub fn test_align_up() { + // align 1 + assert_eq!(align_up(0, 1), 0); + assert_eq!(align_up(1234, 1), 1234); + assert_eq!(align_up(0xffffffffffffffff, 1), 0xffffffffffffffff); + // align 2 + assert_eq!(align_up(0, 2), 0); + assert_eq!(align_up(1233, 2), 1234); + assert_eq!(align_up(0xfffffffffffffffe, 2), 0xfffffffffffffffe); + // address 0 + assert_eq!(align_up(0, 128), 0); + assert_eq!(align_up(0, 1), 0); + assert_eq!(align_up(0, 2), 0); + assert_eq!(align_up(0, 0x8000000000000000), 0); + } +} diff --git a/crate/aarch64/src/asm.rs b/crate/aarch64/src/asm.rs new file mode 100644 index 0000000..d97fa78 --- /dev/null +++ b/crate/aarch64/src/asm.rs @@ -0,0 +1,162 @@ +use paging::PhysFrame; +use addr::PhysAddr; +use regs::*; + +#[inline(always)] +pub fn tlb_invalidate() { + unsafe{ + asm!("dsb ishst + tlbi vmalle1is + dsb ish + tlbi vmalle1is + isb"); + } +} + +/// Returns the current stack pointer. +#[inline(always)] +pub fn sp() -> *const u8 { + let ptr: usize; + unsafe { + asm!("mov $0, sp" : "=r"(ptr)); + } + + ptr as *const u8 +} + +#[inline(always)] +pub unsafe fn get_pc() -> usize { + let pc: usize; + asm!("ADR $0, ." : "=r"(pc)); + pc +} + +/// Returns the current exception level. +/// +/// # Safety +/// This function should only be called when EL is >= 1. +#[inline(always)] +pub unsafe fn current_el() -> u8 { + let el_reg: u64; + asm!("mrs $0, CurrentEL" : "=r"(el_reg)); + ((el_reg & 0b1100) >> 2) as u8 +} + +#[inline(always)] +pub unsafe fn get_far() -> usize { + let far: usize; + asm!("mrs $0, far_el1" : "=r"(far)); + far +} + +#[inline(always)] +pub unsafe fn get_ttbr0() -> usize { + let ttbr0: usize; + asm!("mrs $0, ttbr0_el1" : "=r"(ttbr0)); + ttbr0 +} + +#[inline(always)] +pub unsafe fn get_ttbr1() -> usize { + let ttbr0: usize; + asm!("mrs $0, ttbr1_el1" : "=r"(ttbr0)); + ttbr0 +} + +/// Returns the SPSel value. +#[inline(always)] +pub fn sp_sel() -> u8 { + let ptr: u32; + unsafe { + asm!("mrs $0, SPSel" : "=r"(ptr)); + } + + (ptr & 1) as u8 +} + +/// Returns the core currently executing. +/// +/// # Safety +/// +/// This function should only be called when EL is >= 1. +pub unsafe fn affinity() -> usize { + let x: usize; + asm!("mrs $0, mpidr_el1 + and $0, $0, #3" + : "=r"(x)); + + x +} + +pub fn wfi() { + unsafe { + asm!("wfi" :::: "volatile"); + } +} + + +/// The classic no-op +#[inline] +pub fn nop() { + match () { + #[cfg(target_arch = "aarch64")] + () => unsafe { asm!("nop" :::: "volatile") }, + + #[cfg(not(target_arch = "aarch64"))] + () => unimplemented!(), + } +} + +/// Wait For Event +#[inline] +pub fn wfe() { + match () { + #[cfg(target_arch = "aarch64")] + () => unsafe { asm!("wfe" :::: "volatile") }, + + #[cfg(not(target_arch = "aarch64"))] + () => unimplemented!(), + } +} + +/// Exception return +/// +/// Will jump to wherever the corresponding link register points to, and +/// therefore never return. +#[inline] +pub fn eret() -> ! { + use core; + + match () { + #[cfg(target_arch = "aarch64")] + () => unsafe { + asm!("eret" :::: "volatile"); + core::intrinsics::unreachable() + }, + + #[cfg(not(target_arch = "aarch64"))] + () => unimplemented!(), + } +} + +bitflags! { + /// Controls cache settings for the level 4 page table. + pub struct ttbr0_el1_Flags: u64 { + + const COMMON_NOT_PRIVATE = 1 << 0; + } +} + +pub fn ttbr0_el1_read() -> (PhysFrame, ttbr0_el1_Flags) { + let value = TTBR0_EL1.get(); + let flags = ttbr0_el1_Flags::from_bits_truncate(value); + let addr = PhysAddr::new(value & 0x_000f_ffff_ffff_f000); + let frame = PhysFrame::containing_address(addr); + (frame, flags) +} + +pub fn ttbr0_el1_write(frame: PhysFrame) { + let addr = frame.start_address(); + let value = addr.as_u64(); + TTBR0_EL1.set_baddr(value); +} diff --git a/crate/aarch64/src/barrier.rs b/crate/aarch64/src/barrier.rs new file mode 100644 index 0000000..0e48209 --- /dev/null +++ b/crate/aarch64/src/barrier.rs @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2018 by the author(s) + * + * ============================================================================= + * + * Licensed under either of + * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + * - MIT License (http://opensource.org/licenses/MIT) + * at your option. + * + * ============================================================================= + * + * Author(s): + * - Andre Richter + */ + +// Borrow implementations from the pending upstream ACLE implementation until it +// is merged. Afterwards, we'll probably just reexport them, hoping that the API +// doesn't change. +// +// https://github.com/rust-lang-nursery/stdsimd/pull/557 + +mod sealed { + pub trait Dmb { + unsafe fn __dmb(&self); + } + + pub trait Dsb { + unsafe fn __dsb(&self); + } + + pub trait Isb { + unsafe fn __isb(&self); + } +} + +macro_rules! dmb_dsb { + ($A:ident) => { + impl sealed::Dmb for $A { + #[inline(always)] + unsafe fn __dmb(&self) { + asm!(concat!("DMB ", stringify!($A)) : : : "memory" : "volatile") + } + } + impl sealed::Dsb for $A { + #[inline(always)] + unsafe fn __dsb(&self) { + asm!(concat!("DSB ", stringify!($A)) : : : "memory" : "volatile") + } + } + }; +} + +pub struct SY; + +dmb_dsb!(SY); + +impl sealed::Isb for SY { + #[inline(always)] + unsafe fn __isb(&self) { + asm!("ISB SY" : : : "memory" : "volatile") + } +} + +#[inline(always)] +pub unsafe fn dmb(arg: A) +where + A: sealed::Dmb, +{ + arg.__dmb() +} + +#[inline(always)] +pub unsafe fn dsb(arg: A) +where + A: sealed::Dsb, +{ + arg.__dsb() +} + +#[inline(always)] +pub unsafe fn isb(arg: A) +where + A: sealed::Isb, +{ + arg.__isb() +} diff --git a/crate/aarch64/src/lib.rs b/crate/aarch64/src/lib.rs new file mode 100644 index 0000000..bb8bf72 --- /dev/null +++ b/crate/aarch64/src/lib.rs @@ -0,0 +1,29 @@ +#![no_std] +//#![deny(warnings)] +#![feature(asm)] +#![feature(const_fn)] +#![feature(core_intrinsics)] +#![feature(try_from)] + +extern crate bare_metal; +#[macro_use] +extern crate register; +#[macro_use] +extern crate bitflags; +extern crate bit_field; +extern crate os_bootinfo; +extern crate usize_conversions; + +/// Provides the non-standard-width integer types `u2`–`u63`. +/// +/// We use these integer types in various APIs, for example `u9` for page tables indices. +pub extern crate ux; + +pub use addr::{align_down, align_up, PhysAddr, VirtAddr}; + +pub mod asm; +pub mod addr; +pub mod paging; +pub mod barrier; +pub mod regs; + diff --git a/crate/aarch64/src/paging/frame_alloc.rs b/crate/aarch64/src/paging/frame_alloc.rs new file mode 100644 index 0000000..f3ea4c8 --- /dev/null +++ b/crate/aarch64/src/paging/frame_alloc.rs @@ -0,0 +1,15 @@ +//! Traits for abstracting away frame allocation and deallocation. + +use paging::{PageSize, PhysFrame}; + +/// A trait for types that can allocate a frame of memory. +pub trait FrameAllocator { + /// Allocate a frame of the appropriate size and return it if possible. + fn alloc(&mut self) -> Option>; +} + +/// A trait for types that can deallocate a frame of memory. +pub trait FrameDeallocator { + /// Deallocate the given frame of memory. + fn dealloc(&mut self, frame: PhysFrame); +} diff --git a/crate/aarch64/src/paging/mod.rs b/crate/aarch64/src/paging/mod.rs new file mode 100644 index 0000000..56073b2 --- /dev/null +++ b/crate/aarch64/src/paging/mod.rs @@ -0,0 +1,528 @@ +//! Abstractions for page tables and other paging related structures. +//! +//! Page tables translate virtual memory “pages” to physical memory “frames”. + +pub use self::frame_alloc::*; +pub use self::page_table::*; +#[cfg(target_arch = "aarch64")] +pub use self::recursive::*; + +use core::fmt; +use core::marker::PhantomData; +use core::ops::{Add, AddAssign, Sub, SubAssign}; +use os_bootinfo; +use ux::*; +use addr::{PhysAddr, VirtAddr}; + +mod frame_alloc; +mod page_table; +mod recursive; + +/// Trait for abstracting over the three possible page sizes on x86_64, 4KiB, 2MiB, 1GiB. +pub trait PageSize: Copy + Eq + PartialOrd + Ord { + /// The page size in bytes. + const SIZE: u64; + + /// A string representation of the page size for debug output. + const SIZE_AS_DEBUG_STR: &'static str; +} + +/// This trait is implemented for 4KiB and 2MiB pages, but not for 1GiB pages. +pub trait NotGiantPageSize: PageSize {} + +/// A standard 4KiB page. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum Size4KiB {} + +/// A “huge” 2MiB page. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum Size2MiB {} + +/// A “giant” 1GiB page. +/// +/// (Only available on newer x86_64 CPUs.) +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum Size1GiB {} + +impl PageSize for Size4KiB { + const SIZE: u64 = 4096; + const SIZE_AS_DEBUG_STR: &'static str = "4KiB"; +} + +impl NotGiantPageSize for Size4KiB {} + +impl PageSize for Size2MiB { + const SIZE: u64 = Size4KiB::SIZE * 512; + const SIZE_AS_DEBUG_STR: &'static str = "2MiB"; +} + +impl NotGiantPageSize for Size2MiB {} + +impl PageSize for Size1GiB { + const SIZE: u64 = Size2MiB::SIZE * 512; + const SIZE_AS_DEBUG_STR: &'static str = "1GiB"; +} + +/// A virtual memory page. +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +#[repr(C)] +pub struct Page { + start_address: VirtAddr, + size: PhantomData, +} + +impl Page { + /// Returns the page that starts at the given virtual address. + /// + /// Returns an error if the address is not correctly aligned (i.e. is not a valid page start). + pub fn from_start_address(address: VirtAddr) -> Result { + if !address.is_aligned(S::SIZE) { + return Err(()); + } + Ok(Page::containing_address(address)) + } + + /// Returns the page that contains the given virtual address. + pub fn containing_address(address: VirtAddr) -> Self { + Page { + start_address: address.align_down(S::SIZE), + size: PhantomData, + } + } + + /// Returns the start address of the page. + pub fn start_address(&self) -> VirtAddr { + self.start_address + } + + /// Returns the size the page (4KB, 2MB or 1GB). + pub const fn size(&self) -> u64 { + S::SIZE + } + + /// Returns the level 4 page table index of this page. + pub fn p4_index(&self) -> u9 { + self.start_address().p4_index() + } + + /// Returns the level 3 page table index of this page. + pub fn p3_index(&self) -> u9 { + self.start_address().p3_index() + } + + /// Returns a range of pages, exclusive `end`. + pub fn range(start: Self, end: Self) -> PageRange { + PageRange { start, end } + } + + /// Returns a range of pages, inclusive `end`. + pub fn range_inclusive(start: Self, end: Self) -> PageRangeInclusive { + PageRangeInclusive { start, end } + } +} + +impl Page { + /// Returns the level 2 page table index of this page. + pub fn p2_index(&self) -> u9 { + self.start_address().p2_index() + } +} + +impl Page { + /// Returns the 1GiB memory page with the specified page table indices. + pub fn from_page_table_indices_1gib(p4_index: u9, p3_index: u9) -> Self { + use bit_field::BitField; + + let mut addr = 0; + addr.set_bits(39..48, u64::from(p4_index)); + addr.set_bits(30..39, u64::from(p3_index)); + Page::containing_address(VirtAddr::new(addr)) + } +} + +impl Page { + /// Returns the 2MiB memory page with the specified page table indices. + pub fn from_page_table_indices_2mib(p4_index: u9, p3_index: u9, p2_index: u9) -> Self { + use bit_field::BitField; + + let mut addr = 0; + addr.set_bits(39..48, u64::from(p4_index)); + addr.set_bits(30..39, u64::from(p3_index)); + addr.set_bits(21..30, u64::from(p2_index)); + Page::containing_address(VirtAddr::new(addr)) + } +} + +impl Page { + /// Returns the 4KiB memory page with the specified page table indices. + pub fn from_page_table_indices(p4_index: u9, p3_index: u9, p2_index: u9, p1_index: u9) -> Self { + use bit_field::BitField; + + let mut addr = 0; + addr.set_bits(39..48, u64::from(p4_index)); + addr.set_bits(30..39, u64::from(p3_index)); + addr.set_bits(21..30, u64::from(p2_index)); + addr.set_bits(12..21, u64::from(p1_index)); + Page::containing_address(VirtAddr::new(addr)) + } + + /// Returns the level 1 page table index of this page. + pub fn p1_index(&self) -> u9 { + self.start_address().p1_index() + } +} + +impl fmt::Debug for Page { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_fmt(format_args!( + "Page[{}]({:#x})", + S::SIZE_AS_DEBUG_STR, + self.start_address().as_u64() + )) + } +} + +impl Add for Page { + type Output = Self; + fn add(self, rhs: u64) -> Self::Output { + Page::containing_address(self.start_address() + rhs * u64::from(S::SIZE)) + } +} + +impl AddAssign for Page { + fn add_assign(&mut self, rhs: u64) { + *self = self.clone() + rhs; + } +} + +impl Sub for Page { + type Output = Self; + fn sub(self, rhs: u64) -> Self::Output { + Page::containing_address(self.start_address() - rhs * u64::from(S::SIZE)) + } +} + +impl SubAssign for Page { + fn sub_assign(&mut self, rhs: u64) { + *self = self.clone() - rhs; + } +} + +impl Sub for Page { + type Output = u64; + fn sub(self, rhs: Self) -> Self::Output { + (self.start_address - rhs.start_address) / S::SIZE + } +} + +/// A range of pages with exclusive upper bound. +#[derive(Clone, Copy, PartialEq, Eq)] +#[repr(C)] +pub struct PageRange { + /// The start of the range, inclusive. + pub start: Page, + /// The end of the range, exclusive. + pub end: Page, +} + +impl PageRange { + /// Returns wether this range contains no pages. + pub fn is_empty(&self) -> bool { + !(self.start < self.end) + } +} + +impl Iterator for PageRange { + type Item = Page; + + fn next(&mut self) -> Option { + if self.start < self.end { + let page = self.start.clone(); + self.start += 1; + Some(page) + } else { + None + } + } +} + +impl PageRange { + /// Converts the range of 2MiB pages to a range of 4KiB pages. + pub fn as_4kib_page_range(self) -> PageRange { + PageRange { + start: Page::containing_address(self.start.start_address()), + end: Page::containing_address(self.end.start_address()), + } + } +} + +impl fmt::Debug for PageRange { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_struct("PageRange") + .field("start", &self.start) + .field("end", &self.end) + .finish() + } +} + +/// A range of pages with inclusive upper bound. +#[derive(Clone, Copy, PartialEq, Eq)] +#[repr(C)] +pub struct PageRangeInclusive { + /// The start of the range, inclusive. + pub start: Page, + /// The end of the range, inclusive. + pub end: Page, +} + +impl PageRangeInclusive { + /// Returns wether this range contains no pages. + pub fn is_empty(&self) -> bool { + !(self.start <= self.end) + } +} + +impl Iterator for PageRangeInclusive { + type Item = Page; + + fn next(&mut self) -> Option { + if self.start <= self.end { + let page = self.start.clone(); + self.start += 1; + Some(page) + } else { + None + } + } +} + +impl fmt::Debug for PageRangeInclusive { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_struct("PageRangeInclusive") + .field("start", &self.start) + .field("end", &self.end) + .finish() + } +} + +/// A physical memory frame. +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +#[repr(C)] +pub struct PhysFrame { + start_address: PhysAddr, + size: PhantomData, +} + +impl PhysFrame { + /// Returns the frame that starts at the given virtual address. + /// + /// Returns an error if the address is not correctly aligned (i.e. is not a valid frame start). + pub fn from_start_address(address: PhysAddr) -> Result { + if !address.is_aligned(S::SIZE) { + return Err(()); + } + Ok(PhysFrame::containing_address(address)) + } + + /// Returns the frame that contains the given physical address. + pub fn containing_address(address: PhysAddr) -> Self { + PhysFrame { + start_address: address.align_down(S::SIZE), + size: PhantomData, + } + } + + /// Returns the start address of the frame. + pub fn start_address(&self) -> PhysAddr { + self.start_address + } + + /// Returns the size the frame (4KB, 2MB or 1GB). + pub fn size(&self) -> u64 { + S::SIZE + } + + /// Returns a range of frames, exclusive `end`. + pub fn range(start: PhysFrame, end: PhysFrame) -> PhysFrameRange { + PhysFrameRange { start, end } + } + + /// Returns a range of frames, inclusive `end`. + pub fn range_inclusive(start: PhysFrame, end: PhysFrame) -> PhysFrameRangeInclusive { + PhysFrameRangeInclusive { start, end } + } +} + +impl fmt::Debug for PhysFrame { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_fmt(format_args!( + "PhysFrame[{}]({:#x})", + S::SIZE_AS_DEBUG_STR, + self.start_address().as_u64() + )) + } +} + +impl Add for PhysFrame { + type Output = Self; + fn add(self, rhs: u64) -> Self::Output { + PhysFrame::containing_address(self.start_address() + rhs * u64::from(S::SIZE)) + } +} + +impl AddAssign for PhysFrame { + fn add_assign(&mut self, rhs: u64) { + *self = self.clone() + rhs; + } +} + +impl Sub for PhysFrame { + type Output = Self; + fn sub(self, rhs: u64) -> Self::Output { + PhysFrame::containing_address(self.start_address() - rhs * u64::from(S::SIZE)) + } +} + +impl SubAssign for PhysFrame { + fn sub_assign(&mut self, rhs: u64) { + *self = self.clone() - rhs; + } +} + +impl Sub> for PhysFrame { + type Output = u64; + fn sub(self, rhs: PhysFrame) -> Self::Output { + (self.start_address - rhs.start_address) / S::SIZE + } +} + +/// An range of physical memory frames, exclusive the upper bound. +#[derive(Clone, Copy, PartialEq, Eq)] +#[repr(C)] +pub struct PhysFrameRange { + /// The start of the range, inclusive. + pub start: PhysFrame, + /// The end of the range, exclusive. + pub end: PhysFrame, +} + +impl PhysFrameRange { + /// Returns whether the range contains no frames. + pub fn is_empty(&self) -> bool { + !(self.start < self.end) + } +} + +impl Iterator for PhysFrameRange { + type Item = PhysFrame; + + fn next(&mut self) -> Option { + if self.start < self.end { + let frame = self.start.clone(); + self.start += 1; + Some(frame) + } else { + None + } + } +} + +impl From for PhysFrameRange { + fn from(range: os_bootinfo::FrameRange) -> Self { + PhysFrameRange { + start: PhysFrame::from_start_address(PhysAddr::new(range.start_addr())).unwrap(), + end: PhysFrame::from_start_address(PhysAddr::new(range.end_addr())).unwrap(), + } + } +} + +impl Into for PhysFrameRange { + fn into(self) -> os_bootinfo::FrameRange { + os_bootinfo::FrameRange::new( + self.start.start_address().as_u64(), + self.end.start_address().as_u64(), + ) + } +} + +impl fmt::Debug for PhysFrameRange { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_struct("PhysFrameRange") + .field("start", &self.start) + .field("end", &self.end) + .finish() + } +} + +/// An range of physical memory frames, inclusive the upper bound. +#[derive(Clone, Copy, PartialEq, Eq)] +#[repr(C)] +pub struct PhysFrameRangeInclusive { + /// The start of the range, inclusive. + pub start: PhysFrame, + /// The start of the range, exclusive. + pub end: PhysFrame, +} + +impl PhysFrameRangeInclusive { + /// Returns whether the range contains no frames. + pub fn is_empty(&self) -> bool { + !(self.start <= self.end) + } +} + +impl Iterator for PhysFrameRangeInclusive { + type Item = PhysFrame; + + fn next(&mut self) -> Option { + if self.start <= self.end { + let frame = self.start.clone(); + self.start += 1; + Some(frame) + } else { + None + } + } +} + +impl fmt::Debug for PhysFrameRangeInclusive { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_struct("PhysFrameRangeInclusive") + .field("start", &self.start) + .field("end", &self.end) + .finish() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + pub fn test_page_ranges() { + let page_size = Size4KiB::SIZE; + let number = 1000; + + let start_addr = VirtAddr::new(0xdeadbeaf); + let start: Page = Page::containing_address(start_addr); + let end = start.clone() + number; + + let mut range = Page::range(start.clone(), end.clone()); + for i in 0..number { + assert_eq!( + range.next(), + Some(Page::containing_address(start_addr + page_size * i)) + ); + } + assert_eq!(range.next(), None); + + let mut range_inclusive = Page::range_inclusive(start, end); + for i in 0..=number { + assert_eq!( + range_inclusive.next(), + Some(Page::containing_address(start_addr + page_size * i)) + ); + } + assert_eq!(range_inclusive.next(), None); + } +} diff --git a/crate/aarch64/src/paging/page_table.rs b/crate/aarch64/src/paging/page_table.rs new file mode 100644 index 0000000..b074768 --- /dev/null +++ b/crate/aarch64/src/paging/page_table.rs @@ -0,0 +1,185 @@ +use core::fmt; +use core::ops::{Index, IndexMut}; + +use super::{PageSize, PhysFrame, Size4KiB}; +use addr::PhysAddr; + +use usize_conversions::usize_from; +use ux::*; + +/// The error returned by the `PageTableEntry::frame` method. +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum FrameError { + /// The entry does not have the `PRESENT` flag set, so it isn't currently mapped to a frame. + FrameNotPresent, + /// The entry does have the `HUGE_PAGE` flag set. The `frame` method has a standard 4KiB frame + /// as return type, so a huge frame can't be returned. + HugeFrame, +} + +/// A 64-bit page table entry. +#[derive(Clone)] +#[repr(transparent)] +pub struct PageTableEntry { + entry: u64, +} + +impl PageTableEntry { + /// Returns whether this entry is zero. + pub fn is_unused(&self) -> bool { + self.entry == 0 + } + + /// Sets this entry to zero. + pub fn set_unused(&mut self) { + self.entry = 0; + } + + /// Returns the flags of this entry. + pub fn flags(&self) -> PageTableFlags { + PageTableFlags::from_bits_truncate(self.entry) + } + + /// Returns the physical address mapped by this entry, might be zero. + pub fn addr(&self) -> PhysAddr { + PhysAddr::new(self.entry & 0x000fffff_fffff000) + } + + /// Returns the physical frame mapped by this entry. + /// + /// Returns the following errors: + /// + /// - `FrameError::FrameNotPresent` if the entry doesn't have the `PRESENT` flag set. + /// - `FrameError::HugeFrame` if the entry has the `HUGE_PAGE` flag set (for huge pages the + /// `addr` function must be used) + pub fn frame(&self) -> Result { + if !self.flags().contains(PageTableFlags::PRESENT) { + Err(FrameError::FrameNotPresent) + } else if self.flags().contains(PageTableFlags::HUGE_PAGE) { + Err(FrameError::HugeFrame) + } else { + Ok(PhysFrame::containing_address(self.addr())) + } + } + + /// Map the entry to the specified physical address with the specified flags. + pub fn set_addr(&mut self, addr: PhysAddr, flags: PageTableFlags) { + assert!(addr.is_aligned(Size4KiB::SIZE)); + self.entry = (addr.as_u64()) | flags.bits(); + } + + /// Map the entry to the specified physical frame with the specified flags. + pub fn set_frame(&mut self, frame: PhysFrame, flags: PageTableFlags) { + assert!(!flags.contains(PageTableFlags::HUGE_PAGE)); + self.set_addr(frame.start_address(), flags) + } + + /// Sets the flags of this entry. + pub fn set_flags(&mut self, flags: PageTableFlags) { + self.entry = self.addr().as_u64() | flags.bits(); + } +} + +impl fmt::Debug for PageTableEntry { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let mut f = f.debug_struct("PageTableEntry"); + f.field("addr", &self.addr()); + f.field("flags", &self.flags()); + f.finish() + } +} + +bitflags! { + /// Possible flags for a page table entry. + pub struct PageTableFlags: u64 { + const ALL = 0xffffffff_ffffffff; + const TYPE_MASK = 3 << 0; + const TYPE_FAULT = 0 << 0; + const TYPE_PAGE = 3 << 0; + const TABLE_BIT = 1 << 1; + + const PRESENT = 1 << 0; + const USER_ACCESSIBLE = 1 << 6; /* AP[1] */ + const RDONLY = 1 << 7; /* AP[2] */ + const SHARED = 3 << 8; /* SH[1:0], inner shareable */ + const BIT_8 = 1 << 8; + const BIT_9 = 1 << 9; + /* + pub const ATTRIB_SH_NON_SHAREABLE: usize = 0x0 << 8; + pub const ATTRIB_SH_OUTER_SHAREABLE: usize = 0x2 << 8; + pub const ATTRIB_SH_INNER_SHAREABLE: usize = 0x3 << 8; + */ + + const ACCESSED = 1 << 10; /* AF, Access Flag */ + const NONE_GLOBAL = 1 << 11; /* None Global */ + const GLOBAL = (!(1 << 11)); + const DBM = 1 << 51; /* Dirty Bit Management */ + const WRITE = 1 << 51; /* DBM */ + const CONT = 1 << 52; /* Contiguous range */ + const PXN = 1 << 53; /* Privileged XN */ + const UXN = 1 << 54; /* User XN */ + const HYP_XN = 1 << 54; /* HYP XN */ + const DIRTY = 1 << 55; + const SWAPPED = 1 << 56; + const HUGE_PAGE = 1 << 57; + const PROT_NONE = 1 << 58; + + } +} + +/// The number of entries in a page table. +const ENTRY_COUNT: usize = 512; + +/// Represents a page table. +/// +/// Always page-sized. +/// +/// This struct implements the `Index` and `IndexMut` traits, so the entries can be accessed +/// through index operations. For example, `page_table[15]` returns the 15th page table entry. +#[repr(transparent)] +pub struct PageTable { + entries: [PageTableEntry; ENTRY_COUNT], +} + +impl PageTable { + /// Clears all entries. + pub fn zero(&mut self) { + for entry in self.entries.iter_mut() { + entry.set_unused(); + } + } +} + +impl Index for PageTable { + type Output = PageTableEntry; + + fn index(&self, index: usize) -> &Self::Output { + &self.entries[index] + } +} + +impl IndexMut for PageTable { + fn index_mut(&mut self, index: usize) -> &mut Self::Output { + &mut self.entries[index] + } +} + +impl Index for PageTable { + type Output = PageTableEntry; + + fn index(&self, index: u9) -> &Self::Output { + &self.entries[usize_from(u16::from(index))] + } +} + +impl IndexMut for PageTable { + fn index_mut(&mut self, index: u9) -> &mut Self::Output { + &mut self.entries[usize_from(u16::from(index))] + } +} + +impl fmt::Debug for PageTable { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + self.entries[..].fmt(f) + } +} diff --git a/crate/aarch64/src/paging/recursive.rs b/crate/aarch64/src/paging/recursive.rs new file mode 100644 index 0000000..0cbc6d0 --- /dev/null +++ b/crate/aarch64/src/paging/recursive.rs @@ -0,0 +1,623 @@ +#![cfg(target_arch = "aarch64")] + +use asm::tlb_invalidate; +use paging::{ + frame_alloc::FrameAllocator, + page_table::{FrameError, PageTable, PageTableEntry, PageTableFlags}, + NotGiantPageSize, Page, PageSize, PhysFrame, Size1GiB, Size2MiB, Size4KiB, +}; +use paging::page_table::PageTableFlags as Flags; +use asm::ttbr0_el1_read; +use ux::u9; +use addr::{PhysAddr, VirtAddr}; + +/// This type represents a page whose mapping has changed in the page table. +/// +/// The old mapping might be still cached in the translation lookaside buffer (TLB), so it needs +/// to be flushed from the TLB before it's accessed. This type is returned from function that +/// change the mapping of a page to ensure that the TLB flush is not forgotten. +#[derive(Debug)] +#[must_use = "Page Table changes must be flushed or ignored."] +pub struct MapperFlush(Page); + +impl MapperFlush { + /// Create a new flush promise + fn new(page: Page) -> Self { + MapperFlush(page) + } + + /// Flush the page from the TLB to ensure that the newest mapping is used. + pub fn flush(self) { + tlb_invalidate(); + } + + /// Don't flush the TLB and silence the “must be used” warning. + pub fn ignore(self) {} +} + +/// A trait for common page table operations. +pub trait Mapper { + /// Creates a new mapping in the page table. + /// + /// This function might need additional physical frames to create new page tables. These + /// frames are allocated from the `allocator` argument. At most three frames are required. + fn map_to( + &mut self, + page: Page, + frame: PhysFrame, + flags: PageTableFlags, + allocator: &mut A, + ) -> Result, MapToError> + where + A: FrameAllocator; + + /// Removes a mapping from the page table and returns the frame that used to be mapped. + /// + /// Note that no page tables or pages are deallocated. + fn unmap(&mut self, page: Page) -> Result<(PhysFrame, MapperFlush), UnmapError>; + + /// Updates the flags of an existing mapping. + fn update_flags( + &mut self, + page: Page, + flags: PageTableFlags, + ) -> Result, FlagUpdateError>; + + /// Return the frame that the specified page is mapped to. + fn translate_page(&self, page: Page) -> Option>; + + /// Maps the given frame to the virtual page with the same address. + fn identity_map( + &mut self, + frame: PhysFrame, + flags: PageTableFlags, + allocator: &mut A, + ) -> Result, MapToError> + where + A: FrameAllocator, + S: PageSize, + Self: Mapper, + { + let page = Page::containing_address(VirtAddr::new(frame.start_address().as_u64())); + self.map_to(page, frame, flags, allocator) + } +} + +/// A recursive page table is a last level page table with an entry mapped to the table itself. +/// +/// This recursive mapping allows accessing all page tables in the hierarchy: +/// +/// - To access the level 4 page table, we “loop“ (i.e. follow the recursively mapped entry) four +/// times. +/// - To access a level 3 page table, we “loop” three times and then use the level 4 index. +/// - To access a level 2 page table, we “loop” two times, then use the level 4 index, then the +/// level 3 index. +/// - To access a level 1 page table, we “loop” once, then use the level 4 index, then the +/// level 3 index, then the level 2 index. +/// +/// This struct implements the `Mapper` trait. +#[derive(Debug)] +pub struct RecursivePageTable<'a> { + p4: &'a mut PageTable, + recursive_index: u9, +} + +/// An error indicating that the given page table is not recursively mapped. +/// +/// Returned from `RecursivePageTable::new`. +#[derive(Debug)] +pub struct NotRecursivelyMapped; + +/// This error is returned from `map_to` and similar methods. +#[derive(Debug)] +pub enum MapToError { + /// An additional frame was needed for the mapping process, but the frame allocator + /// returned `None`. + FrameAllocationFailed, + /// An upper level page table entry has the `HUGE_PAGE` flag set, which means that the + /// given page is part of an already mapped huge page. + ParentEntryHugePage, + /// The given page is already mapped to a physical frame. + PageAlreadyMapped, +} + +/// An error indicating that an `unmap` call failed. +#[derive(Debug)] +pub enum UnmapError { + /// An upper level page table entry has the `HUGE_PAGE` flag set, which means that the + /// given page is part of a huge page and can't be freed individually. + ParentEntryHugePage, + /// The given page is not mapped to a physical frame. + PageNotMapped, + /// The page table entry for the given page points to an invalid physical address. + InvalidFrameAddress(PhysAddr), +} + +/// An error indicating that an `update_flags` call failed. +#[derive(Debug)] +pub enum FlagUpdateError { + /// The given page is not mapped to a physical frame. + PageNotMapped, +} + +impl<'a> RecursivePageTable<'a> { + /// Creates a new RecursivePageTable from the passed level 4 PageTable. + /// + /// The page table must be recursively mapped, that means: + /// + /// - The page table must have one recursive entry, i.e. an entry that points to the table + /// itself. + /// - The reference must use that “loop”, i.e. be of the form `0o_xxx_xxx_xxx_xxx_0000` + /// where `xxx` is the recursive entry. + /// - The page table must be active, i.e. the CR3 register must contain its physical address. + /// + /// Otherwise `Err(NotRecursivelyMapped)` is returned. + pub fn new(table: &'a mut PageTable) -> Result { + let page = Page::containing_address(VirtAddr::new(table as *const _ as u64)); + let recursive_index = page.p4_index(); + + if page.p3_index() != recursive_index + || page.p2_index() != recursive_index + || page.p1_index() != recursive_index + { + return Err(NotRecursivelyMapped); + } + if Ok(ttbr0_el1_read().0) != table[recursive_index].frame() { + return Err(NotRecursivelyMapped); + } + + Ok(RecursivePageTable { + p4: table, + recursive_index, + }) + } + + /// Creates a new RecursivePageTable without performing any checks. + /// + /// The `recursive_index` parameter must be the index of the recursively mapped entry. + pub unsafe fn new_unchecked(table: &'a mut PageTable, recursive_index: u9) -> Self { + RecursivePageTable { + p4: table, + recursive_index, + } + } + + /// Internal helper function to create the page table of the next level if needed. + /// + /// If the passed entry is unused, a new frame is allocated from the given allocator, zeroed, + /// and the entry is updated to that address. If the passed entry is already mapped, the next + /// table is returned directly. + /// + /// The `next_page_table` page must be the page of the next page table in the hierarchy. + /// + /// Returns `MapToError::FrameAllocationFailed` if the entry is unused and the allocator + /// returned `None`. Returns `MapToError::ParentEntryHugePage` if the `HUGE_PAGE` flag is set + /// in the passed entry. + unsafe fn create_next_table<'b, A>( + entry: &'b mut PageTableEntry, + next_table_page: Page, + allocator: &mut A, + ) -> Result<&'b mut PageTable, MapToError> + where + A: FrameAllocator, + { + /// This inner function is used to limit the scope of `unsafe`. + /// + /// This is a safe function, so we need to use `unsafe` blocks when we do something unsafe. + fn inner<'b, A>( + entry: &'b mut PageTableEntry, + next_table_page: Page, + allocator: &mut A, + ) -> Result<&'b mut PageTable, MapToError> + where + A: FrameAllocator, + { + + let created; + + if entry.is_unused() { + if let Some(frame) = allocator.alloc() { + entry.set_frame(frame, Flags::PRESENT | Flags::WRITE); + created = true; + } else { + return Err(MapToError::FrameAllocationFailed); + } + } else { + created = false; + } + if entry.flags().contains(Flags::HUGE_PAGE) { + return Err(MapToError::ParentEntryHugePage); + } + + let page_table_ptr = next_table_page.start_address().as_mut_ptr(); + let page_table: &mut PageTable = unsafe { &mut *(page_table_ptr) }; + if created { + page_table.zero(); + } + Ok(page_table) + } + + inner(entry, next_table_page, allocator) + } +} + +impl<'a> Mapper for RecursivePageTable<'a> { + fn map_to( + &mut self, + page: Page, + frame: PhysFrame, + flags: PageTableFlags, + allocator: &mut A, + ) -> Result, MapToError> + where + A: FrameAllocator, + { + let p4 = &mut self.p4; + + let p3_page = p3_page(page, self.recursive_index); + let p3 = unsafe { Self::create_next_table(&mut p4[page.p4_index()], p3_page, allocator)? }; + + if !p3[page.p3_index()].is_unused() { + return Err(MapToError::PageAlreadyMapped); + } + p3[page.p3_index()].set_addr(frame.start_address(), flags | Flags::HUGE_PAGE); + + Ok(MapperFlush::new(page)) + } + + fn unmap( + &mut self, + page: Page, + ) -> Result<(PhysFrame, MapperFlush), UnmapError> { + let p4 = &mut self.p4; + let p4_entry = &p4[page.p4_index()]; + + p4_entry.frame().map_err(|err| match err { + FrameError::FrameNotPresent => UnmapError::PageNotMapped, + FrameError::HugeFrame => UnmapError::ParentEntryHugePage, + })?; + + let p3 = unsafe { &mut *(p3_ptr(page, self.recursive_index)) }; + let p3_entry = &mut p3[page.p3_index()]; + let flags = p3_entry.flags(); + + if !flags.contains(PageTableFlags::PRESENT) { + return Err(UnmapError::PageNotMapped); + } + if !flags.contains(PageTableFlags::HUGE_PAGE) { + return Err(UnmapError::ParentEntryHugePage); + } + + let frame = PhysFrame::from_start_address(p3_entry.addr()) + .map_err(|()| UnmapError::InvalidFrameAddress(p3_entry.addr()))?; + + p3_entry.set_unused(); + Ok((frame, MapperFlush::new(page))) + } + + fn update_flags( + &mut self, + page: Page, + flags: PageTableFlags, + ) -> Result, FlagUpdateError> { + let p4 = &mut self.p4; + + if p4[page.p4_index()].is_unused() { + return Err(FlagUpdateError::PageNotMapped); + } + + let p3 = unsafe { &mut *(p3_ptr(page, self.recursive_index)) }; + + if p3[page.p3_index()].is_unused() { + return Err(FlagUpdateError::PageNotMapped); + } + p3[page.p3_index()].set_flags(flags | Flags::HUGE_PAGE); + + Ok(MapperFlush::new(page)) + } + + fn translate_page(&self, page: Page) -> Option> { + let p4 = &self.p4; + + if p4[page.p4_index()].is_unused() { + return None; + } + + let p3 = unsafe { &*(p3_ptr(page, self.recursive_index)) }; + let p3_entry = &p3[page.p3_index()]; + + if p3_entry.is_unused() { + return None; + } + + PhysFrame::from_start_address(p3_entry.addr()).ok() + } +} + +impl<'a> Mapper for RecursivePageTable<'a> { + fn map_to( + &mut self, + page: Page, + frame: PhysFrame, + flags: PageTableFlags, + allocator: &mut A, + ) -> Result, MapToError> + where + A: FrameAllocator, + { + let p4 = &mut self.p4; + + let p3_page = p3_page(page, self.recursive_index); + let p3 = unsafe { Self::create_next_table(&mut p4[page.p4_index()], p3_page, allocator)? }; + + let p2_page = p2_page(page, self.recursive_index); + let p2 = unsafe { Self::create_next_table(&mut p3[page.p3_index()], p2_page, allocator)? }; + + if !p2[page.p2_index()].is_unused() { + return Err(MapToError::PageAlreadyMapped); + } + p2[page.p2_index()].set_addr(frame.start_address(), flags | Flags::HUGE_PAGE); + + Ok(MapperFlush::new(page)) + } + + fn unmap( + &mut self, + page: Page, + ) -> Result<(PhysFrame, MapperFlush), UnmapError> { + let p4 = &mut self.p4; + let p4_entry = &p4[page.p4_index()]; + p4_entry.frame().map_err(|err| match err { + FrameError::FrameNotPresent => UnmapError::PageNotMapped, + FrameError::HugeFrame => UnmapError::ParentEntryHugePage, + })?; + + let p3 = unsafe { &mut *(p3_ptr(page, self.recursive_index)) }; + let p3_entry = &p3[page.p3_index()]; + p3_entry.frame().map_err(|err| match err { + FrameError::FrameNotPresent => UnmapError::PageNotMapped, + FrameError::HugeFrame => UnmapError::ParentEntryHugePage, + })?; + + let p2 = unsafe { &mut *(p2_ptr(page, self.recursive_index)) }; + let p2_entry = &mut p2[page.p2_index()]; + let flags = p2_entry.flags(); + + if !flags.contains(PageTableFlags::PRESENT) { + return Err(UnmapError::PageNotMapped); + } + if !flags.contains(PageTableFlags::HUGE_PAGE) { + return Err(UnmapError::ParentEntryHugePage); + } + + let frame = PhysFrame::from_start_address(p2_entry.addr()) + .map_err(|()| UnmapError::InvalidFrameAddress(p2_entry.addr()))?; + + p2_entry.set_unused(); + Ok((frame, MapperFlush::new(page))) + } + + fn update_flags( + &mut self, + page: Page, + flags: PageTableFlags, + ) -> Result, FlagUpdateError> { + let p4 = &mut self.p4; + + if p4[page.p4_index()].is_unused() { + return Err(FlagUpdateError::PageNotMapped); + } + + let p3 = unsafe { &mut *(p3_ptr(page, self.recursive_index)) }; + + if p3[page.p3_index()].is_unused() { + return Err(FlagUpdateError::PageNotMapped); + } + + let p2 = unsafe { &mut *(p2_ptr(page, self.recursive_index)) }; + + if p2[page.p2_index()].is_unused() { + return Err(FlagUpdateError::PageNotMapped); + } + + p2[page.p2_index()].set_flags(flags | Flags::HUGE_PAGE); + + Ok(MapperFlush::new(page)) + } + + fn translate_page(&self, page: Page) -> Option> { + let p4 = &self.p4; + + if p4[page.p4_index()].is_unused() { + return None; + } + + let p3 = unsafe { &*(p3_ptr(page, self.recursive_index)) }; + let p3_entry = &p3[page.p3_index()]; + + if p3_entry.is_unused() { + return None; + } + + let p2 = unsafe { &*(p2_ptr(page, self.recursive_index)) }; + let p2_entry = &p2[page.p2_index()]; + + if p2_entry.is_unused() { + return None; + } + + PhysFrame::from_start_address(p2_entry.addr()).ok() + } +} + +impl<'a> Mapper for RecursivePageTable<'a> { + fn map_to( + &mut self, + page: Page, + frame: PhysFrame, + flags: PageTableFlags, + allocator: &mut A, + ) -> Result, MapToError> + where + A: FrameAllocator, + { + let p4 = &mut self.p4; + + let p3_page = p3_page(page, self.recursive_index); + let p3 = unsafe { Self::create_next_table(&mut p4[page.p4_index()], p3_page, allocator)? }; + + let p2_page = p2_page(page, self.recursive_index); + let p2 = unsafe { Self::create_next_table(&mut p3[page.p3_index()], p2_page, allocator)? }; + + let p1_page = p1_page(page, self.recursive_index); + let p1 = unsafe { Self::create_next_table(&mut p2[page.p2_index()], p1_page, allocator)? }; + + if !p1[page.p1_index()].is_unused() { + return Err(MapToError::PageAlreadyMapped); + } + p1[page.p1_index()].set_frame(frame, flags); + + Ok(MapperFlush::new(page)) + } + + fn unmap( + &mut self, + page: Page, + ) -> Result<(PhysFrame, MapperFlush), UnmapError> { + let p4 = &mut self.p4; + let p4_entry = &p4[page.p4_index()]; + p4_entry.frame().map_err(|err| match err { + FrameError::FrameNotPresent => UnmapError::PageNotMapped, + FrameError::HugeFrame => UnmapError::ParentEntryHugePage, + })?; + + let p3 = unsafe { &mut *(p3_ptr(page, self.recursive_index)) }; + let p3_entry = &p3[page.p3_index()]; + p3_entry.frame().map_err(|err| match err { + FrameError::FrameNotPresent => UnmapError::PageNotMapped, + FrameError::HugeFrame => UnmapError::ParentEntryHugePage, + })?; + + let p2 = unsafe { &mut *(p2_ptr(page, self.recursive_index)) }; + let p2_entry = &p2[page.p2_index()]; + p2_entry.frame().map_err(|err| match err { + FrameError::FrameNotPresent => UnmapError::PageNotMapped, + FrameError::HugeFrame => UnmapError::ParentEntryHugePage, + })?; + + let p1 = unsafe { &mut *(p1_ptr(page, self.recursive_index)) }; + let p1_entry = &mut p1[page.p1_index()]; + + let frame = p1_entry.frame().map_err(|err| match err { + FrameError::FrameNotPresent => UnmapError::PageNotMapped, + FrameError::HugeFrame => UnmapError::ParentEntryHugePage, + })?; + + p1_entry.set_unused(); + Ok((frame, MapperFlush::new(page))) + } + + fn update_flags( + &mut self, + page: Page, + flags: PageTableFlags, + ) -> Result, FlagUpdateError> { + let p4 = &mut self.p4; + + if p4[page.p4_index()].is_unused() { + return Err(FlagUpdateError::PageNotMapped); + } + + let p3 = unsafe { &mut *(p3_ptr(page, self.recursive_index)) }; + + if p3[page.p3_index()].is_unused() { + return Err(FlagUpdateError::PageNotMapped); + } + + let p2 = unsafe { &mut *(p2_ptr(page, self.recursive_index)) }; + + if p2[page.p2_index()].is_unused() { + return Err(FlagUpdateError::PageNotMapped); + } + + let p1 = unsafe { &mut *(p1_ptr(page, self.recursive_index)) }; + + if p1[page.p1_index()].is_unused() { + return Err(FlagUpdateError::PageNotMapped); + } + + p1[page.p1_index()].set_flags(flags); + + Ok(MapperFlush::new(page)) + } + + fn translate_page(&self, page: Page) -> Option> { + let p4 = &self.p4; + + if p4[page.p4_index()].is_unused() { + return None; + } + + let p3 = unsafe { &*(p3_ptr(page, self.recursive_index)) }; + let p3_entry = &p3[page.p3_index()]; + + if p3_entry.is_unused() { + return None; + } + + let p2 = unsafe { &*(p2_ptr(page, self.recursive_index)) }; + let p2_entry = &p2[page.p2_index()]; + + if p2_entry.is_unused() { + return None; + } + + let p1 = unsafe { &*(p1_ptr(page, self.recursive_index)) }; + let p1_entry = &p1[page.p1_index()]; + + if p1_entry.is_unused() { + return None; + } + + PhysFrame::from_start_address(p1_entry.addr()).ok() + } +} + +fn p3_ptr(page: Page, recursive_index: u9) -> *mut PageTable { + p3_page(page, recursive_index).start_address().as_mut_ptr() +} + +fn p3_page(page: Page, recursive_index: u9) -> Page { + Page::from_page_table_indices( + recursive_index, + recursive_index, + recursive_index, + page.p4_index(), + ) +} + +fn p2_ptr(page: Page, recursive_index: u9) -> *mut PageTable { + p2_page(page, recursive_index).start_address().as_mut_ptr() +} + +fn p2_page(page: Page, recursive_index: u9) -> Page { + Page::from_page_table_indices( + recursive_index, + recursive_index, + page.p4_index(), + page.p3_index(), + ) +} + +fn p1_ptr(page: Page, recursive_index: u9) -> *mut PageTable { + p1_page(page, recursive_index).start_address().as_mut_ptr() +} + +fn p1_page(page: Page, recursive_index: u9) -> Page { + Page::from_page_table_indices( + recursive_index, + page.p4_index(), + page.p3_index(), + page.p2_index(), + ) +} diff --git a/crate/aarch64/src/regs/cntfrq_el0.rs b/crate/aarch64/src/regs/cntfrq_el0.rs new file mode 100644 index 0000000..df56ac9 --- /dev/null +++ b/crate/aarch64/src/regs/cntfrq_el0.rs @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2018 by the author(s) + * + * ============================================================================= + * + * Licensed under either of + * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + * - MIT License (http://opensource.org/licenses/MIT) + * at your option. + * + * ============================================================================= + * + * Author(s): + * - Andre Richter + */ + +//! Counter-timer Frequency register - EL0 +//! +//! This register is provided so that software can discover the frequency of the +//! system counter. It must be programmed with this value as part of system +//! initialization. The value of the register is not interpreted by hardware. + +use register::cpu::RegisterReadOnly; + +pub struct Reg; + +impl RegisterReadOnly for Reg { + sys_coproc_read_raw!(u32, "CNTFRQ_EL0"); +} + +pub static CNTFRQ_EL0: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/cnthctl_el2.rs b/crate/aarch64/src/regs/cnthctl_el2.rs new file mode 100644 index 0000000..f5e3c2c --- /dev/null +++ b/crate/aarch64/src/regs/cnthctl_el2.rs @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2018 by the author(s) + * + * ============================================================================= + * + * Licensed under either of + * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + * - MIT License (http://opensource.org/licenses/MIT) + * at your option. + * + * ============================================================================= + * + * Author(s): + * - Andre Richter + */ + +//! Counter-timer Hypervisor Control register - EL2 +//! +//! Controls the generation of an event stream from the physical counter, and +//! access from Non-secure EL1 to the physical counter and the Non-secure EL1 +//! physical timer. + +use register::cpu::RegisterReadWrite; + +// When HCR_EL2.E2H == 0: +// TODO: Figure out how we can differentiate depending on HCR_EL2.E2H state +// +// For now, implement the HCR_EL2.E2H == 0 version +register_bitfields! {u32, + CNTHCTL_EL2 [ + /// Traps Non-secure EL0 and EL1 accesses to the physical timer + /// registers to EL2. + /// + /// 0 From AArch64 state: Non-secure EL0 and EL1 accesses to the + /// CNTP_CTL_EL0, CNTP_CVAL_EL0, and CNTP_TVAL_EL0 are trapped to EL2, + /// unless it is trapped by CNTKCTL_EL1.EL0PTEN. + /// + /// From AArch32 state: Non-secure EL0 and EL1 accesses to the + /// CNTP_CTL, CNTP_CVAL, and CNTP_TVAL are trapped to EL2, unless it + /// is trapped by CNTKCTL_EL1.EL0PTEN or CNTKCTL.PL0PTEN. + /// + /// 1 This control does not cause any instructions to be trapped. + /// + /// If EL3 is implemented and EL2 is not implemented, behavior is as if + /// this bit is 1 other than for the purpose of a direct read. + EL1PCEN OFFSET(1) NUMBITS(1) [], + + /// Traps Non-secure EL0 and EL1 accesses to the physical counter + /// register to EL2. + /// + /// 0 From AArch64 state: Non-secure EL0 and EL1 accesses to the + /// CNTPCT_EL0 are trapped to EL2, unless it is trapped by + /// CNTKCTL_EL1.EL0PCTEN. + /// + /// From AArch32 state: Non-secure EL0 and EL1 accesses to the CNTPCT + /// are trapped to EL2, unless it is trapped by CNTKCTL_EL1.EL0PCTEN + /// or CNTKCTL.PL0PCTEN. + /// + /// 1 This control does not cause any instructions to be trapped. + /// + /// If EL3 is implemented and EL2 is not implemented, behavior is as if + /// this bit is 1 other than for the purpose of a direct read. + EL1PCTEN OFFSET(0) NUMBITS(1) [] + ] +} + +pub struct Reg; + +impl RegisterReadWrite for Reg { + sys_coproc_read_raw!(u32, "CNTHCTL_EL2"); + sys_coproc_write_raw!(u32, "CNTHCTL_EL2"); +} + +#[allow(non_upper_case_globals)] +pub static CNTHCTL_EL2: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/cntp_ctl_el0.rs b/crate/aarch64/src/regs/cntp_ctl_el0.rs new file mode 100644 index 0000000..76991eb --- /dev/null +++ b/crate/aarch64/src/regs/cntp_ctl_el0.rs @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2018 by the author(s) + * + * ============================================================================= + * + * Licensed under either of + * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + * - MIT License (http://opensource.org/licenses/MIT) + * at your option. + * + * ============================================================================= + * + * Author(s): + * - Andre Richter + */ + +//! Counter-timer Physical Timer Control register - EL0 +//! +//! Control register for the EL1 physical timer. + +use register::cpu::RegisterReadWrite; + +register_bitfields! {u32, + CNTP_CTL_EL0 [ + /// The status of the timer. This bit indicates whether the timer + /// condition is met: + /// + /// 0 Timer condition is not met. + /// 1 Timer condition is met. + /// + /// When the value of the ENABLE bit is 1, ISTATUS indicates whether the + /// timer condition is met. ISTATUS takes no account of the value of the + /// IMASK bit. If the value of ISTATUS is 1 and the value of IMASK is 0 + /// then the timer interrupt is asserted. + /// + /// When the value of the ENABLE bit is 0, the ISTATUS field is UNKNOWN. + /// + /// This bit is read-only. + ISTATUS OFFSET(2) NUMBITS(1) [], + + /// Timer interrupt mask bit. Permitted values are: + /// + /// 0 Timer interrupt is not masked by the IMASK bit. + /// 1 Timer interrupt is masked by the IMASK bit. + IMASK OFFSET(1) NUMBITS(1) [], + + /// Enables the timer. Permitted values are: + /// + /// 0 Timer disabled. + /// 1 Timer enabled. + ENABLE OFFSET(0) NUMBITS(1) [] + ] +} + +pub struct Reg; + +impl RegisterReadWrite for Reg { + sys_coproc_read_raw!(u32, "CNTP_CTL_EL0"); + sys_coproc_write_raw!(u32, "CNTP_CTL_EL0"); +} + +pub static CNTP_CTL_EL0: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/cntp_tval_el0.rs b/crate/aarch64/src/regs/cntp_tval_el0.rs new file mode 100644 index 0000000..bdf5f6a --- /dev/null +++ b/crate/aarch64/src/regs/cntp_tval_el0.rs @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2018 by the author(s) + * + * ============================================================================= + * + * Licensed under either of + * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + * - MIT License (http://opensource.org/licenses/MIT) + * at your option. + * + * ============================================================================= + * + * Author(s): + * - Andre Richter + */ + +//! Counter-timer Physical Timer TimerValue register - EL0 +//! +//! Holds the timer value for the EL1 physical timer. + +use register::cpu::RegisterReadWrite; + +pub struct Reg; + +impl RegisterReadWrite for Reg { + sys_coproc_read_raw!(u32, "CNTP_TVAL_EL0"); + sys_coproc_write_raw!(u32, "CNTP_TVAL_EL0"); +} + +pub static CNTP_TVAL_EL0: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/cntpct_el0.rs b/crate/aarch64/src/regs/cntpct_el0.rs new file mode 100644 index 0000000..b381d99 --- /dev/null +++ b/crate/aarch64/src/regs/cntpct_el0.rs @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2018 by the author(s) + * + * ============================================================================= + * + * Licensed under either of + * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + * - MIT License (http://opensource.org/licenses/MIT) + * at your option. + * + * ============================================================================= + * + * Author(s): + * - Andre Richter + */ + +//! Counter-timer Physical Count register - EL0 +//! +//! Holds the 64-bit physical count value. + +use register::cpu::RegisterReadOnly; + +pub struct Reg; + +impl RegisterReadOnly for Reg { + sys_coproc_read_raw!(u64, "CNTPCT_EL0"); +} + +pub static CNTPCT_EL0: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/cntvoff_el2.rs b/crate/aarch64/src/regs/cntvoff_el2.rs new file mode 100644 index 0000000..aff7074 --- /dev/null +++ b/crate/aarch64/src/regs/cntvoff_el2.rs @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2018 by the author(s) + * + * ============================================================================= + * + * Licensed under either of + * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + * - MIT License (http://opensource.org/licenses/MIT) + * at your option. + * + * ============================================================================= + * + * Author(s): + * - Andre Richter + */ + +//! Counter-timer Virtual Offset register - EL2 +//! +//! Holds the 64-bit virtual offset. This is the offset between the physical +//! count value visible in CNTPCT_EL0 and the virtual count value visible in +//! CNTVCT_EL0. + +use register::cpu::RegisterReadWrite; + +pub struct Reg; + +impl RegisterReadWrite for Reg { + sys_coproc_read_raw!(u64, "CNTVOFF_EL2"); + sys_coproc_write_raw!(u64, "CNTVOFF_EL2"); +} + +pub static CNTVOFF_EL2: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/currentel.rs b/crate/aarch64/src/regs/currentel.rs new file mode 100644 index 0000000..91b8e0a --- /dev/null +++ b/crate/aarch64/src/regs/currentel.rs @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2018 by the author(s) + * + * ============================================================================= + * + * Licensed under either of + * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + * - MIT License (http://opensource.org/licenses/MIT) + * at your option. + * + * ============================================================================= + * + * Author(s): + * - Andre Richter + */ + +//! Current Exception Level +//! +//! Holds the current Exception level. + +use register::cpu::RegisterReadOnly; + +register_bitfields! {u32, + CurrentEL [ + /// Current Exception level. Possible values of this field are: + /// + /// 00 EL0 + /// 01 EL1 + /// 10 EL2 + /// 11 EL3 + /// + /// When the HCR_EL2.NV bit is 1, Non-secure EL1 read accesses to the + /// CurrentEL register return the value of 0x2 in this field. + /// + /// This field resets to a value that is architecturally UNKNOWN. + EL OFFSET(2) NUMBITS(2) [ + EL0 = 0, + EL1 = 1, + EL2 = 2, + EL3 = 3 + ] + ] +} + +pub struct Reg; + +impl RegisterReadOnly for Reg { + sys_coproc_read_raw!(u32, "CurrentEL"); +} + +#[allow(non_upper_case_globals)] +pub static CurrentEL: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/daif.rs b/crate/aarch64/src/regs/daif.rs new file mode 100644 index 0000000..bf810a2 --- /dev/null +++ b/crate/aarch64/src/regs/daif.rs @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2018 by the author(s) + * + * ============================================================================= + * + * Licensed under either of + * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + * - MIT License (http://opensource.org/licenses/MIT) + * at your option. + * + * ============================================================================= + * + * Author(s): + * - Andre Richter + */ + +//! Interrupt Mask Bits +//! +//! Allows access to the interrupt mask bits. + +use register::cpu::RegisterReadWrite; + +register_bitfields! {u32, + DAIF [ + /// Process state D mask. The possible values of this bit are: + /// + /// 0 Watchpoint, Breakpoint, and Software Step exceptions targeted at + /// the current Exception level are not masked. + /// + /// 1 Watchpoint, Breakpoint, and Software Step exceptions targeted at + /// the current Exception level are masked. + /// + /// When the target Exception level of the debug exception is higher + /// than the current Exception level, the exception is not masked by + /// this bit. + /// + /// When this register has an architecturally-defined reset value, this + /// field resets to 1. + D OFFSET(9) NUMBITS(1) [ + Unmasked = 0, + Masked = 1 + ], + + /// SError interrupt mask bit. The possible values of this bit are: + /// + /// 0 Exception not masked. + /// 1 Exception masked. + /// + /// When this register has an architecturally-defined reset value, this + /// field resets to 1 . + A OFFSET(8) NUMBITS(1) [ + Unmasked = 0, + Masked = 1 + ], + + /// IRQ mask bit. The possible values of this bit are: + /// + /// 0 Exception not masked. + /// 1 Exception masked. + /// + /// When this register has an architecturally-defined reset value, this + /// field resets to 1 . + I OFFSET(7) NUMBITS(1) [ + Unmasked = 0, + Masked = 1 + ], + + /// FIQ mask bit. The possible values of this bit are: + /// + /// 0 Exception not masked. + /// 1 Exception masked. + /// + /// When this register has an architecturally-defined reset value, this + /// field resets to 1 . + F OFFSET(6) NUMBITS(1) [ + Unmasked = 0, + Masked = 1 + ] + ] +} + + +pub struct Reg; + +impl RegisterReadWrite for Reg { + sys_coproc_read_raw!(u32, "DAIF"); + sys_coproc_write_raw!(u32, "DAIF"); +} + +pub static DAIF: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/elr_el2.rs b/crate/aarch64/src/regs/elr_el2.rs new file mode 100644 index 0000000..0786fbb --- /dev/null +++ b/crate/aarch64/src/regs/elr_el2.rs @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2018 by the author(s) + * + * ============================================================================= + * + * Licensed under either of + * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + * - MIT License (http://opensource.org/licenses/MIT) + * at your option. + * + * ============================================================================= + * + * Author(s): + * - Andre Richter + */ + +//! Exception Link Register - EL2 +//! +//! When taking an exception to EL2, holds the address to return to. + +use register::cpu::RegisterReadWrite; + +pub struct Reg; + +impl RegisterReadWrite for Reg { + sys_coproc_read_raw!(u64, "ELR_EL2"); + sys_coproc_write_raw!(u64, "ELR_EL2"); +} + +pub static ELR_EL2: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/hcr_el2.rs b/crate/aarch64/src/regs/hcr_el2.rs new file mode 100644 index 0000000..683bbef --- /dev/null +++ b/crate/aarch64/src/regs/hcr_el2.rs @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2018 by the author(s) + * + * ============================================================================= + * + * Licensed under either of + * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + * - MIT License (http://opensource.org/licenses/MIT) + * at your option. + * + * ============================================================================= + * + * Author(s): + * - Andre Richter + */ + +//! Hypervisor Configuration Register - EL2 +//! +//! Provides configuration controls for virtualization, including defining +//! whether various Non-secure operations are trapped to EL2. + +use register::cpu::RegisterReadWrite; + +register_bitfields! {u64, + HCR_EL2 [ + /// Execution state control for lower Exception levels: + /// + /// 0 Lower levels are all AArch32. + /// 1 The Execution state for EL1 is AArch64. The Execution state for + /// EL0 is determined by the current value of PSTATE.nRW when + /// executing at EL0. + /// + /// If all lower Exception levels cannot use AArch32 then this bit is + /// RAO/WI. + /// + /// In an implementation that includes EL3, when SCR_EL3.NS==0, the PE + /// behaves as if this bit has the same value as the SCR_EL3.RW bit for + /// all purposes other than a direct read or write access of HCR_EL2. + /// + /// The RW bit is permitted to be cached in a TLB. + /// + /// When ARMv8.1-VHE is implemented, and the value of HCR_EL2.{E2H, TGE} + /// is {1, 1}, this field behaves as 1 for all purposes other than a + /// direct read of the value of this bit. + RW OFFSET(31) NUMBITS(1) [ + AllLowerELsAreAarch32 = 0, + EL1IsAarch64 = 1 + ], + + /// Default Cacheability. + /// + /// 0 This control has no effect on the Non-secure EL1&0 translation + /// regime. + /// + /// 1 In Non-secure state: + /// - When EL1 is using AArch64, the PE behaves as if the value of + /// the SCTLR_EL1.M field is 0 for all purposes other than + /// returning the value of a direct read of SCTLR_EL1. + /// + /// - When EL1 is using AArch32, the PE behaves as if the value of + /// the SCTLR.M field is 0 for all purposes other than returning + /// the value of a direct read of SCTLR. + /// + /// - The PE behaves as if the value of the HCR_EL2.VM field is 1 + /// for all purposes other than returning the value of a direct + /// read of HCR_EL2. + /// + /// - The memory type produced by stage 1 of the EL1&0 translation + /// regime is Normal Non-Shareable, Inner Write-Back Read-Allocate + /// Write-Allocate, Outer Write-Back Read-Allocate Write-Allocate. + /// + /// This field has no effect on the EL2, EL2&0, and EL3 translation + /// regimes. + /// + /// This field is permitted to be cached in a TLB. + /// + /// In an implementation that includes EL3, when the value of SCR_EL3.NS + /// is 0 the PE behaves as if this field is 0 for all purposes other + /// than a direct read or write access of HCR_EL2. + /// + /// When ARMv8.1-VHE is implemented, and the value of HCR_EL2.{E2H, TGE} + /// is {1, 1}, this field behaves as 0 for all purposes other than a + /// direct read of the value of this field. + DC OFFSET(12) NUMBITS(1) [], + + /// Set/Way Invalidation Override. Causes Non-secure EL1 execution of + /// the data cache invalidate by set/way instructions to perform a data + /// cache clean and invalidate by set/way: + /// + /// 0 This control has no effect on the operation of data cache + /// invalidate by set/way instructions. + /// + /// 1 Data cache invalidate by set/way instructions perform a data cache + /// clean and invalidate by set/way. + /// + /// When the value of this bit is 1: + /// + /// AArch32: DCISW performs the same invalidation as a DCCISW + /// instruction. + /// + /// AArch64: DC ISW performs the same invalidation as a DC CISW + /// instruction. + /// + /// This bit can be implemented as RES 1. + /// + /// In an implementation that includes EL3, when the value of SCR_EL3.NS + /// is 0 the PE behaves as if this field is 0 for all purposes other + /// than a direct read or write access of HCR_EL2. + /// + /// When HCR_EL2.TGE is 1, the PE ignores the value of this field for + /// all purposes other than a direct read of this field. + SWIO OFFSET(1) NUMBITS(1) [] + ] +} + +pub struct Reg; + +impl RegisterReadWrite for Reg { + sys_coproc_read_raw!(u64, "HCR_EL2"); + sys_coproc_write_raw!(u64, "HCR_EL2"); +} + +pub static HCR_EL2: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/id_aa64mmfr0_el1.rs b/crate/aarch64/src/regs/id_aa64mmfr0_el1.rs new file mode 100644 index 0000000..f75813c --- /dev/null +++ b/crate/aarch64/src/regs/id_aa64mmfr0_el1.rs @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2018 by the author(s) + * + * ============================================================================= + * + * Licensed under either of + * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + * - MIT License (http://opensource.org/licenses/MIT) + * at your option. + * + * ============================================================================= + * + * Author(s): + * - Andre Richter + */ + +//! AArch64 Memory Model Feature Register 0 - EL1 +//! +//! Provides information about the implemented memory model and memory +//! management support in AArch64 state. + +use register::cpu::RegisterReadOnly; + +register_bitfields! {u64, + ID_AA64MMFR0_EL1 [ + /// Support for 4KiB memory translation granule size. Defined values + /// are: + /// + /// 0000 4KiB granule supported. + /// 1111 4KiB granule not supported. + /// + /// All other values are reserved. + TGran4 OFFSET(28) NUMBITS(4) [ + Supported = 0b0000, + NotSupported = 0b1111 + ], + + /// Support for 64KiB memory translation granule size. Defined values + /// are: + /// + /// 0000 64KiB granule supported. + /// 1111 64KiB granule not supported. + /// + /// All other values are reserved. + TGran64 OFFSET(24) NUMBITS(4) [ + Supported = 0b0000, + NotSupported = 0b1111 + ], + + /// Physical Address range supported. Defined values are: + /// + /// 0000 32 bits, 4GiB. + /// 0001 36 bits, 64GiB. + /// 0010 40 bits, 1TiB. + /// 0011 42 bits, 4TiB. + /// 0100 44 bits, 16TiB. + /// 0101 48 bits, 256TiB. + /// 0110 52 bits, 4PiB. + /// + /// All other values are reserved. + /// + /// The value 0110 is permitted only if the implementation includes + /// ARMv8.2-LPA, otherwise it is reserved. + PARange OFFSET(0) NUMBITS(4) [ + Bits_32 = 0b0000, + Bits_36 = 0b0001, + Bits_40 = 0b0010, + Bits_42 = 0b0011, + Bits_44 = 0b0100, + Bits_48 = 0b0101, + Bits_52 = 0b0110 + ] + ] +} + +pub struct Reg; + +impl RegisterReadOnly for Reg { + sys_coproc_read_raw!(u64, "ID_AA64MMFR0_EL1"); +} + +pub static ID_AA64MMFR0_EL1: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/macros.rs b/crate/aarch64/src/regs/macros.rs new file mode 100644 index 0000000..bd4439c --- /dev/null +++ b/crate/aarch64/src/regs/macros.rs @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2018 by the author(s) + * + * ============================================================================= + * + * Licensed under either of + * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + * - MIT License (http://opensource.org/licenses/MIT) + * at your option. + * + * ============================================================================= + * + * Author(s): + * - Andre Richter + */ + +macro_rules! __read_raw { + ($width:ty, $asm_instr:tt, $asm_reg_name:tt) => { + /// Reads the raw bits of the CPU register. + #[inline] + fn get(&self) -> $width { + match () { + #[cfg(target_arch = "aarch64")] + () => { + let reg; + unsafe { + asm!(concat!($asm_instr, " $0, ", $asm_reg_name) : "=r"(reg) ::: "volatile"); + } + reg + } + + #[cfg(not(target_arch = "aarch64"))] + () => unimplemented!(), + } + } + }; +} + +macro_rules! __write_raw { + ($width:ty, $asm_instr:tt, $asm_reg_name:tt) => { + /// Writes raw bits to the CPU register. + #[cfg_attr(not(target_arch = "aarch64"), allow(unused_variables))] + #[inline] + fn set(&self, value: $width) { + match () { + #[cfg(target_arch = "aarch64")] + () => { + unsafe { + asm!(concat!($asm_instr, " ", $asm_reg_name, ", $0") :: "r"(value) :: "volatile") + } + } + + #[cfg(not(target_arch = "aarch64"))] + () => unimplemented!(), + } + } + }; +} + +/// Raw read from system coprocessor registers +macro_rules! sys_coproc_read_raw { + ($width:ty, $asm_reg_name:tt) => { + __read_raw!($width, "mrs", $asm_reg_name); + }; +} + +/// Raw write to system coprocessor registers +macro_rules! sys_coproc_write_raw { + ($width:ty, $asm_reg_name:tt) => { + __write_raw!($width, "msr", $asm_reg_name); + }; +} + +/// Raw read from (ordinary) registers +macro_rules! read_raw { + ($width:ty, $asm_reg_name:tt) => { + __read_raw!($width, "mov", $asm_reg_name); + }; +} +/// Raw write to (ordinary) registers +macro_rules! write_raw { + ($width:ty, $asm_reg_name:tt) => { + __write_raw!($width, "mov", $asm_reg_name); + }; +} diff --git a/crate/aarch64/src/regs/mair_el1.rs b/crate/aarch64/src/regs/mair_el1.rs new file mode 100644 index 0000000..2c7c7da --- /dev/null +++ b/crate/aarch64/src/regs/mair_el1.rs @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2018 by the author(s) + * + * ============================================================================= + * + * Licensed under either of + * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + * - MIT License (http://opensource.org/licenses/MIT) + * at your option. + * + * ============================================================================= + * + * Author(s): + * - Andre Richter + */ + +//! Memory Attribute Indirection Register - EL1 +//! +//! Provides the memory attribute encodings corresponding to the possible +//! AttrIndx values in a Long-descriptor format translation table entry for +//! stage 1 translations at EL1. + +use register::cpu::RegisterReadWrite; + +register_bitfields! {u64, + MAIR_EL1 [ + // TODO: Macrofy this + + /// Attribute 7 + Attr7_HIGH OFFSET(60) NUMBITS(4) [], + Attr7_LOW_DEVICE OFFSET(56) NUMBITS(4) [], + Attr7_LOW_MEMORY OFFSET(56) NUMBITS(4) [], + + /// Attribute 6 + Attr6_HIGH OFFSET(52) NUMBITS(4) [], + Attr6_LOW_DEVICE OFFSET(48) NUMBITS(4) [], + Attr6_LOW_MEMORY OFFSET(48) NUMBITS(4) [], + + /// Attribute 5 + Attr5_HIGH OFFSET(44) NUMBITS(4) [], + Attr5_LOW_DEVICE OFFSET(40) NUMBITS(4) [], + Attr5_LOW_MEMORY OFFSET(40) NUMBITS(4) [], + + /// Attribute 4 + Attr4_HIGH OFFSET(36) NUMBITS(4) [], + Attr4_LOW_DEVICE OFFSET(32) NUMBITS(4) [], + Attr4_LOW_MEMORY OFFSET(32) NUMBITS(4) [], + + /// Attribute 3 + Attr3_HIGH OFFSET(28) NUMBITS(4) [], + Attr3_LOW_DEVICE OFFSET(24) NUMBITS(4) [], + Attr3_LOW_MEMORY OFFSET(24) NUMBITS(4) [], + + /// Attribute 2 + Attr2_HIGH OFFSET(20) NUMBITS(4) [ + Device = 0b0000, + Memory_OuterNonCacheable = 0b0100, + Memory_OuterWriteBack_NonTransient_ReadAlloc_WriteAlloc = 0b1111 + ], + Attr2_LOW_DEVICE OFFSET(16) NUMBITS(4) [ + Device_nGnRE = 0b0100 + ], + Attr2_LOW_MEMORY OFFSET(16) NUMBITS(4) [ + InnerNonCacheable = 0b0100, + InnerWriteBack_NonTransient_ReadAlloc_WriteAlloc = 0b1111 + ], + + /// Attribute 1 + Attr1_HIGH OFFSET(12) NUMBITS(4) [ + Device = 0b0000, + Memory_OuterNonCacheable = 0b0100, + Memory_OuterWriteBack_NonTransient_ReadAlloc_WriteAlloc = 0b1111 + ], + Attr1_LOW_DEVICE OFFSET(8) NUMBITS(4) [ + Device_nGnRE = 0b0100 + ], + Attr1_LOW_MEMORY OFFSET(8) NUMBITS(4) [ + InnerNonCacheable = 0b0100, + InnerWriteBack_NonTransient_ReadAlloc_WriteAlloc = 0b1111 + ], + + /// Attribute 0 + Attr0_HIGH OFFSET(4) NUMBITS(4) [ + Device = 0b0000, + Memory_OuterNonCacheable = 0b0100, + Memory_OuterWriteBack_NonTransient_ReadAlloc_WriteAlloc = 0b1111 + ], + Attr0_LOW_DEVICE OFFSET(0) NUMBITS(4) [ + Device_nGnRE = 0b0100 + ], + Attr0_LOW_MEMORY OFFSET(0) NUMBITS(4) [ + InnerNonCacheable = 0b0100, + InnerWriteBack_NonTransient_ReadAlloc_WriteAlloc = 0b1111 + ] + ] +} + +pub struct Reg; + +impl RegisterReadWrite for Reg { + sys_coproc_read_raw!(u64, "MAIR_EL1"); + sys_coproc_write_raw!(u64, "MAIR_EL1"); +} + +pub static MAIR_EL1: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/mod.rs b/crate/aarch64/src/regs/mod.rs new file mode 100644 index 0000000..f6a0e3d --- /dev/null +++ b/crate/aarch64/src/regs/mod.rs @@ -0,0 +1,51 @@ +//! Processor core registers + +#[macro_use] +mod macros; + +mod cntfrq_el0; +mod cnthctl_el2; +mod cntp_ctl_el0; +mod cntp_tval_el0; +mod cntpct_el0; +mod cntvoff_el2; +mod currentel; +mod daif; +mod elr_el2; +mod hcr_el2; +mod id_aa64mmfr0_el1; +mod mair_el1; +mod mpidr_el1; +mod sctlr_el1; +mod sp; +mod sp_el0; +mod sp_el1; +mod spsel; +mod spsr_el2; +mod tcr_el1; +mod ttbr0_el1; + +// Export only the R/W traits and the static reg definitions +pub use register::cpu::*; + +pub use self::cntfrq_el0::CNTFRQ_EL0; +pub use self::cnthctl_el2::CNTHCTL_EL2; +pub use self::cntp_ctl_el0::CNTP_CTL_EL0; +pub use self::cntp_tval_el0::CNTP_TVAL_EL0; +pub use self::cntpct_el0::CNTPCT_EL0; +pub use self::cntvoff_el2::CNTVOFF_EL2; +pub use self::currentel::CurrentEL; +pub use self::daif::DAIF; +pub use self::elr_el2::ELR_EL2; +pub use self::hcr_el2::HCR_EL2; +pub use self::id_aa64mmfr0_el1::ID_AA64MMFR0_EL1; +pub use self::mair_el1::MAIR_EL1; +pub use self::mpidr_el1::MPIDR_EL1; +pub use self::sctlr_el1::SCTLR_EL1; +pub use self::sp::SP; +pub use self::sp_el0::SP_EL0; +pub use self::sp_el1::SP_EL1; +pub use self::spsel::SPSel; +pub use self::spsr_el2::SPSR_EL2; +pub use self::tcr_el1::TCR_EL1; +pub use self::ttbr0_el1::TTBR0_EL1; diff --git a/crate/aarch64/src/regs/mpidr_el1.rs b/crate/aarch64/src/regs/mpidr_el1.rs new file mode 100644 index 0000000..6fbfea0 --- /dev/null +++ b/crate/aarch64/src/regs/mpidr_el1.rs @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2018 by the author(s) + * + * ============================================================================= + * + * Licensed under either of + * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + * - MIT License (http://opensource.org/licenses/MIT) + * at your option. + * + * ============================================================================= + * + * Author(s): + * - Andre Richter + */ + +//! Multiprocessor Affinity Register - EL1 +//! +//! In a multiprocessor system, provides an additional PE +//! identification mechanism for scheduling purposes. + +use register::cpu::RegisterReadOnly; + +pub struct Reg; + +impl RegisterReadOnly for Reg { + sys_coproc_read_raw!(u64, "MPIDR_EL1"); +} + +pub static MPIDR_EL1: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/sctlr_el1.rs b/crate/aarch64/src/regs/sctlr_el1.rs new file mode 100644 index 0000000..1f463b4 --- /dev/null +++ b/crate/aarch64/src/regs/sctlr_el1.rs @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2018 by the author(s) + * + * ============================================================================= + * + * Licensed under either of + * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + * - MIT License (http://opensource.org/licenses/MIT) + * at your option. + * + * ============================================================================= + * + * Author(s): + * - Andre Richter + */ + +//! System Control Register - EL1 +//! +//! Provides top level control of the system, including its memory system, at +//! EL1 and EL0. + +use register::cpu::RegisterReadWrite; + +register_bitfields! {u32, + SCTLR_EL1 [ + /// Instruction access Cacheability control, for accesses at EL0 and + /// EL1: + /// + /// 0 All instruction access to Normal memory from EL0 and EL1 are + /// Non-cacheable for all levels of instruction and unified cache. + /// + /// If the value of SCTLR_EL1.M is 0, instruction accesses from stage + /// 1 of the EL1&0 translation regime are to Normal, Outer Shareable, + /// Inner Non-cacheable, Outer Non-cacheable memory. + /// + /// 1 This control has no effect on the Cacheability of instruction + /// access to Normal memory from EL0 and EL1. + /// + /// If the value of SCTLR_EL1.M is 0, instruction accesses from stage + /// 1 of the EL1&0 translation regime are to Normal, Outer Shareable, + /// Inner Write-Through, Outer Write-Through memory. + /// + /// When the value of the HCR_EL2.DC bit is 1, then instruction access + /// to Normal memory from EL0 and EL1 are Cacheable regardless of the + /// value of the SCTLR_EL1.I bit. + /// + /// When ARMv8.1-VHE is implemented, and the value of HCR_EL2.{E2H, TGE} + /// is {1, 1}, this bit has no effect on the PE. + /// + /// When this register has an architecturally-defined reset value, this + /// field resets to 0. + I OFFSET(12) NUMBITS(1) [ + NonCacheable = 0, + Cacheable = 1 + ], + + /// Cacheability control, for data accesses. + /// + /// 0 All data access to Normal memory from EL0 and EL1, and all Normal + /// memory accesses to the EL1&0 stage 1 translation tables, are + /// Non-cacheable for all levels of data and unified cache. + /// + /// 1 This control has no effect on the Cacheability of: + /// - Data access to Normal memory from EL0 and EL1. + /// - Normal memory accesses to the EL1&0 stage 1 translation + /// tables. + /// + /// When the value of the HCR_EL2.DC bit is 1, the PE ignores + /// SCLTR.C. This means that Non-secure EL0 and Non-secure EL1 data + /// accesses to Normal memory are Cacheable. + /// + /// When ARMv8.1-VHE is implemented, and the value of HCR_EL2.{E2H, TGE} + /// is {1, 1}, this bit has no effect on the PE. + /// + /// When this register has an architecturally-defined reset value, this + /// field resets to 0. + C OFFSET(2) NUMBITS(1) [ + NonCacheable = 0, + Cacheable = 1 + ], + + /// MMU enable for EL1 and EL0 stage 1 address translation. Possible + /// values of this bit are: + /// + /// 0 EL1 and EL0 stage 1 address translation disabled. + /// See the SCTLR_EL1.I field for the behavior of instruction accesses + /// to Normal memory. + /// 1 EL1 and EL0 stage 1 address translation enabled. + M OFFSET(0) NUMBITS(1) [ + Disable = 0, + Enable = 1 + ] + ] +} + +pub struct Reg; + +impl RegisterReadWrite for Reg { + sys_coproc_read_raw!(u32, "SCTLR_EL1"); + sys_coproc_write_raw!(u32, "SCTLR_EL1"); +} + +pub static SCTLR_EL1: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/sp.rs b/crate/aarch64/src/regs/sp.rs new file mode 100644 index 0000000..f9f578b --- /dev/null +++ b/crate/aarch64/src/regs/sp.rs @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2018 by the author(s) + * + * ============================================================================= + * + * Licensed under either of + * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + * - MIT License (http://opensource.org/licenses/MIT) + * at your option. + * + * ============================================================================= + * + * Author(s): + * - Andre Richter + */ + +//! The stack pointer + +use register::cpu::RegisterReadWrite; + +pub struct Reg; + +impl RegisterReadWrite for Reg { + read_raw!(u64, "sp"); + write_raw!(u64, "sp"); +} + +pub static SP: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/sp_el0.rs b/crate/aarch64/src/regs/sp_el0.rs new file mode 100644 index 0000000..aa82fdb --- /dev/null +++ b/crate/aarch64/src/regs/sp_el0.rs @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2018 by the author(s) + * + * ============================================================================= + * + * Licensed under either of + * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + * - MIT License (http://opensource.org/licenses/MIT) + * at your option. + * + * ============================================================================= + * + * Author(s): + * - Andre Richter + */ + +//! The stack pointer - EL0 +//! +//! Holds the stack pointer associated with EL0. At higher Exception levels, +//! this is used as the current stack pointer when the value of SPSel.SP is 0. + +use register::cpu::RegisterReadWrite; + +pub struct Reg; + +impl RegisterReadWrite for Reg { + sys_coproc_read_raw!(u64, "SP_EL0"); + sys_coproc_write_raw!(u64, "SP_EL0"); +} + +pub static SP_EL0: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/sp_el1.rs b/crate/aarch64/src/regs/sp_el1.rs new file mode 100644 index 0000000..4357412 --- /dev/null +++ b/crate/aarch64/src/regs/sp_el1.rs @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2018 by the author(s) + * + * ============================================================================= + * + * Licensed under either of + * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + * - MIT License (http://opensource.org/licenses/MIT) + * at your option. + * + * ============================================================================= + * + * Author(s): + * - Andre Richter + */ + +//! The stack pointer - EL1 +//! +//! Holds the stack pointer associated with EL1. When executing at EL1, the +//! value of SPSel.SP determines the current stack pointer: +//! +//! SPSel.SP | current stack pointer +//! -------------------------------- +//! 0 | SP_EL0 +//! 1 | SP_EL1 + +use register::cpu::RegisterReadWrite; + +pub struct Reg; + +impl RegisterReadWrite for Reg { + sys_coproc_read_raw!(u64, "SP_EL1"); + sys_coproc_write_raw!(u64, "SP_EL1"); +} + +pub static SP_EL1: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/spsel.rs b/crate/aarch64/src/regs/spsel.rs new file mode 100644 index 0000000..91e3694 --- /dev/null +++ b/crate/aarch64/src/regs/spsel.rs @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2018 by the author(s) + * + * ============================================================================= + * + * Licensed under either of + * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + * - MIT License (http://opensource.org/licenses/MIT) + * at your option. + * + * ============================================================================= + * + * Author(s): + * - Andre Richter + */ + +//! Stack Pointer Select +//! +//! Allows the Stack Pointer to be selected between SP_EL0 and SP_ELx. + +use register::cpu::RegisterReadWrite; + +register_bitfields! {u32, + SPSel [ + /// Stack pointer to use. Possible values of this bit are: + /// + /// 0 Use SP_EL0 at all Exception levels. + /// 1 Use SP_ELx for Exception level ELx. + /// + /// When this register has an architecturally-defined reset value, this + /// field resets to 1. + SP OFFSET(0) NUMBITS(1) [ + EL0 = 0, + ELx = 1 + ] + ] +} + + +pub struct Reg; + +impl RegisterReadWrite for Reg { + sys_coproc_read_raw!(u32, "SPSEL"); + sys_coproc_write_raw!(u32, "SPSEL"); +} + +#[allow(non_upper_case_globals)] +pub static SPSel: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/spsr_el2.rs b/crate/aarch64/src/regs/spsr_el2.rs new file mode 100644 index 0000000..56078a4 --- /dev/null +++ b/crate/aarch64/src/regs/spsr_el2.rs @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2018 by the author(s) + * + * ============================================================================= + * + * Licensed under either of + * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + * - MIT License (http://opensource.org/licenses/MIT) + * at your option. + * + * ============================================================================= + * + * Author(s): + * - Andre Richter + */ + +//! Saved Program Status Register - EL2 +//! +//! Holds the saved process state when an exception is taken to EL2. + +use register::cpu::RegisterReadWrite; + +register_bitfields! {u32, + SPSR_EL2 [ + /// Process state D mask. The possible values of this bit are: + /// + /// 0 Watchpoint, Breakpoint, and Software Step exceptions targeted at + /// the current Exception level are not masked. + /// + /// 1 Watchpoint, Breakpoint, and Software Step exceptions targeted at + /// the current Exception level are masked. + /// + /// When the target Exception level of the debug exception is higher + /// than the current Exception level, the exception is not masked by + /// this bit. + D OFFSET(9) NUMBITS(1) [ + Unmasked = 0, + Masked = 1 + ], + + /// SError interrupt mask bit. The possible values of this bit are: + /// + /// 0 Exception not masked. + /// 1 Exception masked. + A OFFSET(8) NUMBITS(1) [ + Unmasked = 0, + Masked = 1 + ], + + /// IRQ mask bit. The possible values of this bit are: + /// + /// 0 Exception not masked. + /// 1 Exception masked. + I OFFSET(7) NUMBITS(1) [ + Unmasked = 0, + Masked = 1 + ], + + /// FIQ mask bit. The possible values of this bit are: + /// + /// 0 Exception not masked. + /// 1 Exception masked. + F OFFSET(6) NUMBITS(1) [ + Unmasked = 0, + Masked = 1 + ], + + /// AArch64 state (Exception level and selected SP) that an exception + /// was taken from. The possible values are: + /// + /// M[3:0] | State + /// -------------- + /// 0b0000 | EL0t + /// 0b0100 | EL1t + /// 0b0101 | EL1h + /// 0b1000 | EL2t + /// 0b1001 | EL2h + /// + /// Other values are reserved, and returning to an Exception level that + /// is using AArch64 with a reserved value in this field is treated as + /// an illegal exception return. + /// + /// The bits in this field are interpreted as follows: + /// - M[3:2] holds the Exception Level. + /// - M[1] is unused and is RES 0 for all non-reserved values. + /// - M[0] is used to select the SP: + /// - 0 means the SP is always SP0. + /// - 1 means the exception SP is determined by the EL. + M OFFSET(0) NUMBITS(4) [ + EL0t = 0b0000, + EL1t = 0b0100, + EL1h = 0b0101, + EL2t = 0b1000, + EL2h = 0b1001 + ] + ] +} + +pub struct Reg; + +impl RegisterReadWrite for Reg { + sys_coproc_read_raw!(u32, "SPSR_EL2"); + sys_coproc_write_raw!(u32, "SPSR_EL2"); +} + +pub static SPSR_EL2: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/tcr_el1.rs b/crate/aarch64/src/regs/tcr_el1.rs new file mode 100644 index 0000000..bcd0425 --- /dev/null +++ b/crate/aarch64/src/regs/tcr_el1.rs @@ -0,0 +1,178 @@ +/* + * Copyright (c) 2018 by the author(s) + * + * ============================================================================= + * + * Licensed under either of + * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + * - MIT License (http://opensource.org/licenses/MIT) + * at your option. + * + * ============================================================================= + * + * Author(s): + * - Andre Richter + */ + +//! Translation Control Register - EL1 +//! +//! The control register for stage 1 of the EL1&0 translation regime. + +use register::cpu::RegisterReadWrite; + +register_bitfields! {u64, + TCR_EL1 [ + /// Top Byte ignored - indicates whether the top byte of an address is + /// used for address match for the TTBR0_EL1 region, or ignored and used + /// for tagged addresses. Defined values are: + /// + /// 0 Top Byte used in the address calculation. + /// 1 Top Byte ignored in the address calculation. + TBI0 OFFSET(37) NUMBITS(1) [ + Used = 0, + Ignored = 1 + ], + + /// Intermediate Physical Address Size. + /// + /// 000 32 bits, 4GiB. + /// 001 36 bits, 64GiB. + /// 010 40 bits, 1TiB. + /// 011 42 bits, 4TiB. + /// 100 44 bits, 16TiB. + /// 101 48 bits, 256TiB. + /// 110 52 bits, 4PiB + /// + /// Other values are reserved. + /// + /// The reserved values behave in the same way as the 101 or 110 + /// encoding, but software must not rely on this property as the + /// behavior of the reserved values might change in a future revision of + /// the architecture. + /// + /// The value 110 is permitted only if ARMv8.2-LPA is implemented and + /// the translation granule size is 64KiB. + /// + /// In an implementation that supports 52-bit PAs, if the value of this + /// field is not 110 , then bits[51:48] of every translation table base + /// address for the stage of translation controlled by TCR_EL1 are 0000 + /// . + IPS OFFSET(32) NUMBITS(3) [ + Bits_32 = 0b000, + Bits_36 = 0b001, + Bits_40 = 0b010, + Bits_42 = 0b011, + Bits_44 = 0b100, + Bits_48 = 0b101, + Bits_52 = 0b110 + ], + + /// Granule size for the TTBR0_EL1. + /// + /// 00 4KiB + /// 01 64KiB + /// 10 16KiB + /// + /// Other values are reserved. + /// + /// If the value is programmed to either a reserved value, or a size + /// that has not been implemented, then the hardware will treat the + /// field as if it has been programmed to an IMPLEMENTATION DEFINED + /// choice of the sizes that has been implemented for all purposes other + /// than the value read back from this register. + /// + /// It is IMPLEMENTATION DEFINED whether the value read back is the + /// value programmed or the value that corresponds to the size chosen. + TG0 OFFSET(14) NUMBITS(2) [ + KiB_4 = 0b00, + KiB_16 = 0b10, + KiB_64 = 0b01 + ], + + /// Shareability attribute for memory associated with translation table + /// walks using TTBR0_EL1. + /// + /// 00 Non-shareable + /// 10 Outer Shareable + /// 11 Inner Shareable + /// + /// Other values are reserved. + SH0 OFFSET(12) NUMBITS(2) [ + None = 0b00, + Outer = 0b10, + Inner = 0b11 + ], + + /// Outer cacheability attribute for memory associated with translation + /// table walks using TTBR0_EL1. + /// + /// 00 Normal memory, Outer Non-cacheable + /// + /// 01 Normal memory, Outer Write-Back Read-Allocate Write-Allocate + /// Cacheable + /// + /// 10 Normal memory, Outer Write-Through Read-Allocate No + /// Write-Allocate Cacheable + /// + /// 11 Normal memory, Outer Write-Back Read-Allocate No Write-Allocate + /// Cacheable + ORGN0 OFFSET(10) NUMBITS(2) [ + NonCacheable = 0b00, + WriteBack_ReadAlloc_WriteAlloc_Cacheable = 0b01, + WriteThrough_ReadAlloc_NoWriteAlloc_Cacheable = 0b10, + WriteBack_ReadAlloc_NoWriteAlloc_Cacheable = 0b11 + ], + + /// Inner cacheability attribute for memory associated with translation + /// table walks using TTBR0_EL1. + /// + /// 00 Normal memory, Inner Non-cacheable + /// + /// 01 Normal memory, Inner Write-Back Read-Allocate Write-Allocate + /// Cacheable + /// + /// 10 Normal memory, Inner Write-Through Read-Allocate No + /// Write-Allocate Cacheable + /// + /// 11 Normal memory, Inner Write-Back Read-Allocate No Write-Allocate + /// Cacheable + IRGN0 OFFSET(8) NUMBITS(2) [ + NonCacheable = 0b00, + WriteBack_ReadAlloc_WriteAlloc_Cacheable = 0b01, + WriteThrough_ReadAlloc_NoWriteAlloc_Cacheable = 0b10, + WriteBack_ReadAlloc_NoWriteAlloc_Cacheable = 0b11 + ], + + /// Translation table walk disable for translations using + /// TTBR0_EL1. This bit controls whether a translation table walk is + /// performed on a TLB miss, for an address that is translated using + /// TTBR0_EL1. The encoding of this bit is: + /// + /// 0 Perform translation table walks using TTBR0_EL1. + /// + /// 1 A TLB miss on an address that is translated using TTBR0_EL1 + /// generates a Translation fault. No translation table walk is + /// performed. + EPD0 OFFSET(7) NUMBITS(1) [ + EnableTTBR0Walks = 0, + DisableTTBR0Walks = 1 + ], + + /// The size offset of the memory region addressed by TTBR0_EL1. The + /// region size is 2^(64-T0SZ) bytes. + /// + /// The maximum and minimum possible values for T0SZ depend on the level + /// of translation table and the memory translation granule size, as + /// described in the AArch64 Virtual Memory System Architecture chapter. + T0SZ OFFSET(0) NUMBITS(6) [] + ] +} + +pub struct Reg; + +impl RegisterReadWrite for Reg { + sys_coproc_read_raw!(u64, "TCR_EL1"); + sys_coproc_write_raw!(u64, "TCR_EL1"); +} + +pub static TCR_EL1: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/ttbr0_el1.rs b/crate/aarch64/src/regs/ttbr0_el1.rs new file mode 100644 index 0000000..c111256 --- /dev/null +++ b/crate/aarch64/src/regs/ttbr0_el1.rs @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2018 by the author(s) + * + * ============================================================================= + * + * Licensed under either of + * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + * - MIT License (http://opensource.org/licenses/MIT) + * at your option. + * + * ============================================================================= + * + * Author(s): + * - Andre Richter + */ + +//! Translation Table Base Register 0 - EL1 +//! +//! Holds the base address of the translation table for the initial lookup for +//! stage 1 of the translation of an address from the lower VA range in the +//! EL1&0 translation regime, and other information for this translation regime. + +use register::cpu::RegisterReadWrite; + +register_bitfields! {u64, + TTBR0_EL1 [ + /// An ASID for the translation table base address. The TCR_EL1.A1 field + /// selects either TTBR0_EL1.ASID or TTBR1_EL1.ASID. + /// + /// If the implementation has only 8 bits of ASID, then the upper 8 bits + /// of this field are RES 0. + ASID OFFSET(48) NUMBITS(16) [], + + /// Translation table base address + BADDR OFFSET(1) NUMBITS(47) [], + + /// Common not Private + CnP OFFSET(0) NUMBITS(1) [] + ] +} + +pub struct Reg; + +impl RegisterReadWrite for Reg { + sys_coproc_read_raw!(u64, "TTBR0_EL1"); + sys_coproc_write_raw!(u64, "TTBR0_EL1"); +} + +impl Reg { + #[inline] + pub fn set_baddr(&self, addr: u64) { + self.write(TTBR0_EL1::BADDR.val(addr >> 1)); + } +} + +pub static TTBR0_EL1: Reg = Reg {}; diff --git a/kernel/Cargo.lock b/kernel/Cargo.lock index e1b7d05..05f0710 100644 --- a/kernel/Cargo.lock +++ b/kernel/Cargo.lock @@ -1,3 +1,16 @@ +[[package]] +name = "aarch64" +version = "0.1.0" +dependencies = [ + "bare-metal 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "bit_field 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", + "os_bootinfo 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "register 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "usize_conversions 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "ux 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "atags" version = "0.1.0" @@ -243,6 +256,7 @@ dependencies = [ name = "ucore" version = "0.1.0" dependencies = [ + "aarch64 0.1.0", "atags 0.1.0", "bbl 0.1.0", "bcm2837 0.1.0", diff --git a/kernel/Cargo.toml b/kernel/Cargo.toml index e2bba97..c80b6ef 100644 --- a/kernel/Cargo.toml +++ b/kernel/Cargo.toml @@ -45,6 +45,7 @@ bbl = { path = "../crate/bbl" } [target.'cfg(target_arch = "aarch64")'.dependencies] cortex-a = "2.2.1" +aarch64 = { path = "../crate/aarch64" } atags = { path = "../crate/atags" } bcm2837 = { path = "../crate/bcm2837", features = ["use_generic_timer"] } diff --git a/kernel/src/arch/aarch64/memory.rs b/kernel/src/arch/aarch64/memory.rs index bf55354..32990d2 100644 --- a/kernel/src/arch/aarch64/memory.rs +++ b/kernel/src/arch/aarch64/memory.rs @@ -1,22 +1,78 @@ //! Memory initialization for aarch64. +use bit_allocator::BitAlloc; use ucore_memory::PAGE_SIZE; +use memory::{FRAME_ALLOCATOR, init_heap}; use super::atags::atags::Atags; -use super::super::HEAP_ALLOCATOR; +//use super::super::HEAP_ALLOCATOR; +use aarch64::{barrier, regs::*}; +use core::ops::Range; /// Memory initialization. pub fn init() { - let (start, end) = memory_map().expect("failed to find memory map"); + /*let (start, end) = memory_map().expect("failed to find memory map"); unsafe { HEAP_ALLOCATOR.lock().init(start, end - start); - } - info!("memory: init end"); + }*/ + + init_frame_allocator(); + init_heap(); + init_mmu(); } extern "C" { static _end: u8; } + +fn init_frame_allocator() { + let mut ba = FRAME_ALLOCATOR.lock(); + let (start, end) = memory_map().expect("failed to find memory map"); + ba.insert(to_range(start, end)); + info!("FrameAllocator init end"); + + fn to_range(start: usize, end: usize) -> Range { + let page_start = start / PAGE_SIZE; + let page_end = (end - 1) / PAGE_SIZE + 1; + page_start..page_end + } +} + +fn init_mmu() { + // device. + MAIR_EL1.write( + // Attribute 1 + MAIR_EL1::Attr1_HIGH::Device + + MAIR_EL1::Attr1_LOW_DEVICE::Device_nGnRE + // Attribute 0 + + MAIR_EL1::Attr0_HIGH::Memory_OuterWriteBack_NonTransient_ReadAlloc_WriteAlloc + + MAIR_EL1::Attr0_LOW_MEMORY::InnerWriteBack_NonTransient_ReadAlloc_WriteAlloc, + ); + // Configure various settings of stage 1 of the EL1 translation regime. + let ips = ID_AA64MMFR0_EL1.read(ID_AA64MMFR0_EL1::PARange); + TCR_EL1.write( + TCR_EL1::TBI0::Ignored + + TCR_EL1::IPS.val(ips) + + TCR_EL1::TG0::KiB_4 // 4 KiB granule + + TCR_EL1::SH0::Inner + + TCR_EL1::ORGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable + + TCR_EL1::IRGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable + + TCR_EL1::EPD0::EnableTTBR0Walks + + TCR_EL1::T0SZ.val(34), // Start walks at level 2 + ); + + // Switch the MMU on. + // + // First, force all previous changes to be seen before the MMU is enabled. + unsafe { barrier::isb(barrier::SY); } + + // Enable the MMU and turn on data and instruction caching. + SCTLR_EL1.modify(SCTLR_EL1::M::Enable + SCTLR_EL1::C::Cacheable + SCTLR_EL1::I::Cacheable); + + // Force MMU init to complete before next instruction + unsafe { barrier::isb(barrier::SY); } +} + /// Returns the (start address, end address) of the available memory on this /// system if it can be determined. If it cannot, `None` is returned. /// @@ -33,3 +89,4 @@ pub fn memory_map() -> Option<(usize, usize)> { None } + diff --git a/kernel/src/arch/aarch64/paging.rs b/kernel/src/arch/aarch64/paging.rs index e9cc43e..b468f8b 100644 --- a/kernel/src/arch/aarch64/paging.rs +++ b/kernel/src/arch/aarch64/paging.rs @@ -1,218 +1,233 @@ //! Page table implementations for aarch64. - +use bit_allocator::{BitAlloc}; +// Depends on kernel +use memory::{active_table, alloc_frame, alloc_stack, dealloc_frame}; use ucore_memory::memory_set::*; +use ucore_memory::PAGE_SIZE; use ucore_memory::paging::*; - -type VirtAddr = usize; -type PhysAddr = usize; - -use alloc::alloc::{alloc, Layout}; -use memory::{active_table, alloc_frame, alloc_stack, dealloc_frame}; - -/// TODO -pub struct ActivePageTable { - // TODO +use aarch64::asm::{tlb_invalidate, ttbr0_el1_read, ttbr0_el1_write}; +use aarch64::{PhysAddr, VirtAddr}; +use aarch64::paging::{Mapper, PageTable as Aarch64PageTable, PageTableEntry, PageTableFlags as EF, RecursivePageTable}; +use aarch64::paging::{FrameAllocator, FrameDeallocator, Page, PageRange, PhysFrame as Frame, Size4KiB}; +use aarch64::{regs::*}; + +pub trait PageExt { + fn of_addr(address: usize) -> Self; + fn range_of(begin: usize, end: usize) -> PageRange; } -impl ActivePageTable { - /// TODO - pub unsafe fn new() -> Self { - unimplemented!() - } -} - -impl PageTable for ActivePageTable { - type Entry = PageEntry; - - fn map(&mut self, addr: VirtAddr, target: PhysAddr) -> &mut Self::Entry { - unimplemented!() - } - fn unmap(&mut self, addr: VirtAddr) { - unimplemented!() - } - - fn get_entry(&mut self, addr: VirtAddr) -> &mut Self::Entry { - unimplemented!() - } - - // For testing with mock - fn get_page_slice_mut<'a, 'b>(&'a mut self, addr: VirtAddr) -> &'b mut [u8] { - unimplemented!() +impl PageExt for Page { + fn of_addr(address: usize) -> Self { + Page::containing_address(VirtAddr::new(address as u64)) } - - fn read(&mut self, addr: VirtAddr) -> u8 { - unimplemented!() - } - - fn write(&mut self, addr: VirtAddr, data: u8) { - unimplemented!() + fn range_of(begin: usize, end: usize) -> PageRange { + Page::range(Page::of_addr(begin), Page::of_addr(end - 1) + 1) } } -/// TODO -pub struct PageEntry { - // TODO +pub trait FrameExt { + fn of_addr(address: usize) -> Self; } -impl Entry for PageEntry { - /// IMPORTANT! - /// This must be called after any change to ensure it become effective. - /// Usually this will make a flush to TLB/MMU. - fn update(&mut self) { - unimplemented!() - } - - /// Will be set when accessed - fn accessed(&self) -> bool { - unimplemented!() +impl FrameExt for Frame { + fn of_addr(address: usize) -> Self { + Frame::containing_address(PhysAddr::new(address as u64)) } +} - /// Will be set when written - fn dirty(&self) -> bool { - unimplemented!() - } +pub struct ActivePageTable(RecursivePageTable<'static>); - /// Will PageFault when try to write page where writable=0 - fn writable(&self) -> bool { - unimplemented!() - } +pub struct PageEntry(PageTableEntry); - /// Will PageFault when try to access page where present=0 - fn present(&self) -> bool { - unimplemented!() - } - - fn clear_accessed(&mut self) { - unimplemented!() - } +impl PageTable for ActivePageTable { + type Entry = PageEntry; - fn clear_dirty(&mut self) { - unimplemented!() + fn map(&mut self, addr: usize, target: usize) -> &mut PageEntry { + let flags = EF::PRESENT | EF::WRITE | EF::UXN; + self.0.map_to(Page::of_addr(addr), Frame::of_addr(target), flags, &mut FrameAllocatorForAarch64) + .unwrap().flush(); + self.get_entry(addr) } - fn set_writable(&mut self, value: bool) { - unimplemented!() + fn unmap(&mut self, addr: usize) { + let (frame, flush) = self.0.unmap(Page::of_addr(addr)).unwrap(); + flush.flush(); } - fn set_present(&mut self, value: bool) { - unimplemented!() + fn get_entry(&mut self, addr: usize) -> &mut PageEntry { + let entry_addr = ((addr >> 9) & 0o777_777_777_7770) | 0xffffff80_00000000; + unsafe { &mut *(entry_addr as *mut PageEntry) } } - fn target(&self) -> PhysAddr { - unimplemented!() + fn get_page_slice_mut<'a, 'b>(&'a mut self, addr: usize) -> &'b mut [u8] { + use core::slice; + unsafe { slice::from_raw_parts_mut((addr & !0xfffusize) as *mut u8, PAGE_SIZE) } } - fn set_target(&mut self, target: PhysAddr) { - unimplemented!() + fn read(&mut self, addr: usize) -> u8 { + unsafe { *(addr as *const u8) } } - // For Copy-on-write extension - fn writable_shared(&self) -> bool { - unimplemented!() + fn write(&mut self, addr: usize, data: u8) { + unsafe { *(addr as *mut u8) = data; } } +} - fn readonly_shared(&self) -> bool { - unimplemented!() +impl ActivePageTable { + pub unsafe fn new() -> Self { + ActivePageTable(RecursivePageTable::new(&mut *(0xffffffff_fffff000 as *mut _)).unwrap()) + } + fn with_temporary_map(&mut self, frame: &Frame, f: impl FnOnce(&mut ActivePageTable, &mut Aarch64PageTable)) { + // Create a temporary page + let page = Page::of_addr(0xcafebabe); + assert!(self.0.translate_page(page).is_none(), "temporary page is already mapped"); + // Map it to table + self.map(page.start_address().as_u64() as usize, frame.start_address().as_u64() as usize); + // Call f + let table = unsafe { &mut *page.start_address().as_mut_ptr() }; + f(self, table); + // Unmap the page + self.unmap(0xcafebabe); } +} - fn set_shared(&mut self, writable: bool) { - unimplemented!() +impl Entry for PageEntry { + fn update(&mut self) { + tlb_invalidate(); } - fn clear_shared(&mut self) { - unimplemented!() - } + fn present(&self) -> bool { self.0.flags().contains(EF::PRESENT) } + fn accessed(&self) -> bool { self.0.flags().contains(EF::ACCESSED) } + fn writable(&self) -> bool { self.0.flags().contains(EF::WRITE) } + fn dirty(&self) -> bool { self.hw_dirty() && self.sw_dirty() } - // For Swap extension - fn swapped(&self) -> bool { - unimplemented!() + fn clear_accessed(&mut self) { self.as_flags().remove(EF::ACCESSED); } + fn clear_dirty(&mut self) + { + self.as_flags().remove(EF::DIRTY); + self.as_flags().insert(EF::RDONLY); } - - fn set_swapped(&mut self, value: bool) { - unimplemented!() + fn set_writable(&mut self, value: bool) + { + self.as_flags().set(EF::RDONLY, !value); + self.as_flags().set(EF::WRITE, value); } - - fn user(&self) -> bool { - unimplemented!() + fn set_present(&mut self, value: bool) { self.as_flags().set(EF::PRESENT, value); } + fn target(&self) -> usize { self.0.addr().as_u64() as usize } + fn set_target(&mut self, target: usize) { + let flags = self.0.flags(); + self.0.set_addr(PhysAddr::new(target as u64), flags); } - + fn writable_shared(&self) -> bool { self.0.flags().contains(EF::BIT_9) } + fn readonly_shared(&self) -> bool { self.0.flags().contains(EF::BIT_9) } + fn set_shared(&mut self, writable: bool) { + let flags = self.as_flags(); + flags.set(EF::BIT_8, writable); + flags.set(EF::BIT_9, writable); + } + fn clear_shared(&mut self) { self.as_flags().remove(EF::BIT_8 | EF::BIT_9); } + fn user(&self) -> bool { self.0.flags().contains(EF::USER_ACCESSIBLE) } + fn swapped(&self) -> bool { self.0.flags().contains(EF::SWAPPED) } + fn set_swapped(&mut self, value: bool) { self.as_flags().set(EF::SWAPPED, value); } fn set_user(&mut self, value: bool) { - unimplemented!() - } - - fn execute(&self) -> bool { - unimplemented!() - } - - fn set_execute(&mut self, value: bool) { - unimplemented!() + self.as_flags().set(EF::USER_ACCESSIBLE, value); + if value { + let mut addr = self as *const _ as usize; + for _ in 0..3 { + // Upper level entry + addr = ((addr >> 9) & 0o777_777_777_7770) | 0xffffff80_00000000; + // set USER_ACCESSIBLE + unsafe { (*(addr as *mut EF)).insert(EF::USER_ACCESSIBLE) }; + } + } } + fn execute(&self) -> bool { !self.0.flags().contains(EF::UXN) } + fn set_execute(&mut self, value: bool) { self.as_flags().set(EF::UXN, !value); } } -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] -pub struct MockFrame(PhysAddr); - -impl MockFrame { - pub fn of_addr(addr: PhysAddr) -> Self { - MockFrame(addr) - } - pub fn start_address(&self) -> PhysAddr { - unimplemented!() - } - pub fn p2_index(&self) -> usize { - unimplemented!() - } - pub fn p1_index(&self) -> usize { - unimplemented!() - } - pub fn number(&self) -> usize { - unimplemented!() +impl PageEntry { + fn read_only(&self) -> bool { self.0.flags().contains(EF::RDONLY) } + fn hw_dirty(&self) -> bool { self.writable() && !self.read_only() } + fn sw_dirty(&self) -> bool { self.0.flags().contains(EF::DIRTY) } + fn as_flags(&mut self) -> &mut EF { + unsafe { &mut *(self as *mut _ as *mut EF) } } } -/// TODO +#[derive(Debug)] pub struct InactivePageTable0 { - p4_frame: MockFrame, + p4_frame: Frame, } -/// TODO impl InactivePageTable for InactivePageTable0 { type Active = ActivePageTable; fn new() -> Self { - unsafe { - let layout = Layout::new::(); - let ptr = alloc(layout); - let frame = MockFrame::of_addr(*ptr as usize); - InactivePageTable0 { p4_frame: frame } - } + let mut pt = Self::new_bare(); + pt.map_kernel(); + pt } fn new_bare() -> Self { - unimplemented!() + let frame = Self::alloc_frame().map(|target| Frame::of_addr(target)) + .expect("failed to allocate frame"); + active_table().with_temporary_map(&frame, |_, table: &mut Aarch64PageTable| { + table.zero(); + // set up recursive mapping for the table + table[511].set_frame(frame.clone(), EF::PRESENT | EF::WRITE); + }); + InactivePageTable0 { p4_frame: frame } } fn edit(&mut self, f: impl FnOnce(&mut Self::Active)) { - unimplemented!() + active_table().with_temporary_map(&ttbr0_el1_read().0, |active_table, p4_table: &mut Aarch64PageTable| { + let backup = p4_table[0o777].clone(); + + // overwrite recursive mapping + p4_table[0o777].set_frame(self.p4_frame.clone(), EF::PRESENT | EF::WRITE); + tlb_invalidate(); + + // execute f in the new context + f(active_table); + + // restore recursive mapping to original p4 table + p4_table[0o777] = backup; + tlb_invalidate(); + }); } unsafe fn activate(&self) { - unimplemented!() + let old_frame = ttbr0_el1_read().0; + let new_frame = self.p4_frame.clone(); + debug!("switch table {:?} -> {:?}", old_frame, new_frame); + if old_frame != new_frame { + ttbr0_el1_write(new_frame); + } } unsafe fn with(&self, f: impl FnOnce()) { - unimplemented!() + let old_frame = ttbr0_el1_read().0; + let new_frame = self.p4_frame.clone(); + debug!("switch table {:?} -> {:?}", old_frame, new_frame); + if old_frame != new_frame { + ttbr0_el1_write(new_frame); + } + f(); + debug!("switch table {:?} -> {:?}", new_frame, old_frame); + if old_frame != new_frame { + ttbr0_el1_write(old_frame); + } } fn token(&self) -> usize { - 0 + self.p4_frame.start_address().as_u64() as usize // as CR3 } - fn alloc_frame() -> Option { + fn alloc_frame() -> Option { alloc_frame() } - fn dealloc_frame(target: PhysAddr) { + fn dealloc_frame(target: usize) { dealloc_frame(target) } @@ -220,3 +235,37 @@ impl InactivePageTable for InactivePageTable0 { alloc_stack() } } + +impl InactivePageTable0 { + fn map_kernel(&mut self) { + let mut table = unsafe { &mut *(0xffffffff_fffff000 as *mut Aarch64PageTable) }; + // Kernel at 0xffff_ff00_0000_0000 + // Kernel stack at 0x0000_57ac_0000_0000 (defined in bootloader crate) + let e0 = table[0].clone(); + self.edit(|_| { + table[0].set_addr(e0.addr(), e0.flags() & EF::GLOBAL); + //table[175].set_addr(estack.addr(), estack.flags() & EF::GLOBAL); + }); + } +} + +impl Drop for InactivePageTable0 { + fn drop(&mut self) { + info!("PageTable dropping: {:?}", self); + Self::dealloc_frame(self.p4_frame.start_address().as_u64() as usize); + } +} + +struct FrameAllocatorForAarch64; + +impl FrameAllocator for FrameAllocatorForAarch64 { + fn alloc(&mut self) -> Option { + alloc_frame().map(|addr| Frame::of_addr(addr)) + } +} + +impl FrameDeallocator for FrameAllocatorForAarch64 { + fn dealloc(&mut self, frame: Frame) { + dealloc_frame(frame.start_address().as_u64() as usize); + } +} \ No newline at end of file diff --git a/kernel/src/lib.rs b/kernel/src/lib.rs index 4474c9c..063b1d0 100644 --- a/kernel/src/lib.rs +++ b/kernel/src/lib.rs @@ -33,6 +33,8 @@ extern crate volatile; #[cfg(target_arch = "x86_64")] extern crate x86_64; extern crate xmas_elf; +#[cfg(target_arch = "aarch64")] +extern crate aarch64; use linked_list_allocator::LockedHeap; From 576ae1d9111050ae57f66bef39a74f272ed3dfc5 Mon Sep 17 00:00:00 2001 From: equation314 Date: Thu, 15 Nov 2018 19:12:02 +0800 Subject: [PATCH 02/22] add setup_page_table --- crate/aarch64/src/paging/page_table.rs | 16 ++++++--- kernel/src/arch/aarch64/memory.rs | 46 ++++++++++++++++++++++---- kernel/src/arch/aarch64/paging.rs | 32 ++++++++++++++++-- 3 files changed, 80 insertions(+), 14 deletions(-) diff --git a/crate/aarch64/src/paging/page_table.rs b/crate/aarch64/src/paging/page_table.rs index b074768..1525b17 100644 --- a/crate/aarch64/src/paging/page_table.rs +++ b/crate/aarch64/src/paging/page_table.rs @@ -104,11 +104,10 @@ bitflags! { const SHARED = 3 << 8; /* SH[1:0], inner shareable */ const BIT_8 = 1 << 8; const BIT_9 = 1 << 9; - /* - pub const ATTRIB_SH_NON_SHAREABLE: usize = 0x0 << 8; - pub const ATTRIB_SH_OUTER_SHAREABLE: usize = 0x2 << 8; - pub const ATTRIB_SH_INNER_SHAREABLE: usize = 0x3 << 8; - */ + + // pub const ATTRIB_SH_NON_SHAREABLE: usize = 0x0 << 8; + const OUTER_SHAREABLE = 0b10 << 8; + const INNER_SHAREABLE = 0b11 << 8; const ACCESSED = 1 << 10; /* AF, Access Flag */ const NONE_GLOBAL = 1 << 11; /* None Global */ @@ -148,6 +147,13 @@ impl PageTable { entry.set_unused(); } } + + /// Setup identity map: VirtPage at pagenumber -> PhysFrame at pagenumber + /// pn: pagenumber = addr>>12 in riscv32. + pub fn map_identity(&mut self, p4num: usize, flags: PageTableFlags) { + let entry = self.entries[p4num].clone(); + self.entries[p4num].set_addr(entry.addr(), flags); + } } impl Index for PageTable { diff --git a/kernel/src/arch/aarch64/memory.rs b/kernel/src/arch/aarch64/memory.rs index 32990d2..126cf57 100644 --- a/kernel/src/arch/aarch64/memory.rs +++ b/kernel/src/arch/aarch64/memory.rs @@ -5,7 +5,8 @@ use ucore_memory::PAGE_SIZE; use memory::{FRAME_ALLOCATOR, init_heap}; use super::atags::atags::Atags; //use super::super::HEAP_ALLOCATOR; -use aarch64::{barrier, regs::*}; +use aarch64::{barrier, regs::*, addr::*}; +use aarch64::paging::{FrameAllocator, FrameDeallocator, Page, PageRange, PhysFrame as Frame, Size4KiB}; use core::ops::Range; /// Memory initialization. @@ -15,9 +16,27 @@ pub fn init() { HEAP_ALLOCATOR.lock().init(start, end - start); }*/ + + + #[repr(align(4096))] + struct PageData([u8; PAGE_SIZE]); + static PAGE_TABLE_ROOT: PageData = PageData([0; PAGE_SIZE]); + + let frame = Frame::containing_address(PhysAddr::new(&PAGE_TABLE_ROOT as *const _ as u64)); + super::paging::setup_page_table(frame); + + init_mmu(); + init_frame_allocator(); init_heap(); - init_mmu(); + + let (start, end) = memory_map().expect("failed to find memory map"); + let mut v = vec![]; + for i in 0..(20 + (start & 0xf)) { + v.push(i); + println!("{:x?} {:x?}", &v[i] as * const _ as usize, v); + } + } extern "C" { @@ -26,19 +45,30 @@ extern "C" { fn init_frame_allocator() { - let mut ba = FRAME_ALLOCATOR.lock(); + use consts::{MEMORY_OFFSET}; let (start, end) = memory_map().expect("failed to find memory map"); + info!("{:x?} {:x?}", start, end); + + let mut ba = FRAME_ALLOCATOR.lock(); + use core::mem::size_of; + use ::memory::FrameAlloc; + info!("{:x?} {:x?}", &FRAME_ALLOCATOR as *const _ as usize, size_of::()); + use consts::{KERNEL_HEAP_OFFSET, KERNEL_HEAP_SIZE}; + info!("{:x?} {:x?}", KERNEL_HEAP_OFFSET + KERNEL_HEAP_SIZE, end); ba.insert(to_range(start, end)); info!("FrameAllocator init end"); fn to_range(start: usize, end: usize) -> Range { - let page_start = start / PAGE_SIZE; - let page_end = (end - 1) / PAGE_SIZE + 1; + let page_start = (start - MEMORY_OFFSET) / PAGE_SIZE; + let page_end = (end - MEMORY_OFFSET - 1) / PAGE_SIZE + 1; + // info!("{:x?} {:x?}", page_start, page_end); page_start..page_end } } fn init_mmu() { + info!("init_mmu"); + // device. MAIR_EL1.write( // Attribute 1 @@ -50,6 +80,7 @@ fn init_mmu() { ); // Configure various settings of stage 1 of the EL1 translation regime. let ips = ID_AA64MMFR0_EL1.read(ID_AA64MMFR0_EL1::PARange); + info!("{:x?}", ips); TCR_EL1.write( TCR_EL1::TBI0::Ignored + TCR_EL1::IPS.val(ips) @@ -58,7 +89,7 @@ fn init_mmu() { + TCR_EL1::ORGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable + TCR_EL1::IRGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable + TCR_EL1::EPD0::EnableTTBR0Walks - + TCR_EL1::T0SZ.val(34), // Start walks at level 2 + + TCR_EL1::T0SZ.val(16), // Start walks at level 2 ); // Switch the MMU on. @@ -66,11 +97,14 @@ fn init_mmu() { // First, force all previous changes to be seen before the MMU is enabled. unsafe { barrier::isb(barrier::SY); } + info!("{:x?}", TCR_EL1.get()); // Enable the MMU and turn on data and instruction caching. SCTLR_EL1.modify(SCTLR_EL1::M::Enable + SCTLR_EL1::C::Cacheable + SCTLR_EL1::I::Cacheable); // Force MMU init to complete before next instruction unsafe { barrier::isb(barrier::SY); } + + info!("mmu enabled!"); } /// Returns the (start address, end address) of the available memory on this diff --git a/kernel/src/arch/aarch64/paging.rs b/kernel/src/arch/aarch64/paging.rs index b468f8b..603346d 100644 --- a/kernel/src/arch/aarch64/paging.rs +++ b/kernel/src/arch/aarch64/paging.rs @@ -11,6 +11,32 @@ use aarch64::paging::{Mapper, PageTable as Aarch64PageTable, PageTableEntry, Pag use aarch64::paging::{FrameAllocator, FrameDeallocator, Page, PageRange, PhysFrame as Frame, Size4KiB}; use aarch64::{regs::*}; +// need 1 page +pub fn setup_page_table(frame: Frame) { + let p4 = unsafe { &mut *(frame.start_address().as_u64() as *mut Aarch64PageTable) }; + p4.zero(); + + + // p4.set_recursive(RECURSIVE_PAGE_PML4, frame.clone()); + + // Set kernel identity map + // 0x10000000 ~ 1K area + p4.map_identity(0o777, EF::PRESENT | EF::PXN | EF::UXN); + + // 0x80000000 ~ 8K area + p4.map_identity(0, EF::PRESENT); + // p2.map_identity(KERNEL_PML4, EF::PRESENT | EF::READABLE | EF::WRITABLE); + // p2.map_identity(KERNEL_PML4 + 1, EF::VALID | EF::READABLE | EF::WRITABLE | EF::EXECUTABLE); + + // use super::riscv::register::satp; + // unsafe { satp::set(satp::Mode::Sv32, 0, frame); } + // sfence_vma_all(); + + ttbr0_el1_write(frame); + tlb_invalidate(); + info!("setup init page table end"); +} + pub trait PageExt { fn of_addr(address: usize) -> Self; fn range_of(begin: usize, end: usize) -> PageRange; @@ -75,7 +101,7 @@ impl PageTable for ActivePageTable { impl ActivePageTable { pub unsafe fn new() -> Self { - ActivePageTable(RecursivePageTable::new(&mut *(0xffffffff_fffff000 as *mut _)).unwrap()) + ActivePageTable(RecursivePageTable::new(&mut *(0xffff_ffff_ffff_f000 as *mut _)).unwrap()) } fn with_temporary_map(&mut self, frame: &Frame, f: impl FnOnce(&mut ActivePageTable, &mut Aarch64PageTable)) { // Create a temporary page @@ -238,7 +264,7 @@ impl InactivePageTable for InactivePageTable0 { impl InactivePageTable0 { fn map_kernel(&mut self) { - let mut table = unsafe { &mut *(0xffffffff_fffff000 as *mut Aarch64PageTable) }; + let mut table = unsafe { &mut *(0xffff_ffff_ffff_f000 as *mut Aarch64PageTable) }; // Kernel at 0xffff_ff00_0000_0000 // Kernel stack at 0x0000_57ac_0000_0000 (defined in bootloader crate) let e0 = table[0].clone(); @@ -268,4 +294,4 @@ impl FrameDeallocator for FrameAllocatorForAarch64 { fn dealloc(&mut self, frame: Frame) { dealloc_frame(frame.start_address().as_u64() as usize); } -} \ No newline at end of file +} From 9e8124abbbf4dace6e2eef451db74793c536fdc2 Mon Sep 17 00:00:00 2001 From: equation314 Date: Tue, 20 Nov 2018 16:46:16 +0800 Subject: [PATCH 03/22] aarch64/mmu: mmu enabled --- crate/aarch64/Cargo.lock | 68 ------ crate/aarch64/src/asm.rs | 27 ++- crate/aarch64/src/paging/mod.rs | 57 ----- crate/aarch64/src/paging/page_table.rs | 9 +- crate/aarch64/src/paging/recursive.rs | 212 +------------------ kernel/Cargo.lock | 17 ++ kernel/Cargo.toml | 1 + kernel/src/arch/aarch64/boot/boot.S | 2 + kernel/src/arch/aarch64/interrupt/handler.rs | 5 +- kernel/src/arch/aarch64/memory.rs | 55 ++--- kernel/src/arch/aarch64/paging.rs | 163 +++++++++++--- kernel/src/consts.rs | 17 +- kernel/src/lib.rs | 4 + kernel/src/memory.rs | 6 +- 14 files changed, 215 insertions(+), 428 deletions(-) delete mode 100644 crate/aarch64/Cargo.lock diff --git a/crate/aarch64/Cargo.lock b/crate/aarch64/Cargo.lock deleted file mode 100644 index 2d0a329..0000000 --- a/crate/aarch64/Cargo.lock +++ /dev/null @@ -1,68 +0,0 @@ -[[package]] -name = "aarch64" -version = "0.1.0" -dependencies = [ - "bit_field 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", - "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", - "cortex-a 2.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "os_bootinfo 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "register 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "usize_conversions 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "ux 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "bit_field" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "bitflags" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "cortex-a" -version = "2.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "register 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "os_bootinfo" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "register" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "tock-registers 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "tock-registers" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "usize_conversions" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "ux" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[metadata] -"checksum bit_field 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ed8765909f9009617974ab6b7d332625b320b33c326b1e9321382ef1999b5d56" -"checksum bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "228047a76f468627ca71776ecdebd732a3423081fcf5125585bcd7c49886ce12" -"checksum cortex-a 2.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "2b187d0d728b4a99ba1d79f9671b976bcdd71a8a2c719585218fd2dc14a4d08c" -"checksum os_bootinfo 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "66481dbeb5e773e7bd85b63cd6042c30786f834338288c5ec4f3742673db360a" -"checksum register 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e10f31b6d2299e5620986ad9fcdd66463e125ad72af4f403f9aedf7592d5ccdb" -"checksum tock-registers 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3a385d94f3f62e60445a0adb9ff8d9621faa272234530d4c0f848ec98f88e316" -"checksum usize_conversions 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f70329e2cbe45d6c97a5112daad40c34cd9a4e18edb5a2a18fefeb584d8d25e5" -"checksum ux 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "53d8df5dd8d07fedccd202de1887d94481fadaea3db70479f459e8163a1fab41" diff --git a/crate/aarch64/src/asm.rs b/crate/aarch64/src/asm.rs index d97fa78..17e6c29 100644 --- a/crate/aarch64/src/asm.rs +++ b/crate/aarch64/src/asm.rs @@ -4,12 +4,13 @@ use regs::*; #[inline(always)] pub fn tlb_invalidate() { - unsafe{ - asm!("dsb ishst - tlbi vmalle1is - dsb ish - tlbi vmalle1is - isb"); + unsafe { + asm!( + "dsb ishst + tlbi vmalle1is + dsb ish + isb" + ); } } @@ -63,6 +64,15 @@ pub unsafe fn get_ttbr1() -> usize { ttbr0 } +#[inline(always)] +pub fn address_translate(vaddr: usize) -> usize { + let paddr: usize; + unsafe { + asm!("at S1E1R, $1; mrs $0, par_el1" : "=r"(paddr) : "r"(vaddr)); + } + paddr +} + /// Returns the SPSel value. #[inline(always)] pub fn sp_sel() -> u8 { @@ -94,7 +104,6 @@ pub fn wfi() { } } - /// The classic no-op #[inline] pub fn nop() { @@ -142,7 +151,7 @@ pub fn eret() -> ! { bitflags! { /// Controls cache settings for the level 4 page table. pub struct ttbr0_el1_Flags: u64 { - + const COMMON_NOT_PRIVATE = 1 << 0; } } @@ -150,7 +159,7 @@ bitflags! { pub fn ttbr0_el1_read() -> (PhysFrame, ttbr0_el1_Flags) { let value = TTBR0_EL1.get(); let flags = ttbr0_el1_Flags::from_bits_truncate(value); - let addr = PhysAddr::new(value & 0x_000f_ffff_ffff_f000); + let addr = PhysAddr::new(value & 0x_0000_ffff_ffff_f000); let frame = PhysFrame::containing_address(addr); (frame, flags) } diff --git a/crate/aarch64/src/paging/mod.rs b/crate/aarch64/src/paging/mod.rs index 56073b2..ebc00dc 100644 --- a/crate/aarch64/src/paging/mod.rs +++ b/crate/aarch64/src/paging/mod.rs @@ -34,16 +34,6 @@ pub trait NotGiantPageSize: PageSize {} #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub enum Size4KiB {} -/// A “huge” 2MiB page. -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] -pub enum Size2MiB {} - -/// A “giant” 1GiB page. -/// -/// (Only available on newer x86_64 CPUs.) -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] -pub enum Size1GiB {} - impl PageSize for Size4KiB { const SIZE: u64 = 4096; const SIZE_AS_DEBUG_STR: &'static str = "4KiB"; @@ -51,18 +41,6 @@ impl PageSize for Size4KiB { impl NotGiantPageSize for Size4KiB {} -impl PageSize for Size2MiB { - const SIZE: u64 = Size4KiB::SIZE * 512; - const SIZE_AS_DEBUG_STR: &'static str = "2MiB"; -} - -impl NotGiantPageSize for Size2MiB {} - -impl PageSize for Size1GiB { - const SIZE: u64 = Size2MiB::SIZE * 512; - const SIZE_AS_DEBUG_STR: &'static str = "1GiB"; -} - /// A virtual memory page. #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] #[repr(C)] @@ -128,31 +106,6 @@ impl Page { } } -impl Page { - /// Returns the 1GiB memory page with the specified page table indices. - pub fn from_page_table_indices_1gib(p4_index: u9, p3_index: u9) -> Self { - use bit_field::BitField; - - let mut addr = 0; - addr.set_bits(39..48, u64::from(p4_index)); - addr.set_bits(30..39, u64::from(p3_index)); - Page::containing_address(VirtAddr::new(addr)) - } -} - -impl Page { - /// Returns the 2MiB memory page with the specified page table indices. - pub fn from_page_table_indices_2mib(p4_index: u9, p3_index: u9, p2_index: u9) -> Self { - use bit_field::BitField; - - let mut addr = 0; - addr.set_bits(39..48, u64::from(p4_index)); - addr.set_bits(30..39, u64::from(p3_index)); - addr.set_bits(21..30, u64::from(p2_index)); - Page::containing_address(VirtAddr::new(addr)) - } -} - impl Page { /// Returns the 4KiB memory page with the specified page table indices. pub fn from_page_table_indices(p4_index: u9, p3_index: u9, p2_index: u9, p1_index: u9) -> Self { @@ -246,16 +199,6 @@ impl Iterator for PageRange { } } -impl PageRange { - /// Converts the range of 2MiB pages to a range of 4KiB pages. - pub fn as_4kib_page_range(self) -> PageRange { - PageRange { - start: Page::containing_address(self.start.start_address()), - end: Page::containing_address(self.end.start_address()), - } - } -} - impl fmt::Debug for PageRange { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { f.debug_struct("PageRange") diff --git a/crate/aarch64/src/paging/page_table.rs b/crate/aarch64/src/paging/page_table.rs index 1525b17..c06123e 100644 --- a/crate/aarch64/src/paging/page_table.rs +++ b/crate/aarch64/src/paging/page_table.rs @@ -21,7 +21,7 @@ pub enum FrameError { #[derive(Clone)] #[repr(transparent)] pub struct PageTableEntry { - entry: u64, + pub entry: u64, } impl PageTableEntry { @@ -42,7 +42,7 @@ impl PageTableEntry { /// Returns the physical address mapped by this entry, might be zero. pub fn addr(&self) -> PhysAddr { - PhysAddr::new(self.entry & 0x000fffff_fffff000) + PhysAddr::new(self.entry & 0x0000_ffff_ffff_f000) } /// Returns the physical frame mapped by this entry. @@ -83,6 +83,7 @@ impl PageTableEntry { impl fmt::Debug for PageTableEntry { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { let mut f = f.debug_struct("PageTableEntry"); + f.field("value", &self.entry); f.field("addr", &self.addr()); f.field("flags", &self.flags()); f.finish() @@ -94,9 +95,11 @@ bitflags! { pub struct PageTableFlags: u64 { const ALL = 0xffffffff_ffffffff; const TYPE_MASK = 3 << 0; - const TYPE_FAULT = 0 << 0; + // const TYPE_FAULT = 0 << 0; const TYPE_PAGE = 3 << 0; const TABLE_BIT = 1 << 1; + // const BLOCK_BIT = 0 << 1; + const PAGE_BIT = 1 << 1; const PRESENT = 1 << 0; const USER_ACCESSIBLE = 1 << 6; /* AP[1] */ diff --git a/crate/aarch64/src/paging/recursive.rs b/crate/aarch64/src/paging/recursive.rs index 0cbc6d0..d735216 100644 --- a/crate/aarch64/src/paging/recursive.rs +++ b/crate/aarch64/src/paging/recursive.rs @@ -4,7 +4,7 @@ use asm::tlb_invalidate; use paging::{ frame_alloc::FrameAllocator, page_table::{FrameError, PageTable, PageTableEntry, PageTableFlags}, - NotGiantPageSize, Page, PageSize, PhysFrame, Size1GiB, Size2MiB, Size4KiB, + NotGiantPageSize, Page, PageSize, PhysFrame, Size4KiB, }; use paging::page_table::PageTableFlags as Flags; use asm::ttbr0_el1_read; @@ -217,7 +217,7 @@ impl<'a> RecursivePageTable<'a> { if entry.is_unused() { if let Some(frame) = allocator.alloc() { - entry.set_frame(frame, Flags::PRESENT | Flags::WRITE); + entry.set_frame(frame, Flags::PRESENT | Flags::WRITE | Flags::ACCESSED | Flags::PAGE_BIT); created = true; } else { return Err(MapToError::FrameAllocationFailed); @@ -241,214 +241,6 @@ impl<'a> RecursivePageTable<'a> { } } -impl<'a> Mapper for RecursivePageTable<'a> { - fn map_to( - &mut self, - page: Page, - frame: PhysFrame, - flags: PageTableFlags, - allocator: &mut A, - ) -> Result, MapToError> - where - A: FrameAllocator, - { - let p4 = &mut self.p4; - - let p3_page = p3_page(page, self.recursive_index); - let p3 = unsafe { Self::create_next_table(&mut p4[page.p4_index()], p3_page, allocator)? }; - - if !p3[page.p3_index()].is_unused() { - return Err(MapToError::PageAlreadyMapped); - } - p3[page.p3_index()].set_addr(frame.start_address(), flags | Flags::HUGE_PAGE); - - Ok(MapperFlush::new(page)) - } - - fn unmap( - &mut self, - page: Page, - ) -> Result<(PhysFrame, MapperFlush), UnmapError> { - let p4 = &mut self.p4; - let p4_entry = &p4[page.p4_index()]; - - p4_entry.frame().map_err(|err| match err { - FrameError::FrameNotPresent => UnmapError::PageNotMapped, - FrameError::HugeFrame => UnmapError::ParentEntryHugePage, - })?; - - let p3 = unsafe { &mut *(p3_ptr(page, self.recursive_index)) }; - let p3_entry = &mut p3[page.p3_index()]; - let flags = p3_entry.flags(); - - if !flags.contains(PageTableFlags::PRESENT) { - return Err(UnmapError::PageNotMapped); - } - if !flags.contains(PageTableFlags::HUGE_PAGE) { - return Err(UnmapError::ParentEntryHugePage); - } - - let frame = PhysFrame::from_start_address(p3_entry.addr()) - .map_err(|()| UnmapError::InvalidFrameAddress(p3_entry.addr()))?; - - p3_entry.set_unused(); - Ok((frame, MapperFlush::new(page))) - } - - fn update_flags( - &mut self, - page: Page, - flags: PageTableFlags, - ) -> Result, FlagUpdateError> { - let p4 = &mut self.p4; - - if p4[page.p4_index()].is_unused() { - return Err(FlagUpdateError::PageNotMapped); - } - - let p3 = unsafe { &mut *(p3_ptr(page, self.recursive_index)) }; - - if p3[page.p3_index()].is_unused() { - return Err(FlagUpdateError::PageNotMapped); - } - p3[page.p3_index()].set_flags(flags | Flags::HUGE_PAGE); - - Ok(MapperFlush::new(page)) - } - - fn translate_page(&self, page: Page) -> Option> { - let p4 = &self.p4; - - if p4[page.p4_index()].is_unused() { - return None; - } - - let p3 = unsafe { &*(p3_ptr(page, self.recursive_index)) }; - let p3_entry = &p3[page.p3_index()]; - - if p3_entry.is_unused() { - return None; - } - - PhysFrame::from_start_address(p3_entry.addr()).ok() - } -} - -impl<'a> Mapper for RecursivePageTable<'a> { - fn map_to( - &mut self, - page: Page, - frame: PhysFrame, - flags: PageTableFlags, - allocator: &mut A, - ) -> Result, MapToError> - where - A: FrameAllocator, - { - let p4 = &mut self.p4; - - let p3_page = p3_page(page, self.recursive_index); - let p3 = unsafe { Self::create_next_table(&mut p4[page.p4_index()], p3_page, allocator)? }; - - let p2_page = p2_page(page, self.recursive_index); - let p2 = unsafe { Self::create_next_table(&mut p3[page.p3_index()], p2_page, allocator)? }; - - if !p2[page.p2_index()].is_unused() { - return Err(MapToError::PageAlreadyMapped); - } - p2[page.p2_index()].set_addr(frame.start_address(), flags | Flags::HUGE_PAGE); - - Ok(MapperFlush::new(page)) - } - - fn unmap( - &mut self, - page: Page, - ) -> Result<(PhysFrame, MapperFlush), UnmapError> { - let p4 = &mut self.p4; - let p4_entry = &p4[page.p4_index()]; - p4_entry.frame().map_err(|err| match err { - FrameError::FrameNotPresent => UnmapError::PageNotMapped, - FrameError::HugeFrame => UnmapError::ParentEntryHugePage, - })?; - - let p3 = unsafe { &mut *(p3_ptr(page, self.recursive_index)) }; - let p3_entry = &p3[page.p3_index()]; - p3_entry.frame().map_err(|err| match err { - FrameError::FrameNotPresent => UnmapError::PageNotMapped, - FrameError::HugeFrame => UnmapError::ParentEntryHugePage, - })?; - - let p2 = unsafe { &mut *(p2_ptr(page, self.recursive_index)) }; - let p2_entry = &mut p2[page.p2_index()]; - let flags = p2_entry.flags(); - - if !flags.contains(PageTableFlags::PRESENT) { - return Err(UnmapError::PageNotMapped); - } - if !flags.contains(PageTableFlags::HUGE_PAGE) { - return Err(UnmapError::ParentEntryHugePage); - } - - let frame = PhysFrame::from_start_address(p2_entry.addr()) - .map_err(|()| UnmapError::InvalidFrameAddress(p2_entry.addr()))?; - - p2_entry.set_unused(); - Ok((frame, MapperFlush::new(page))) - } - - fn update_flags( - &mut self, - page: Page, - flags: PageTableFlags, - ) -> Result, FlagUpdateError> { - let p4 = &mut self.p4; - - if p4[page.p4_index()].is_unused() { - return Err(FlagUpdateError::PageNotMapped); - } - - let p3 = unsafe { &mut *(p3_ptr(page, self.recursive_index)) }; - - if p3[page.p3_index()].is_unused() { - return Err(FlagUpdateError::PageNotMapped); - } - - let p2 = unsafe { &mut *(p2_ptr(page, self.recursive_index)) }; - - if p2[page.p2_index()].is_unused() { - return Err(FlagUpdateError::PageNotMapped); - } - - p2[page.p2_index()].set_flags(flags | Flags::HUGE_PAGE); - - Ok(MapperFlush::new(page)) - } - - fn translate_page(&self, page: Page) -> Option> { - let p4 = &self.p4; - - if p4[page.p4_index()].is_unused() { - return None; - } - - let p3 = unsafe { &*(p3_ptr(page, self.recursive_index)) }; - let p3_entry = &p3[page.p3_index()]; - - if p3_entry.is_unused() { - return None; - } - - let p2 = unsafe { &*(p2_ptr(page, self.recursive_index)) }; - let p2_entry = &p2[page.p2_index()]; - - if p2_entry.is_unused() { - return None; - } - - PhysFrame::from_start_address(p2_entry.addr()).ok() - } -} impl<'a> Mapper for RecursivePageTable<'a> { fn map_to( diff --git a/kernel/Cargo.lock b/kernel/Cargo.lock index 05f0710..7795f1f 100644 --- a/kernel/Cargo.lock +++ b/kernel/Cargo.lock @@ -5,6 +5,7 @@ dependencies = [ "bare-metal 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", "bit_field 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", "os_bootinfo 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "register 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "usize_conversions 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -176,6 +177,14 @@ name = "redox_syscall" version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "register" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "tock-registers 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "register" version = "0.2.1" @@ -238,6 +247,11 @@ dependencies = [ "remove_dir_all 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "tock-registers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "tock-registers" version = "0.2.0" @@ -271,6 +285,7 @@ dependencies = [ "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", "once 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", + "register 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "riscv 0.3.0", "simple-filesystem 0.0.1 (git+https://github.com/wangrunji0408/SimpleFileSystem-Rust)", "spin 0.4.9 (registry+https://github.com/rust-lang/crates.io-index)", @@ -384,6 +399,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum pulldown-cmark 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "8361e81576d2e02643b04950e487ec172b687180da65c731c03cf336784e6c07" "checksum rand 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "8356f47b32624fef5b3301c1be97e5944ecdd595409cc5da11d05f211db6cfbd" "checksum redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "c214e91d3ecf43e9a4e41e578973adeb14b474f2bee858742d127af75a0112b1" +"checksum register 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0e985243ba7e1c336b62444ef2a10d7bd87cf41a222285ae3de605c859006479" "checksum register 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e10f31b6d2299e5620986ad9fcdd66463e125ad72af4f403f9aedf7592d5ccdb" "checksum remove_dir_all 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3488ba1b9a2084d38645c4c08276a1752dcbf2c7130d74f1569681ad5d2799c5" "checksum simple-filesystem 0.0.1 (git+https://github.com/wangrunji0408/SimpleFileSystem-Rust)" = "" @@ -391,6 +407,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum spin 0.4.9 (registry+https://github.com/rust-lang/crates.io-index)" = "37b5646825922b96b5d7d676b5bb3458a54498e96ed7b0ce09dc43a07038fea4" "checksum static_assertions 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "c19be23126415861cb3a23e501d34a708f7f9b2183c5252d690941c2e69199d5" "checksum tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" +"checksum tock-registers 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2acc33f980e23cee18d234a32d0637fbc1ea55e13ab04012fa857b899fa1b7a9" "checksum tock-registers 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3a385d94f3f62e60445a0adb9ff8d9621faa272234530d4c0f848ec98f88e316" "checksum uart_16550 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "269f953d8de3226f7c065c589c7b4a3e83d10a419c7c3b5e2e0f197e6acc966e" "checksum unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "882386231c45df4700b275c7ff55b6f3698780a650026380e72dabe76fa46526" diff --git a/kernel/Cargo.toml b/kernel/Cargo.toml index c80b6ef..06ea088 100644 --- a/kernel/Cargo.toml +++ b/kernel/Cargo.toml @@ -32,6 +32,7 @@ bit-allocator = { path = "../crate/bit-allocator" } ucore-memory = { path = "../crate/memory" } ucore-process = { path = "../crate/process" } simple-filesystem = { git = "https://github.com/wangrunji0408/SimpleFileSystem-Rust" } +register="0.1.0" [target.'cfg(target_arch = "x86_64")'.dependencies] bootloader = "0.3" diff --git a/kernel/src/arch/aarch64/boot/boot.S b/kernel/src/arch/aarch64/boot/boot.S index b8a142e..57a4239 100644 --- a/kernel/src/arch/aarch64/boot/boot.S +++ b/kernel/src/arch/aarch64/boot/boot.S @@ -17,6 +17,8 @@ halt: setup: # store the desired EL1 stack pointer in x1 adr x1, _start + # FIXME + lsl x1, x1, #2 # use SP_ELx for Exception level ELx msr SPsel, #1 diff --git a/kernel/src/arch/aarch64/interrupt/handler.rs b/kernel/src/arch/aarch64/interrupt/handler.rs index 85d1faf..787cbf5 100644 --- a/kernel/src/arch/aarch64/interrupt/handler.rs +++ b/kernel/src/arch/aarch64/interrupt/handler.rs @@ -38,10 +38,11 @@ pub struct Info { /// the trap frame for the exception. #[no_mangle] pub extern "C" fn rust_trap(info: Info, esr: u32, tf: &mut TrapFrame) { - let syndrome = Syndrome::from(esr); - trace!("Interrupt: {:?} from: {:?}", syndrome, info); + trace!("Interrupt: {:?}, ELR: {:#x?}", info, tf.elr); match info.kind { Kind::Synchronous => { + let syndrome = Syndrome::from(esr); + trace!("ESR: {:#x?}, Syndrome: {:?}", esr, syndrome); // syndrome is only valid with sync match syndrome { Syndrome::Brk(brk) => handle_break(brk, tf), diff --git a/kernel/src/arch/aarch64/memory.rs b/kernel/src/arch/aarch64/memory.rs index 126cf57..3e7f6d8 100644 --- a/kernel/src/arch/aarch64/memory.rs +++ b/kernel/src/arch/aarch64/memory.rs @@ -11,64 +11,55 @@ use core::ops::Range; /// Memory initialization. pub fn init() { - /*let (start, end) = memory_map().expect("failed to find memory map"); - unsafe { - HEAP_ALLOCATOR.lock().init(start, end - start); - }*/ - - - #[repr(align(4096))] struct PageData([u8; PAGE_SIZE]); - static PAGE_TABLE_ROOT: PageData = PageData([0; PAGE_SIZE]); + static PAGE_TABLE_LVL4: PageData = PageData([0; PAGE_SIZE]); + static PAGE_TABLE_LVL3: PageData = PageData([0; PAGE_SIZE]); + static PAGE_TABLE_LVL2: PageData = PageData([0; PAGE_SIZE]); - let frame = Frame::containing_address(PhysAddr::new(&PAGE_TABLE_ROOT as *const _ as u64)); - super::paging::setup_page_table(frame); + let frame_lvl4 = Frame::containing_address(PhysAddr::new(&PAGE_TABLE_LVL4 as *const _ as u64)); + let frame_lvl3 = Frame::containing_address(PhysAddr::new(&PAGE_TABLE_LVL3 as *const _ as u64)); + let frame_lvl2 = Frame::containing_address(PhysAddr::new(&PAGE_TABLE_LVL2 as *const _ as u64)); + super::paging::setup_page_table(frame_lvl4, frame_lvl3, frame_lvl2); init_mmu(); - init_frame_allocator(); init_heap(); - let (start, end) = memory_map().expect("failed to find memory map"); - let mut v = vec![]; - for i in 0..(20 + (start & 0xf)) { - v.push(i); - println!("{:x?} {:x?}", &v[i] as * const _ as usize, v); - } - + info!("memory: init end"); } extern "C" { static _end: u8; } - fn init_frame_allocator() { + use bit_allocator::BitAlloc; + use core::ops::Range; use consts::{MEMORY_OFFSET}; - let (start, end) = memory_map().expect("failed to find memory map"); - info!("{:x?} {:x?}", start, end); + let (start, end) = memory_map().expect("failed to find memory map"); let mut ba = FRAME_ALLOCATOR.lock(); - use core::mem::size_of; - use ::memory::FrameAlloc; - info!("{:x?} {:x?}", &FRAME_ALLOCATOR as *const _ as usize, size_of::()); - use consts::{KERNEL_HEAP_OFFSET, KERNEL_HEAP_SIZE}; - info!("{:x?} {:x?}", KERNEL_HEAP_OFFSET + KERNEL_HEAP_SIZE, end); ba.insert(to_range(start, end)); info!("FrameAllocator init end"); + /* + * @param: + * start: start address + * end: end address + * @brief: + * transform the memory address to the page number + * @retval: + * the page number range from start address to end address + */ fn to_range(start: usize, end: usize) -> Range { let page_start = (start - MEMORY_OFFSET) / PAGE_SIZE; let page_end = (end - MEMORY_OFFSET - 1) / PAGE_SIZE + 1; - // info!("{:x?} {:x?}", page_start, page_end); page_start..page_end } } fn init_mmu() { - info!("init_mmu"); - // device. MAIR_EL1.write( // Attribute 1 @@ -80,7 +71,6 @@ fn init_mmu() { ); // Configure various settings of stage 1 of the EL1 translation regime. let ips = ID_AA64MMFR0_EL1.read(ID_AA64MMFR0_EL1::PARange); - info!("{:x?}", ips); TCR_EL1.write( TCR_EL1::TBI0::Ignored + TCR_EL1::IPS.val(ips) @@ -97,21 +87,20 @@ fn init_mmu() { // First, force all previous changes to be seen before the MMU is enabled. unsafe { barrier::isb(barrier::SY); } - info!("{:x?}", TCR_EL1.get()); // Enable the MMU and turn on data and instruction caching. SCTLR_EL1.modify(SCTLR_EL1::M::Enable + SCTLR_EL1::C::Cacheable + SCTLR_EL1::I::Cacheable); // Force MMU init to complete before next instruction unsafe { barrier::isb(barrier::SY); } - info!("mmu enabled!"); + info!("mmu enabled"); } /// Returns the (start address, end address) of the available memory on this /// system if it can be determined. If it cannot, `None` is returned. /// /// This function is expected to return `Some` under all normal cirumstances. -pub fn memory_map() -> Option<(usize, usize)> { +fn memory_map() -> Option<(usize, usize)> { let binary_end = unsafe { (&_end as *const u8) as u32 }; let mut atags: Atags = Atags::get(); diff --git a/kernel/src/arch/aarch64/paging.rs b/kernel/src/arch/aarch64/paging.rs index 603346d..9696db9 100644 --- a/kernel/src/arch/aarch64/paging.rs +++ b/kernel/src/arch/aarch64/paging.rs @@ -1,6 +1,6 @@ //! Page table implementations for aarch64. -use bit_allocator::{BitAlloc}; // Depends on kernel +use consts::{KERNEL_PML4, RECURSIVE_INDEX}; use memory::{active_table, alloc_frame, alloc_stack, dealloc_frame}; use ucore_memory::memory_set::*; use ucore_memory::PAGE_SIZE; @@ -11,29 +11,121 @@ use aarch64::paging::{Mapper, PageTable as Aarch64PageTable, PageTableEntry, Pag use aarch64::paging::{FrameAllocator, FrameDeallocator, Page, PageRange, PhysFrame as Frame, Size4KiB}; use aarch64::{regs::*}; -// need 1 page -pub fn setup_page_table(frame: Frame) { - let p4 = unsafe { &mut *(frame.start_address().as_u64() as *mut Aarch64PageTable) }; - p4.zero(); - - - // p4.set_recursive(RECURSIVE_PAGE_PML4, frame.clone()); +register_bitfields! {u64, + // AArch64 Reference Manual page 2150 + STAGE1_DESCRIPTOR [ + /// Execute-never + XN OFFSET(54) NUMBITS(1) [ + False = 0, + True = 1 + ], + + /// Various address fields, depending on use case + LVL4_OUTPUT_ADDR_4KiB OFFSET(39) NUMBITS(9) [], // [47:39] + LVL3_OUTPUT_ADDR_4KiB OFFSET(30) NUMBITS(18) [], // [47:30] + LVL2_OUTPUT_ADDR_4KiB OFFSET(21) NUMBITS(27) [], // [47:21] + NEXT_LVL_TABLE_ADDR_4KiB OFFSET(12) NUMBITS(36) [], // [47:12] + + /// Access flag + AF OFFSET(10) NUMBITS(1) [ + False = 0, + True = 1 + ], + + /// Shareability field + SH OFFSET(8) NUMBITS(2) [ + OuterShareable = 0b10, + InnerShareable = 0b11 + ], + + /// Access Permissions + AP OFFSET(6) NUMBITS(2) [ + RW_EL1 = 0b00, + RW_EL1_EL0 = 0b01, + RO_EL1 = 0b10, + RO_EL1_EL0 = 0b11 + ], + + /// Memory attributes index into the MAIR_EL1 register + AttrIndx OFFSET(2) NUMBITS(3) [], + + TYPE OFFSET(1) NUMBITS(1) [ + Block = 0, + Table = 1 + ], + + VALID OFFSET(0) NUMBITS(1) [ + False = 0, + True = 1 + ] + ] +} - // Set kernel identity map - // 0x10000000 ~ 1K area - p4.map_identity(0o777, EF::PRESENT | EF::PXN | EF::UXN); +// need 3 page +pub fn setup_page_table(frame_lvl4: Frame, frame_lvl3: Frame, frame_lvl2: Frame) { + let p4 = unsafe { &mut *(frame_lvl4.start_address().as_u64() as *mut Aarch64PageTable) }; + let p3 = unsafe { &mut *(frame_lvl3.start_address().as_u64() as *mut Aarch64PageTable) }; + let p2 = unsafe { &mut *(frame_lvl2.start_address().as_u64() as *mut Aarch64PageTable) }; + p4.zero(); + p3.zero(); + p2.zero(); - // 0x80000000 ~ 8K area - p4.map_identity(0, EF::PRESENT); - // p2.map_identity(KERNEL_PML4, EF::PRESENT | EF::READABLE | EF::WRITABLE); - // p2.map_identity(KERNEL_PML4 + 1, EF::VALID | EF::READABLE | EF::WRITABLE | EF::EXECUTABLE); + mod mair { + pub const NORMAL: u64 = 0; + pub const DEVICE: u64 = 1; + } - // use super::riscv::register::satp; - // unsafe { satp::set(satp::Mode::Sv32, 0, frame); } - // sfence_vma_all(); + // Fill the rest of the LVL2 (2MiB) entries as block + // descriptors. Differentiate between normal and device mem. + const MMIO_BASE: u64 = 0x3F000000; + let mmio_base: u64 = MMIO_BASE >> 21; + let mut common = STAGE1_DESCRIPTOR::VALID::True + + STAGE1_DESCRIPTOR::TYPE::Block + + STAGE1_DESCRIPTOR::AP::RW_EL1 + + STAGE1_DESCRIPTOR::AF::True; + // + STAGE1_DESCRIPTOR::XN::True; + + for i in 0..512 { + let j: u64 = i as u64; + + let mem_attr = if j >= mmio_base { + STAGE1_DESCRIPTOR::SH::OuterShareable + STAGE1_DESCRIPTOR::AttrIndx.val(mair::DEVICE) + } else { + STAGE1_DESCRIPTOR::SH::InnerShareable + STAGE1_DESCRIPTOR::AttrIndx.val(mair::NORMAL) + }; + + p2[i].entry = (common + mem_attr + STAGE1_DESCRIPTOR::LVL2_OUTPUT_ADDR_4KiB.val(j)).value; + } - ttbr0_el1_write(frame); + common = common + STAGE1_DESCRIPTOR::SH::InnerShareable + STAGE1_DESCRIPTOR::AttrIndx.val(mair::NORMAL); + + p3[0].entry = (common + STAGE1_DESCRIPTOR::TYPE::Table + STAGE1_DESCRIPTOR::NEXT_LVL_TABLE_ADDR_4KiB.val(frame_lvl2.start_address().as_u64() >> 12)).value; + p3[1].entry = (common + STAGE1_DESCRIPTOR::LVL3_OUTPUT_ADDR_4KiB.val(1)).value; + p4[0].entry = (common + STAGE1_DESCRIPTOR::TYPE::Table + STAGE1_DESCRIPTOR::NEXT_LVL_TABLE_ADDR_4KiB.val(frame_lvl3.start_address().as_u64() >> 12)).value; + p4[RECURSIVE_INDEX].entry = (common + STAGE1_DESCRIPTOR::TYPE::Table + STAGE1_DESCRIPTOR::NEXT_LVL_TABLE_ADDR_4KiB.val(frame_lvl4.start_address().as_u64() >> 12)).value; + + // warn!("p2"); + // for i in 0..512 { + // if p2[i].flags().bits() != 0 { + // info!("{:x?} {:x?} {:x?}",i, &p2[i] as *const _ as usize, p2[i]); + // } + // } + // warn!("p3"); + // for i in 0..512 { + // if p3[i].flags().bits() != 0 { + // info!("{:x?} {:x?} {:x?}",i, &p3[i] as *const _ as usize, p3[i]); + // } + // } + // warn!("p4"); + // for i in 0..512 { + // if p4[i].flags().bits() != 0 { + // info!("{:x?} {:x?} {:x?}",i, &p4[i] as *const _ as usize, p4[i]); + // } + // } + + ttbr0_el1_write(frame_lvl4); tlb_invalidate(); + info!("setup init page table end"); } @@ -69,7 +161,7 @@ impl PageTable for ActivePageTable { type Entry = PageEntry; fn map(&mut self, addr: usize, target: usize) -> &mut PageEntry { - let flags = EF::PRESENT | EF::WRITE | EF::UXN; + let flags = EF::PRESENT | EF::WRITE | EF::ACCESSED | EF::UXN | EF::PAGE_BIT; self.0.map_to(Page::of_addr(addr), Frame::of_addr(target), flags, &mut FrameAllocatorForAarch64) .unwrap().flush(); self.get_entry(addr) @@ -81,7 +173,7 @@ impl PageTable for ActivePageTable { } fn get_entry(&mut self, addr: usize) -> &mut PageEntry { - let entry_addr = ((addr >> 9) & 0o777_777_777_7770) | 0xffffff80_00000000; + let entry_addr = ((addr >> 9) & 0o777_777_777_7770) | (RECURSIVE_INDEX << 39); unsafe { &mut *(entry_addr as *mut PageEntry) } } @@ -99,9 +191,12 @@ impl PageTable for ActivePageTable { } } +const ROOT_PAGE_TABLE: *mut Aarch64PageTable = + ((RECURSIVE_INDEX << 39) | (RECURSIVE_INDEX << 30) | (RECURSIVE_INDEX << 21) | (RECURSIVE_INDEX << 12)) as *mut Aarch64PageTable; + impl ActivePageTable { pub unsafe fn new() -> Self { - ActivePageTable(RecursivePageTable::new(&mut *(0xffff_ffff_ffff_f000 as *mut _)).unwrap()) + ActivePageTable(RecursivePageTable::new(&mut *(ROOT_PAGE_TABLE as *mut _)).unwrap()) } fn with_temporary_map(&mut self, frame: &Frame, f: impl FnOnce(&mut ActivePageTable, &mut Aarch64PageTable)) { // Create a temporary page @@ -161,7 +256,7 @@ impl Entry for PageEntry { let mut addr = self as *const _ as usize; for _ in 0..3 { // Upper level entry - addr = ((addr >> 9) & 0o777_777_777_7770) | 0xffffff80_00000000; + addr = ((addr >> 9) & 0o777_777_777_7770) | (RECURSIVE_INDEX << 39); // set USER_ACCESSIBLE unsafe { (*(addr as *mut EF)).insert(EF::USER_ACCESSIBLE) }; } @@ -200,24 +295,24 @@ impl InactivePageTable for InactivePageTable0 { active_table().with_temporary_map(&frame, |_, table: &mut Aarch64PageTable| { table.zero(); // set up recursive mapping for the table - table[511].set_frame(frame.clone(), EF::PRESENT | EF::WRITE); + table[RECURSIVE_INDEX].set_frame(frame.clone(), EF::PRESENT | EF::WRITE | EF::ACCESSED | EF::PAGE_BIT); }); InactivePageTable0 { p4_frame: frame } } fn edit(&mut self, f: impl FnOnce(&mut Self::Active)) { active_table().with_temporary_map(&ttbr0_el1_read().0, |active_table, p4_table: &mut Aarch64PageTable| { - let backup = p4_table[0o777].clone(); + let backup = p4_table[RECURSIVE_INDEX].clone(); // overwrite recursive mapping - p4_table[0o777].set_frame(self.p4_frame.clone(), EF::PRESENT | EF::WRITE); + p4_table[RECURSIVE_INDEX].set_frame(self.p4_frame.clone(), EF::PRESENT | EF::WRITE | EF::ACCESSED | EF::PAGE_BIT); tlb_invalidate(); // execute f in the new context f(active_table); // restore recursive mapping to original p4 table - p4_table[0o777] = backup; + p4_table[RECURSIVE_INDEX] = backup; tlb_invalidate(); }); } @@ -228,6 +323,7 @@ impl InactivePageTable for InactivePageTable0 { debug!("switch table {:?} -> {:?}", old_frame, new_frame); if old_frame != new_frame { ttbr0_el1_write(new_frame); + tlb_invalidate(); } } @@ -237,11 +333,13 @@ impl InactivePageTable for InactivePageTable0 { debug!("switch table {:?} -> {:?}", old_frame, new_frame); if old_frame != new_frame { ttbr0_el1_write(new_frame); + tlb_invalidate(); } f(); debug!("switch table {:?} -> {:?}", new_frame, old_frame); if old_frame != new_frame { ttbr0_el1_write(old_frame); + tlb_invalidate(); } } @@ -264,13 +362,12 @@ impl InactivePageTable for InactivePageTable0 { impl InactivePageTable0 { fn map_kernel(&mut self) { - let mut table = unsafe { &mut *(0xffff_ffff_ffff_f000 as *mut Aarch64PageTable) }; - // Kernel at 0xffff_ff00_0000_0000 - // Kernel stack at 0x0000_57ac_0000_0000 (defined in bootloader crate) - let e0 = table[0].clone(); + let table = unsafe { &mut *ROOT_PAGE_TABLE }; + let e0 = table[KERNEL_PML4].clone(); + assert!(!e0.is_unused()); + self.edit(|_| { - table[0].set_addr(e0.addr(), e0.flags() & EF::GLOBAL); - //table[175].set_addr(estack.addr(), estack.flags() & EF::GLOBAL); + table[KERNEL_PML4].set_addr(e0.addr(), e0.flags() & EF::GLOBAL); }); } } diff --git a/kernel/src/consts.rs b/kernel/src/consts.rs index b2cdefe..e879173 100644 --- a/kernel/src/consts.rs +++ b/kernel/src/consts.rs @@ -131,15 +131,12 @@ mod x86_64 { #[cfg(target_arch = "aarch64")] mod aarch64 { - //! TODO: replace unmiplemented consts with real value - const UNIMPLEMENTED: usize = 0; - pub const KERNEL_OFFSET: usize = UNIMPLEMENTED; - pub const KERNEL_PML4: usize = UNIMPLEMENTED; - pub const KERNEL_HEAP_OFFSET: usize = UNIMPLEMENTED; + pub const RECURSIVE_INDEX: usize = 0o777; + pub const KERNEL_OFFSET: usize = 0; + pub const KERNEL_PML4: usize = 0; pub const KERNEL_HEAP_SIZE: usize = 8 * 1024 * 1024; - pub const MEMORY_OFFSET: usize = UNIMPLEMENTED; - pub const MEMORY_END: usize = UNIMPLEMENTED; - pub const USER_STACK_OFFSET: usize = UNIMPLEMENTED; - pub const USER_STACK_SIZE: usize = UNIMPLEMENTED; - pub const USER32_STACK_OFFSET: usize = UNIMPLEMENTED; + pub const MEMORY_OFFSET: usize = 0; + pub const USER_STACK_OFFSET: usize = 0x3000_0000; + pub const USER_STACK_SIZE: usize = 1 * 1024 * 1024; + pub const USER32_STACK_OFFSET: usize = USER_STACK_OFFSET; } diff --git a/kernel/src/lib.rs b/kernel/src/lib.rs index 063b1d0..05cc181 100644 --- a/kernel/src/lib.rs +++ b/kernel/src/lib.rs @@ -38,6 +38,10 @@ extern crate aarch64; use linked_list_allocator::LockedHeap; + +#[macro_use] +extern crate register; + #[macro_use] // print! pub mod logging; mod memory; diff --git a/kernel/src/memory.rs b/kernel/src/memory.rs index bbe78db..100925e 100644 --- a/kernel/src/memory.rs +++ b/kernel/src/memory.rs @@ -1,5 +1,5 @@ pub use arch::paging::*; -use bit_allocator::{BitAlloc, BitAlloc4K, BitAlloc64K}; +use bit_allocator::{BitAlloc, BitAlloc4K, BitAlloc64K, BitAlloc1M}; use consts::MEMORY_OFFSET; use spin::{Mutex, MutexGuard}; use super::HEAP_ALLOCATOR; @@ -19,7 +19,7 @@ pub type FrameAlloc = BitAlloc4K; // Raspberry Pi 3 has 1G memory #[cfg(target_arch = "aarch64")] -pub type FrameAlloc = BitAlloc64K; +pub type FrameAlloc = BitAlloc1M; lazy_static! { pub static ref FRAME_ALLOCATOR: Mutex = Mutex::new(FrameAlloc::default()); @@ -76,4 +76,4 @@ pub fn init_heap() { // use ucore_memory::cow::test::test_with; // test_with(&mut active_table()); // } -//} \ No newline at end of file +//} From f398945ad3e1f04841bca744d8c3a3f50aca7d56 Mon Sep 17 00:00:00 2001 From: equation314 Date: Tue, 20 Nov 2018 18:29:51 +0800 Subject: [PATCH 04/22] aarch64: hard link user program --- kernel/src/fs.rs | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/kernel/src/fs.rs b/kernel/src/fs.rs index 6e4e0a3..6810916 100644 --- a/kernel/src/fs.rs +++ b/kernel/src/fs.rs @@ -9,9 +9,23 @@ use spin::Mutex; global_asm!(r#" .section .rodata .align 12 -_binary_user_riscv_img_start: + .global _user_img_start + .global _user_img_end +_user_img_start: .incbin "../user/user-riscv.img" -_binary_user_riscv_img_end: +_user_img_end: +"#); + +// Hard link user program +#[cfg(target_arch = "aarch64")] +global_asm!(r#" + .section .rodata + .align 12 + .global _user_img_start + .global _user_img_end +_user_img_start: + .incbin "../user/user-riscv.img" +_user_img_end: "#); const LOGO: &str = r#" @@ -85,22 +99,18 @@ pub fn test_shell(prefix: &str) -> ! { pub fn shell() { show_logo(); - #[cfg(target_arch = "riscv32")] + #[cfg(any(target_arch = "riscv32", target_arch = "aarch64"))] let device = { extern { - fn _binary_user_riscv_img_start(); - fn _binary_user_riscv_img_end(); + fn _user_img_start(); + fn _user_img_end(); } - Box::new(unsafe { MemBuf::new(_binary_user_riscv_img_start, _binary_user_riscv_img_end) }) + Box::new(unsafe { MemBuf::new(_user_img_start, _user_img_end) }) }; #[cfg(target_arch = "x86_64")] let device = Box::new(&ide::DISK1); - #[cfg(target_arch = "aarch64")] - // TODO - let device: Box = unimplemented!(); - let sfs = SimpleFileSystem::open(device).expect("failed to open SFS"); let root = sfs.root_inode(); let files = root.borrow().list().unwrap(); From 13be52133da4825c9430f3f40bbd8cc8a7cfe6e2 Mon Sep 17 00:00:00 2001 From: equation314 Date: Tue, 20 Nov 2018 18:32:26 +0800 Subject: [PATCH 05/22] aarch64: move kernel stack top to 0x100000 --- kernel/src/arch/aarch64/boot/boot.S | 14 +++++++++----- kernel/src/arch/aarch64/boot/linker.ld | 9 ++++++--- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/kernel/src/arch/aarch64/boot/boot.S b/kernel/src/arch/aarch64/boot/boot.S index b8a142e..e525340 100644 --- a/kernel/src/arch/aarch64/boot/boot.S +++ b/kernel/src/arch/aarch64/boot/boot.S @@ -2,8 +2,7 @@ .section .text.boot -.global _start -_start: +boot: # read cpu affinity, start core 0, halt rest mrs x1, mpidr_el1 and x1, x1, #3 @@ -16,7 +15,7 @@ halt: setup: # store the desired EL1 stack pointer in x1 - adr x1, _start + ldr x1, =_start # use SP_ELx for Exception level ELx msr SPsel, #1 @@ -99,12 +98,17 @@ zero_bss: zero_bss_loop: # zero out the BSS section, 64-bits at a time - cbz x2, go_kmain + cbz x2, zero_bss_loop_end str xzr, [x1], #8 sub x2, x2, #8 cbnz x2, zero_bss_loop -go_kmain: +zero_bss_loop_end: + b _start + +.section .text.entry +.globl _start +_start: # jump to rust_main, which shouldn't return. halt if it does bl rust_main b halt diff --git a/kernel/src/arch/aarch64/boot/linker.ld b/kernel/src/arch/aarch64/boot/linker.ld index ce4c0c2..34858e3 100644 --- a/kernel/src/arch/aarch64/boot/linker.ld +++ b/kernel/src/arch/aarch64/boot/linker.ld @@ -3,11 +3,14 @@ ENTRY(_start) SECTIONS { . = 0x80000; /* Raspbery Pi 3 Aarch64 (kernel8.img) load address */ - /* start of the binary */ - _start = .; + .boot : { + KEEP(*(.text.boot)) /* from boot.S */ + } + + . = 0x100000; /* Load the kernel at this address. It's also kernel stack top address */ .text : { - KEEP(*(.text.boot)) /* from boot.S */ + *(.text.entry) *(.text .text.* .gnu.linkonce.t*) } From a9de99d3a9edc73e1b555efb38de5cc0e49886c1 Mon Sep 17 00:00:00 2001 From: equation314 Date: Wed, 21 Nov 2018 01:50:34 +0800 Subject: [PATCH 06/22] aarch64/mmu: remap kernel memory ranges at the end of memory::init() --- crate/aarch64/src/paging/mod.rs | 69 ++++++++++++++- crate/aarch64/src/paging/recursive.rs | 115 +++++++++++++------------ crate/memory/src/memory_set.rs | 5 +- kernel/src/arch/aarch64/boot/boot.S | 2 +- kernel/src/arch/aarch64/boot/linker.ld | 18 +++- kernel/src/arch/aarch64/memory.rs | 52 ++++++++--- kernel/src/arch/aarch64/paging.rs | 64 ++++++++------ 7 files changed, 224 insertions(+), 101 deletions(-) diff --git a/crate/aarch64/src/paging/mod.rs b/crate/aarch64/src/paging/mod.rs index ebc00dc..855b876 100644 --- a/crate/aarch64/src/paging/mod.rs +++ b/crate/aarch64/src/paging/mod.rs @@ -18,7 +18,7 @@ mod frame_alloc; mod page_table; mod recursive; -/// Trait for abstracting over the three possible page sizes on x86_64, 4KiB, 2MiB, 1GiB. +/// Trait for abstracting over the three possible block/page sizes on aarch64, 4KiB, 2MiB, 1GiB. pub trait PageSize: Copy + Eq + PartialOrd + Ord { /// The page size in bytes. const SIZE: u64; @@ -34,6 +34,14 @@ pub trait NotGiantPageSize: PageSize {} #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub enum Size4KiB {} +/// A “huge” 2MiB page. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum Size2MiB {} + +/// A “giant” 1GiB page. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum Size1GiB {} + impl PageSize for Size4KiB { const SIZE: u64 = 4096; const SIZE_AS_DEBUG_STR: &'static str = "4KiB"; @@ -41,6 +49,18 @@ impl PageSize for Size4KiB { impl NotGiantPageSize for Size4KiB {} +impl PageSize for Size2MiB { + const SIZE: u64 = Size4KiB::SIZE * 512; + const SIZE_AS_DEBUG_STR: &'static str = "2MiB"; +} + +impl NotGiantPageSize for Size2MiB {} + +impl PageSize for Size1GiB { + const SIZE: u64 = Size2MiB::SIZE * 512; + const SIZE_AS_DEBUG_STR: &'static str = "1GiB"; +} + /// A virtual memory page. #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] #[repr(C)] @@ -97,6 +117,14 @@ impl Page { pub fn range_inclusive(start: Self, end: Self) -> PageRangeInclusive { PageRangeInclusive { start, end } } + + pub fn of_addr(address: usize) -> Self { + Self::containing_address(VirtAddr::new(address as u64)) + } + + pub fn range_of(begin: usize, end: usize) -> PageRange { + Self::range(Page::of_addr(begin), Page::of_addr(end - 1) + 1) + } } impl Page { @@ -106,6 +134,31 @@ impl Page { } } +impl Page { + /// Returns the 1GiB memory page with the specified page table indices. + pub fn from_page_table_indices_1gib(p4_index: u9, p3_index: u9) -> Self { + use bit_field::BitField; + + let mut addr = 0; + addr.set_bits(39..48, u64::from(p4_index)); + addr.set_bits(30..39, u64::from(p3_index)); + Page::containing_address(VirtAddr::new(addr)) + } +} + +impl Page { + /// Returns the 2MiB memory page with the specified page table indices. + pub fn from_page_table_indices_2mib(p4_index: u9, p3_index: u9, p2_index: u9) -> Self { + use bit_field::BitField; + + let mut addr = 0; + addr.set_bits(39..48, u64::from(p4_index)); + addr.set_bits(30..39, u64::from(p3_index)); + addr.set_bits(21..30, u64::from(p2_index)); + Page::containing_address(VirtAddr::new(addr)) + } +} + impl Page { /// Returns the 4KiB memory page with the specified page table indices. pub fn from_page_table_indices(p4_index: u9, p3_index: u9, p2_index: u9, p1_index: u9) -> Self { @@ -199,6 +252,16 @@ impl Iterator for PageRange { } } +impl PageRange { + /// Converts the range of 2MiB pages to a range of 4KiB pages. + pub fn as_4kib_page_range(self) -> PageRange { + PageRange { + start: Page::containing_address(self.start.start_address()), + end: Page::containing_address(self.end.start_address()), + } + } +} + impl fmt::Debug for PageRange { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { f.debug_struct("PageRange") @@ -294,6 +357,10 @@ impl PhysFrame { pub fn range_inclusive(start: PhysFrame, end: PhysFrame) -> PhysFrameRangeInclusive { PhysFrameRangeInclusive { start, end } } + + pub fn of_addr(address: usize) -> Self { + Self::containing_address(PhysAddr::new(address as u64)) + } } impl fmt::Debug for PhysFrame { diff --git a/crate/aarch64/src/paging/recursive.rs b/crate/aarch64/src/paging/recursive.rs index d735216..31d1802 100644 --- a/crate/aarch64/src/paging/recursive.rs +++ b/crate/aarch64/src/paging/recursive.rs @@ -239,6 +239,45 @@ impl<'a> RecursivePageTable<'a> { inner(entry, next_table_page, allocator) } + + pub fn p3_ptr(&self, page: Page) -> *mut PageTable { + self.p3_page(page).start_address().as_mut_ptr() + } + + pub fn p2_ptr(&self, page: Page) -> *mut PageTable { + self.p2_page(page).start_address().as_mut_ptr() + } + + pub fn p1_ptr(&self, page: Page) -> *mut PageTable { + self.p1_page(page).start_address().as_mut_ptr() + } + + fn p3_page(&self, page: Page) -> Page { + Page::from_page_table_indices( + self.recursive_index, + self.recursive_index, + self.recursive_index, + page.p4_index(), + ) + } + + fn p2_page(&self, page: Page) -> Page { + Page::from_page_table_indices( + self.recursive_index, + self.recursive_index, + page.p4_index(), + page.p3_index(), + ) + } + + fn p1_page(&self, page: Page) -> Page { + Page::from_page_table_indices( + self.recursive_index, + page.p4_index(), + page.p3_index(), + page.p2_index(), + ) + } } @@ -253,15 +292,16 @@ impl<'a> Mapper for RecursivePageTable<'a> { where A: FrameAllocator, { - let p4 = &mut self.p4; + let self_mut = unsafe{ &mut *(self as *const _ as *mut Self) }; + let p4 = &mut self_mut.p4; - let p3_page = p3_page(page, self.recursive_index); + let p3_page = self.p3_page(page); let p3 = unsafe { Self::create_next_table(&mut p4[page.p4_index()], p3_page, allocator)? }; - let p2_page = p2_page(page, self.recursive_index); + let p2_page = self.p2_page(page); let p2 = unsafe { Self::create_next_table(&mut p3[page.p3_index()], p2_page, allocator)? }; - let p1_page = p1_page(page, self.recursive_index); + let p1_page = self.p1_page(page); let p1 = unsafe { Self::create_next_table(&mut p2[page.p2_index()], p1_page, allocator)? }; if !p1[page.p1_index()].is_unused() { @@ -276,28 +316,30 @@ impl<'a> Mapper for RecursivePageTable<'a> { &mut self, page: Page, ) -> Result<(PhysFrame, MapperFlush), UnmapError> { - let p4 = &mut self.p4; + let self_mut = unsafe{ &mut *(self as *const _ as *mut Self) }; + let p4 = &mut self_mut.p4; + let p4_entry = &p4[page.p4_index()]; p4_entry.frame().map_err(|err| match err { FrameError::FrameNotPresent => UnmapError::PageNotMapped, FrameError::HugeFrame => UnmapError::ParentEntryHugePage, })?; - let p3 = unsafe { &mut *(p3_ptr(page, self.recursive_index)) }; + let p3 = unsafe { &mut *(self.p3_ptr(page)) }; let p3_entry = &p3[page.p3_index()]; p3_entry.frame().map_err(|err| match err { FrameError::FrameNotPresent => UnmapError::PageNotMapped, FrameError::HugeFrame => UnmapError::ParentEntryHugePage, })?; - let p2 = unsafe { &mut *(p2_ptr(page, self.recursive_index)) }; + let p2 = unsafe { &mut *(self.p2_ptr(page)) }; let p2_entry = &p2[page.p2_index()]; p2_entry.frame().map_err(|err| match err { FrameError::FrameNotPresent => UnmapError::PageNotMapped, FrameError::HugeFrame => UnmapError::ParentEntryHugePage, })?; - let p1 = unsafe { &mut *(p1_ptr(page, self.recursive_index)) }; + let p1 = unsafe { &mut *(self.p1_ptr(page)) }; let p1_entry = &mut p1[page.p1_index()]; let frame = p1_entry.frame().map_err(|err| match err { @@ -314,25 +356,26 @@ impl<'a> Mapper for RecursivePageTable<'a> { page: Page, flags: PageTableFlags, ) -> Result, FlagUpdateError> { - let p4 = &mut self.p4; + let self_mut = unsafe{ &mut *(self as *const _ as *mut Self) }; + let p4 = &mut self_mut.p4; if p4[page.p4_index()].is_unused() { return Err(FlagUpdateError::PageNotMapped); } - let p3 = unsafe { &mut *(p3_ptr(page, self.recursive_index)) }; + let p3 = unsafe { &mut *(self.p3_ptr(page)) }; if p3[page.p3_index()].is_unused() { return Err(FlagUpdateError::PageNotMapped); } - let p2 = unsafe { &mut *(p2_ptr(page, self.recursive_index)) }; + let p2 = unsafe { &mut *(self.p2_ptr(page)) }; if p2[page.p2_index()].is_unused() { return Err(FlagUpdateError::PageNotMapped); } - let p1 = unsafe { &mut *(p1_ptr(page, self.recursive_index)) }; + let p1 = unsafe { &mut *(self.p1_ptr(page)) }; if p1[page.p1_index()].is_unused() { return Err(FlagUpdateError::PageNotMapped); @@ -344,27 +387,28 @@ impl<'a> Mapper for RecursivePageTable<'a> { } fn translate_page(&self, page: Page) -> Option> { - let p4 = &self.p4; + let self_mut = unsafe{ &mut *(self as *const _ as *mut Self) }; + let p4 = &mut self_mut.p4; if p4[page.p4_index()].is_unused() { return None; } - let p3 = unsafe { &*(p3_ptr(page, self.recursive_index)) }; + let p3 = unsafe { &*(self.p3_ptr(page)) }; let p3_entry = &p3[page.p3_index()]; if p3_entry.is_unused() { return None; } - let p2 = unsafe { &*(p2_ptr(page, self.recursive_index)) }; + let p2 = unsafe { &*(self.p2_ptr(page)) }; let p2_entry = &p2[page.p2_index()]; if p2_entry.is_unused() { return None; } - let p1 = unsafe { &*(p1_ptr(page, self.recursive_index)) }; + let p1 = unsafe { &*(self.p1_ptr(page)) }; let p1_entry = &p1[page.p1_index()]; if p1_entry.is_unused() { @@ -374,42 +418,3 @@ impl<'a> Mapper for RecursivePageTable<'a> { PhysFrame::from_start_address(p1_entry.addr()).ok() } } - -fn p3_ptr(page: Page, recursive_index: u9) -> *mut PageTable { - p3_page(page, recursive_index).start_address().as_mut_ptr() -} - -fn p3_page(page: Page, recursive_index: u9) -> Page { - Page::from_page_table_indices( - recursive_index, - recursive_index, - recursive_index, - page.p4_index(), - ) -} - -fn p2_ptr(page: Page, recursive_index: u9) -> *mut PageTable { - p2_page(page, recursive_index).start_address().as_mut_ptr() -} - -fn p2_page(page: Page, recursive_index: u9) -> Page { - Page::from_page_table_indices( - recursive_index, - recursive_index, - page.p4_index(), - page.p3_index(), - ) -} - -fn p1_ptr(page: Page, recursive_index: u9) -> *mut PageTable { - p1_page(page, recursive_index).start_address().as_mut_ptr() -} - -fn p1_page(page: Page, recursive_index: u9) -> Page { - Page::from_page_table_indices( - recursive_index, - page.p4_index(), - page.p3_index(), - page.p2_index(), - ) -} diff --git a/crate/memory/src/memory_set.rs b/crate/memory/src/memory_set.rs index ca3e7dc..f67c956 100644 --- a/crate/memory/src/memory_set.rs +++ b/crate/memory/src/memory_set.rs @@ -166,6 +166,9 @@ impl MemorySet { pub fn iter(&self) -> impl Iterator { self.areas.iter() } + pub fn edit(&mut self, f: impl FnOnce(&mut T::Active)) { + self.page_table.edit(f); + } pub unsafe fn with(&self, f: impl FnOnce()) { self.page_table.with(f); } @@ -223,4 +226,4 @@ impl Debug for MemorySet { pub struct Stack { pub top: usize, pub bottom: usize, -} \ No newline at end of file +} diff --git a/kernel/src/arch/aarch64/boot/boot.S b/kernel/src/arch/aarch64/boot/boot.S index e525340..7a126bd 100644 --- a/kernel/src/arch/aarch64/boot/boot.S +++ b/kernel/src/arch/aarch64/boot/boot.S @@ -93,7 +93,7 @@ set_stack: zero_bss: # load the start address and number of bytes in BSS section - ldr x1, =__bss_start + ldr x1, =sbss ldr x2, =__bss_length zero_bss_loop: diff --git a/kernel/src/arch/aarch64/boot/linker.ld b/kernel/src/arch/aarch64/boot/linker.ld index 34858e3..2c7bdce 100644 --- a/kernel/src/arch/aarch64/boot/linker.ld +++ b/kernel/src/arch/aarch64/boot/linker.ld @@ -8,34 +8,44 @@ SECTIONS { } . = 0x100000; /* Load the kernel at this address. It's also kernel stack top address */ + bootstacktop = .; .text : { + stext = .; *(.text.entry) *(.text .text.* .gnu.linkonce.t*) + . = ALIGN(4K); + etext = .; } .rodata : { + srodata = .; *(.rodata .rodata.* .gnu.linkonce.r*) + . = ALIGN(4K); + erodata = .; } .data : { + sdata = .; *(.data .data.* .gnu.linkonce.d*) + . = ALIGN(4K); + edata = .; } .bss (NOLOAD) : { . = ALIGN(32); - __bss_start = .; + sbss = .; *(.bss .bss.*) *(COMMON) - . = ALIGN(8); - __bss_end = .; + . = ALIGN(4K); + ebss = .; } /* end of the binary */ _end = ALIGN(8); /* number of bytes in BSS section and complete binary */ - __bss_length = (__bss_end - __bss_start); + __bss_length = (ebss - sbss); __binary_length = (_end - _start); /DISCARD/ : { *(.comment) *(.gnu*) *(.note*) *(.eh_frame*) } diff --git a/kernel/src/arch/aarch64/memory.rs b/kernel/src/arch/aarch64/memory.rs index 3e7f6d8..c04e9c3 100644 --- a/kernel/src/arch/aarch64/memory.rs +++ b/kernel/src/arch/aarch64/memory.rs @@ -1,13 +1,9 @@ //! Memory initialization for aarch64. -use bit_allocator::BitAlloc; use ucore_memory::PAGE_SIZE; -use memory::{FRAME_ALLOCATOR, init_heap}; +use memory::{FRAME_ALLOCATOR, init_heap, MemoryArea, MemoryAttr, MemorySet, Stack}; use super::atags::atags::Atags; -//use super::super::HEAP_ALLOCATOR; -use aarch64::{barrier, regs::*, addr::*}; -use aarch64::paging::{FrameAllocator, FrameDeallocator, Page, PageRange, PhysFrame as Frame, Size4KiB}; -use core::ops::Range; +use aarch64::{barrier, regs::*, addr::*, paging::PhysFrame as Frame}; /// Memory initialization. pub fn init() { @@ -25,14 +21,11 @@ pub fn init() { init_mmu(); init_frame_allocator(); init_heap(); + remap_the_kernel(); info!("memory: init end"); } -extern "C" { - static _end: u8; -} - fn init_frame_allocator() { use bit_allocator::BitAlloc; use core::ops::Range; @@ -96,12 +89,36 @@ fn init_mmu() { info!("mmu enabled"); } +fn remap_the_kernel() { + let (bottom, top) = (0, bootstacktop as usize); + let kstack = Stack { + top, + bottom, + }; + static mut SPACE: [u8; 0x1000] = [0; 0x1000]; + let mut ms = unsafe { MemorySet::new_from_raw_space(&mut SPACE, kstack) }; + ms.push(MemoryArea::new_identity(bottom, top, MemoryAttr::default(), "kstack")); + ms.push(MemoryArea::new_identity(stext as usize, etext as usize, MemoryAttr::default().execute().readonly(), "text")); + ms.push(MemoryArea::new_identity(sdata as usize, edata as usize, MemoryAttr::default(), "data")); + ms.push(MemoryArea::new_identity(srodata as usize, erodata as usize, MemoryAttr::default().readonly(), "rodata")); + ms.push(MemoryArea::new_identity(sbss as usize, ebss as usize, MemoryAttr::default(), "bss")); + + // ensure the level 2 page table exists + ms.push(MemoryArea::new_identity(0x40000000, 0x40200000, MemoryAttr::default(), "arm_control")); + super::paging::remap_device_2mib(&mut ms, 0x3F000000, 0x40200000); + + unsafe { ms.activate(); } + use core::mem::forget; + forget(ms); + info!("kernel remap end"); +} + /// Returns the (start address, end address) of the available memory on this /// system if it can be determined. If it cannot, `None` is returned. /// /// This function is expected to return `Some` under all normal cirumstances. fn memory_map() -> Option<(usize, usize)> { - let binary_end = unsafe { (&_end as *const u8) as u32 }; + let binary_end = unsafe { _end as u32 }; let mut atags: Atags = Atags::get(); while let Some(atag) = atags.next() { @@ -113,3 +130,16 @@ fn memory_map() -> Option<(usize, usize)> { None } +extern { + fn bootstacktop(); + fn stext(); + fn etext(); + fn sdata(); + fn edata(); + fn srodata(); + fn erodata(); + fn sbss(); + fn ebss(); + fn _start(); + fn _end(); +} diff --git a/kernel/src/arch/aarch64/paging.rs b/kernel/src/arch/aarch64/paging.rs index 9696db9..a023237 100644 --- a/kernel/src/arch/aarch64/paging.rs +++ b/kernel/src/arch/aarch64/paging.rs @@ -8,8 +8,7 @@ use ucore_memory::paging::*; use aarch64::asm::{tlb_invalidate, ttbr0_el1_read, ttbr0_el1_write}; use aarch64::{PhysAddr, VirtAddr}; use aarch64::paging::{Mapper, PageTable as Aarch64PageTable, PageTableEntry, PageTableFlags as EF, RecursivePageTable}; -use aarch64::paging::{FrameAllocator, FrameDeallocator, Page, PageRange, PhysFrame as Frame, Size4KiB}; -use aarch64::{regs::*}; +use aarch64::paging::{FrameAllocator, FrameDeallocator, Page, PageRange, PhysFrame as Frame, Size4KiB, Size2MiB}; register_bitfields! {u64, // AArch64 Reference Manual page 2150 @@ -61,6 +60,11 @@ register_bitfields! {u64, ] } +mod mair { + pub const NORMAL: u64 = 0; + pub const DEVICE: u64 = 1; +} + // need 3 page pub fn setup_page_table(frame_lvl4: Frame, frame_lvl3: Frame, frame_lvl2: Frame) { let p4 = unsafe { &mut *(frame_lvl4.start_address().as_u64() as *mut Aarch64PageTable) }; @@ -70,11 +74,6 @@ pub fn setup_page_table(frame_lvl4: Frame, frame_lvl3: Frame, frame_lvl2: Frame) p3.zero(); p2.zero(); - mod mair { - pub const NORMAL: u64 = 0; - pub const DEVICE: u64 = 1; - } - // Fill the rest of the LVL2 (2MiB) entries as block // descriptors. Differentiate between normal and device mem. const MMIO_BASE: u64 = 0x3F000000; @@ -129,28 +128,37 @@ pub fn setup_page_table(frame_lvl4: Frame, frame_lvl3: Frame, frame_lvl2: Frame) info!("setup init page table end"); } -pub trait PageExt { - fn of_addr(address: usize) -> Self; - fn range_of(begin: usize, end: usize) -> PageRange; -} - -impl PageExt for Page { - fn of_addr(address: usize) -> Self { - Page::containing_address(VirtAddr::new(address as u64)) - } - fn range_of(begin: usize, end: usize) -> PageRange { - Page::range(Page::of_addr(begin), Page::of_addr(end - 1) + 1) - } -} - -pub trait FrameExt { - fn of_addr(address: usize) -> Self; -} +/// map the range [start, end) as device memory, insert to the MemorySet +pub fn remap_device_2mib(ms: &mut MemorySet, start_addr: usize, end_addr: usize) { + ms.edit(|active_table| { + let common = STAGE1_DESCRIPTOR::VALID::True + + STAGE1_DESCRIPTOR::TYPE::Block + + STAGE1_DESCRIPTOR::AP::RW_EL1 + + STAGE1_DESCRIPTOR::AF::True + + STAGE1_DESCRIPTOR::XN::True; + + let mem_attr = STAGE1_DESCRIPTOR::SH::OuterShareable + STAGE1_DESCRIPTOR::AttrIndx.val(mair::DEVICE); + + type Page2MiB = Page; + for page in Page2MiB::range_of(start_addr, end_addr) { + let p2 = unsafe { &mut *active_table.0.p2_ptr(page) }; + p2[page.p2_index()].entry = (common + mem_attr + STAGE1_DESCRIPTOR::LVL2_OUTPUT_ADDR_4KiB.val(page.start_address().as_u64() >> 21)).value; + } -impl FrameExt for Frame { - fn of_addr(address: usize) -> Self { - Frame::containing_address(PhysAddr::new(address as u64)) - } + // let p2 = unsafe { &mut *(0o777_777_000_000_0000 as *mut Aarch64PageTable) }; + // for i in 0..512 { + // if p2[i].flags().bits() != 0 { + // info!("{:x?} {:x?} {:x?}",i, &p2[i] as *const _ as usize, p2[i]); + // } + // } + + // let p2 = unsafe { &mut *(0o777_777_000_001_0000 as *mut Aarch64PageTable) }; + // for i in 0..512 { + // if p2[i].flags().bits() != 0 { + // info!("{:x?} {:x?} {:x?}",i, &p2[i] as *const _ as usize, p2[i]); + // } + // } + }); } pub struct ActivePageTable(RecursivePageTable<'static>); From bb1c1abaa480d21c920082950effb5d35df8c55e Mon Sep 17 00:00:00 2001 From: equation314 Date: Wed, 21 Nov 2018 20:02:34 +0800 Subject: [PATCH 07/22] aarch64/mmu: can run on the real raspi3 --- crate/aarch64/src/asm.rs | 16 ++++- crate/aarch64/src/paging/recursive.rs | 3 +- kernel/src/arch/aarch64/board/raspi3/mod.rs | 7 +- .../src/arch/aarch64/board/raspi3/serial.rs | 8 +-- kernel/src/arch/aarch64/io.rs | 12 ++-- kernel/src/arch/aarch64/memory.rs | 70 +++++++++---------- kernel/src/arch/aarch64/mod.rs | 6 +- kernel/src/arch/aarch64/paging.rs | 19 +++-- 8 files changed, 72 insertions(+), 69 deletions(-) diff --git a/crate/aarch64/src/asm.rs b/crate/aarch64/src/asm.rs index 17e6c29..63ce23a 100644 --- a/crate/aarch64/src/asm.rs +++ b/crate/aarch64/src/asm.rs @@ -1,9 +1,9 @@ use paging::PhysFrame; -use addr::PhysAddr; +use addr::{PhysAddr, VirtAddr}; use regs::*; #[inline(always)] -pub fn tlb_invalidate() { +pub fn tlb_invalidate_all() { unsafe { asm!( "dsb ishst @@ -14,6 +14,18 @@ pub fn tlb_invalidate() { } } +#[inline(always)] +pub fn tlb_invalidate(vaddr: VirtAddr) { + unsafe { + asm!( + "dsb ishst + tlbi vaae1is, $0 + dsb ish + isb" :: "r"(vaddr.as_u64() >> 12) + ); + } +} + /// Returns the current stack pointer. #[inline(always)] pub fn sp() -> *const u8 { diff --git a/crate/aarch64/src/paging/recursive.rs b/crate/aarch64/src/paging/recursive.rs index 31d1802..79b0a0d 100644 --- a/crate/aarch64/src/paging/recursive.rs +++ b/crate/aarch64/src/paging/recursive.rs @@ -28,7 +28,7 @@ impl MapperFlush { /// Flush the page from the TLB to ensure that the newest mapping is used. pub fn flush(self) { - tlb_invalidate(); + tlb_invalidate(self.0.start_address()); } /// Don't flush the TLB and silence the “must be used” warning. @@ -232,6 +232,7 @@ impl<'a> RecursivePageTable<'a> { let page_table_ptr = next_table_page.start_address().as_mut_ptr(); let page_table: &mut PageTable = unsafe { &mut *(page_table_ptr) }; if created { + tlb_invalidate(next_table_page.start_address()); page_table.zero(); } Ok(page_table) diff --git a/kernel/src/arch/aarch64/board/raspi3/mod.rs b/kernel/src/arch/aarch64/board/raspi3/mod.rs index 2b32a10..2d13315 100644 --- a/kernel/src/arch/aarch64/board/raspi3/mod.rs +++ b/kernel/src/arch/aarch64/board/raspi3/mod.rs @@ -7,12 +7,9 @@ pub mod timer; pub mod serial; pub fn init() { - // FIXME - // assert_has_not_been_called!("board::init must be called only once"); + assert_has_not_been_called!("board::init must be called only once"); - unsafe { - serial::SERIAL_PORT.init(); - } + serial::SERIAL_PORT.lock().init(); println!("Hello Raspberry Pi!"); } diff --git a/kernel/src/arch/aarch64/board/raspi3/serial.rs b/kernel/src/arch/aarch64/board/raspi3/serial.rs index 5434071..b6c29f7 100644 --- a/kernel/src/arch/aarch64/board/raspi3/serial.rs +++ b/kernel/src/arch/aarch64/board/raspi3/serial.rs @@ -20,8 +20,7 @@ impl SerialPort { /// Init a newly created SerialPort, can only be called once. pub fn init(&mut self) { - // FIXME - // assert_has_not_been_called!("SerialPort::init must be called only once"); + assert_has_not_been_called!("SerialPort::init must be called only once"); self.mu = Some(MiniUart::new()); } @@ -71,7 +70,4 @@ impl fmt::Write for SerialPort { } } -// FIXME -// pub static SERIAL_PORT: Mutex = Mutex::new(SerialPort::new()); -pub static mut SERIAL_PORT: SerialPort = SerialPort::new(); - +pub static SERIAL_PORT: Mutex = Mutex::new(SerialPort::new()); diff --git a/kernel/src/arch/aarch64/io.rs b/kernel/src/arch/aarch64/io.rs index deda39c..1a9d1d4 100644 --- a/kernel/src/arch/aarch64/io.rs +++ b/kernel/src/arch/aarch64/io.rs @@ -4,15 +4,11 @@ use core::fmt::{Arguments, Write}; use super::board::serial::{SerialRead, SERIAL_PORT}; pub fn getchar() -> char { - // FIXME - unsafe { - SERIAL_PORT.receive() as char - } + unsafe { SERIAL_PORT.force_unlock(); } + SERIAL_PORT.lock().receive() as char } pub fn putfmt(fmt: Arguments) { - // FIXME - unsafe { - SERIAL_PORT.write_fmt(fmt).unwrap() - } + unsafe { SERIAL_PORT.force_unlock(); } + SERIAL_PORT.lock().write_fmt(fmt).unwrap() } diff --git a/kernel/src/arch/aarch64/memory.rs b/kernel/src/arch/aarch64/memory.rs index c04e9c3..3f93e31 100644 --- a/kernel/src/arch/aarch64/memory.rs +++ b/kernel/src/arch/aarch64/memory.rs @@ -7,6 +7,14 @@ use aarch64::{barrier, regs::*, addr::*, paging::PhysFrame as Frame}; /// Memory initialization. pub fn init() { + init_frame_allocator(); + init_heap(); + remap_the_kernel(); + info!("memory: init end"); +} + +/// initialize temporary paging and enable mmu immediately after boot. Serial port is disabled at this time. +pub fn init_mmu_early() { #[repr(align(4096))] struct PageData([u8; PAGE_SIZE]); static PAGE_TABLE_LVL4: PageData = PageData([0; PAGE_SIZE]); @@ -18,41 +26,6 @@ pub fn init() { let frame_lvl2 = Frame::containing_address(PhysAddr::new(&PAGE_TABLE_LVL2 as *const _ as u64)); super::paging::setup_page_table(frame_lvl4, frame_lvl3, frame_lvl2); - init_mmu(); - init_frame_allocator(); - init_heap(); - remap_the_kernel(); - - info!("memory: init end"); -} - -fn init_frame_allocator() { - use bit_allocator::BitAlloc; - use core::ops::Range; - use consts::{MEMORY_OFFSET}; - - let (start, end) = memory_map().expect("failed to find memory map"); - let mut ba = FRAME_ALLOCATOR.lock(); - ba.insert(to_range(start, end)); - info!("FrameAllocator init end"); - - /* - * @param: - * start: start address - * end: end address - * @brief: - * transform the memory address to the page number - * @retval: - * the page number range from start address to end address - */ - fn to_range(start: usize, end: usize) -> Range { - let page_start = (start - MEMORY_OFFSET) / PAGE_SIZE; - let page_end = (end - MEMORY_OFFSET - 1) / PAGE_SIZE + 1; - page_start..page_end - } -} - -fn init_mmu() { // device. MAIR_EL1.write( // Attribute 1 @@ -85,10 +58,35 @@ fn init_mmu() { // Force MMU init to complete before next instruction unsafe { barrier::isb(barrier::SY); } +} + +fn init_frame_allocator() { + use bit_allocator::BitAlloc; + use core::ops::Range; + use consts::{MEMORY_OFFSET}; + + let (start, end) = memory_map().expect("failed to find memory map"); + let mut ba = FRAME_ALLOCATOR.lock(); + ba.insert(to_range(start, end)); + info!("FrameAllocator init end"); - info!("mmu enabled"); + /* + * @param: + * start: start address + * end: end address + * @brief: + * transform the memory address to the page number + * @retval: + * the page number range from start address to end address + */ + fn to_range(start: usize, end: usize) -> Range { + let page_start = (start - MEMORY_OFFSET) / PAGE_SIZE; + let page_end = (end - MEMORY_OFFSET - 1) / PAGE_SIZE + 1; + page_start..page_end + } } +/// remap kernel page table after all initialization. fn remap_the_kernel() { let (bottom, top) = (0, bootstacktop as usize); let kstack = Stack { diff --git a/kernel/src/arch/aarch64/mod.rs b/kernel/src/arch/aarch64/mod.rs index 35613ab..ccd6230 100644 --- a/kernel/src/arch/aarch64/mod.rs +++ b/kernel/src/arch/aarch64/mod.rs @@ -17,9 +17,13 @@ pub use self::board::timer; /// The entry point of kernel #[no_mangle] // don't mangle the name of this function pub extern "C" fn rust_main() -> ! { + // Enable mmu and paging + memory::init_mmu_early(); + // Init board to enable serial port. board::init(); - ::logging::init(); // FIXME + + ::logging::init(); interrupt::init(); memory::init(); timer::init(); diff --git a/kernel/src/arch/aarch64/paging.rs b/kernel/src/arch/aarch64/paging.rs index a023237..c7c4e99 100644 --- a/kernel/src/arch/aarch64/paging.rs +++ b/kernel/src/arch/aarch64/paging.rs @@ -5,7 +5,7 @@ use memory::{active_table, alloc_frame, alloc_stack, dealloc_frame}; use ucore_memory::memory_set::*; use ucore_memory::PAGE_SIZE; use ucore_memory::paging::*; -use aarch64::asm::{tlb_invalidate, ttbr0_el1_read, ttbr0_el1_write}; +use aarch64::asm::{tlb_invalidate, tlb_invalidate_all, ttbr0_el1_read, ttbr0_el1_write}; use aarch64::{PhysAddr, VirtAddr}; use aarch64::paging::{Mapper, PageTable as Aarch64PageTable, PageTableEntry, PageTableFlags as EF, RecursivePageTable}; use aarch64::paging::{FrameAllocator, FrameDeallocator, Page, PageRange, PhysFrame as Frame, Size4KiB, Size2MiB}; @@ -123,9 +123,7 @@ pub fn setup_page_table(frame_lvl4: Frame, frame_lvl3: Frame, frame_lvl2: Frame) // } ttbr0_el1_write(frame_lvl4); - tlb_invalidate(); - - info!("setup init page table end"); + tlb_invalidate_all(); } /// map the range [start, end) as device memory, insert to the MemorySet @@ -222,7 +220,8 @@ impl ActivePageTable { impl Entry for PageEntry { fn update(&mut self) { - tlb_invalidate(); + let addr = VirtAddr::new_unchecked((self as *const _ as u64) << 9); + tlb_invalidate(addr); } fn present(&self) -> bool { self.0.flags().contains(EF::PRESENT) } @@ -314,14 +313,14 @@ impl InactivePageTable for InactivePageTable0 { // overwrite recursive mapping p4_table[RECURSIVE_INDEX].set_frame(self.p4_frame.clone(), EF::PRESENT | EF::WRITE | EF::ACCESSED | EF::PAGE_BIT); - tlb_invalidate(); + tlb_invalidate_all(); // execute f in the new context f(active_table); // restore recursive mapping to original p4 table p4_table[RECURSIVE_INDEX] = backup; - tlb_invalidate(); + tlb_invalidate_all(); }); } @@ -331,7 +330,7 @@ impl InactivePageTable for InactivePageTable0 { debug!("switch table {:?} -> {:?}", old_frame, new_frame); if old_frame != new_frame { ttbr0_el1_write(new_frame); - tlb_invalidate(); + tlb_invalidate_all(); } } @@ -341,13 +340,13 @@ impl InactivePageTable for InactivePageTable0 { debug!("switch table {:?} -> {:?}", old_frame, new_frame); if old_frame != new_frame { ttbr0_el1_write(new_frame); - tlb_invalidate(); + tlb_invalidate_all(); } f(); debug!("switch table {:?} -> {:?}", new_frame, old_frame); if old_frame != new_frame { ttbr0_el1_write(old_frame); - tlb_invalidate(); + tlb_invalidate_all(); } } From 55087fc5a2dbfc7d3ea9e6d15a1170f1548ff503 Mon Sep 17 00:00:00 2001 From: equation314 Date: Fri, 23 Nov 2018 00:43:18 +0800 Subject: [PATCH 08/22] aarch64/mmu: use both TTBR0_EL1 & TTBR1_EL1 --- crate/aarch64/src/addr.rs | 23 ++++ crate/aarch64/src/asm.rs | 36 +++-- crate/aarch64/src/paging/mod.rs | 5 + crate/aarch64/src/paging/recursive.rs | 16 +-- crate/aarch64/src/regs/mod.rs | 2 + crate/aarch64/src/regs/tcr_el1.rs | 135 +++++++++++++++++++ crate/aarch64/src/regs/ttbr0_el1.rs | 5 + crate/aarch64/src/regs/ttbr1_el1.rs | 61 +++++++++ kernel/src/arch/aarch64/interrupt/context.rs | 23 ++-- kernel/src/arch/aarch64/memory.rs | 31 +++-- kernel/src/arch/aarch64/paging.rs | 31 ++--- kernel/src/consts.rs | 2 +- kernel/src/process/context.rs | 4 + 13 files changed, 309 insertions(+), 65 deletions(-) create mode 100644 crate/aarch64/src/regs/ttbr1_el1.rs diff --git a/crate/aarch64/src/addr.rs b/crate/aarch64/src/addr.rs index e347567..cf3f092 100644 --- a/crate/aarch64/src/addr.rs +++ b/crate/aarch64/src/addr.rs @@ -6,6 +6,15 @@ use bit_field::BitField; use usize_conversions::FromUsize; use ux::*; +#[derive(Debug)] +#[repr(u8)] +pub enum VirtAddrRange { + /// 0x0000000000000000 to 0x0000FFFFFFFFFFFF + BottomRange = 0, + /// 0xFFFF000000000000 to 0xFFFFFFFFFFFFFFFF. + TopRange = 1, +} + #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] #[repr(transparent)] pub struct VirtAddr(u64); @@ -105,6 +114,20 @@ impl VirtAddr { u12::new((self.0 & 0xfff).try_into().unwrap()) } + /// Returns the VA range + pub fn va_range(&self) -> Result { + match self.va_range_bits() { + 0x0000 => Ok(VirtAddrRange::BottomRange), + 0xffff => Ok(VirtAddrRange::TopRange), + _ => Err(VirtAddrNotValid(self.0)), + } + } + + /// Returns the top 16 bits + pub fn va_range_bits(&self) -> u16 { + ((self.0 >> 48) & 0xffff) as u16 + } + /// Returns the 9-bit level 1 page table index. pub fn p1_index(&self) -> u9 { u9::new(((self.0 >> 12) & 0o777).try_into().unwrap()) diff --git a/crate/aarch64/src/asm.rs b/crate/aarch64/src/asm.rs index 63ce23a..a5b16d5 100644 --- a/crate/aarch64/src/asm.rs +++ b/crate/aarch64/src/asm.rs @@ -160,24 +160,20 @@ pub fn eret() -> ! { } } -bitflags! { - /// Controls cache settings for the level 4 page table. - pub struct ttbr0_el1_Flags: u64 { - - const COMMON_NOT_PRIVATE = 1 << 0; - } -} - -pub fn ttbr0_el1_read() -> (PhysFrame, ttbr0_el1_Flags) { - let value = TTBR0_EL1.get(); - let flags = ttbr0_el1_Flags::from_bits_truncate(value); - let addr = PhysAddr::new(value & 0x_0000_ffff_ffff_f000); - let frame = PhysFrame::containing_address(addr); - (frame, flags) -} - -pub fn ttbr0_el1_write(frame: PhysFrame) { - let addr = frame.start_address(); - let value = addr.as_u64(); - TTBR0_EL1.set_baddr(value); +pub fn ttbr_el1_read(which: u8) -> (PhysFrame) { + let baddr = match which { + 0 => TTBR0_EL1.get_baddr(), + 1 => TTBR1_EL1.get_baddr(), + _ => 0, + }; + PhysFrame::containing_address(PhysAddr::new(baddr)) +} + +pub fn ttbr_el1_write(which: u8, frame: PhysFrame) { + let baddr = frame.start_address().as_u64(); + match which { + 0 => TTBR0_EL1.set_baddr(baddr), + 1 => TTBR1_EL1.set_baddr(baddr), + _ => {} + }; } diff --git a/crate/aarch64/src/paging/mod.rs b/crate/aarch64/src/paging/mod.rs index 855b876..c73d5b4 100644 --- a/crate/aarch64/src/paging/mod.rs +++ b/crate/aarch64/src/paging/mod.rs @@ -98,6 +98,11 @@ impl Page { S::SIZE } + /// Returns the level 4 page table index of this page. + pub fn va_range_bits(&self) -> u16 { + self.start_address().va_range_bits() + } + /// Returns the level 4 page table index of this page. pub fn p4_index(&self) -> u9 { self.start_address().p4_index() diff --git a/crate/aarch64/src/paging/recursive.rs b/crate/aarch64/src/paging/recursive.rs index 79b0a0d..ee61cba 100644 --- a/crate/aarch64/src/paging/recursive.rs +++ b/crate/aarch64/src/paging/recursive.rs @@ -7,7 +7,7 @@ use paging::{ NotGiantPageSize, Page, PageSize, PhysFrame, Size4KiB, }; use paging::page_table::PageTableFlags as Flags; -use asm::ttbr0_el1_read; +use asm::ttbr_el1_read; use ux::u9; use addr::{PhysAddr, VirtAddr}; @@ -162,7 +162,9 @@ impl<'a> RecursivePageTable<'a> { { return Err(NotRecursivelyMapped); } - if Ok(ttbr0_el1_read().0) != table[recursive_index].frame() { + if Ok(ttbr_el1_read(page.start_address().va_range().unwrap() as u8)) != + table[recursive_index].frame() + { return Err(NotRecursivelyMapped); } @@ -212,7 +214,6 @@ impl<'a> RecursivePageTable<'a> { where A: FrameAllocator, { - let created; if entry.is_unused() { @@ -281,7 +282,6 @@ impl<'a> RecursivePageTable<'a> { } } - impl<'a> Mapper for RecursivePageTable<'a> { fn map_to( &mut self, @@ -293,7 +293,7 @@ impl<'a> Mapper for RecursivePageTable<'a> { where A: FrameAllocator, { - let self_mut = unsafe{ &mut *(self as *const _ as *mut Self) }; + let self_mut = unsafe { &mut *(self as *const _ as *mut Self) }; let p4 = &mut self_mut.p4; let p3_page = self.p3_page(page); @@ -317,7 +317,7 @@ impl<'a> Mapper for RecursivePageTable<'a> { &mut self, page: Page, ) -> Result<(PhysFrame, MapperFlush), UnmapError> { - let self_mut = unsafe{ &mut *(self as *const _ as *mut Self) }; + let self_mut = unsafe { &mut *(self as *const _ as *mut Self) }; let p4 = &mut self_mut.p4; let p4_entry = &p4[page.p4_index()]; @@ -357,7 +357,7 @@ impl<'a> Mapper for RecursivePageTable<'a> { page: Page, flags: PageTableFlags, ) -> Result, FlagUpdateError> { - let self_mut = unsafe{ &mut *(self as *const _ as *mut Self) }; + let self_mut = unsafe { &mut *(self as *const _ as *mut Self) }; let p4 = &mut self_mut.p4; if p4[page.p4_index()].is_unused() { @@ -388,7 +388,7 @@ impl<'a> Mapper for RecursivePageTable<'a> { } fn translate_page(&self, page: Page) -> Option> { - let self_mut = unsafe{ &mut *(self as *const _ as *mut Self) }; + let self_mut = unsafe { &mut *(self as *const _ as *mut Self) }; let p4 = &mut self_mut.p4; if p4[page.p4_index()].is_unused() { diff --git a/crate/aarch64/src/regs/mod.rs b/crate/aarch64/src/regs/mod.rs index f6a0e3d..5a7e556 100644 --- a/crate/aarch64/src/regs/mod.rs +++ b/crate/aarch64/src/regs/mod.rs @@ -24,6 +24,7 @@ mod spsel; mod spsr_el2; mod tcr_el1; mod ttbr0_el1; +mod ttbr1_el1; // Export only the R/W traits and the static reg definitions pub use register::cpu::*; @@ -49,3 +50,4 @@ pub use self::spsel::SPSel; pub use self::spsr_el2::SPSR_EL2; pub use self::tcr_el1::TCR_EL1; pub use self::ttbr0_el1::TTBR0_EL1; +pub use self::ttbr1_el1::TTBR1_EL1; diff --git a/crate/aarch64/src/regs/tcr_el1.rs b/crate/aarch64/src/regs/tcr_el1.rs index bcd0425..9ebb6d7 100644 --- a/crate/aarch64/src/regs/tcr_el1.rs +++ b/crate/aarch64/src/regs/tcr_el1.rs @@ -22,6 +22,17 @@ use register::cpu::RegisterReadWrite; register_bitfields! {u64, TCR_EL1 [ + /// Top Byte ignored - indicates whether the top byte of an address is + /// used for address match for the TTBR1_EL1 region, or ignored and used + /// for tagged addresses. Defined values are: + /// + /// 0 Top Byte used in the address calculation. + /// 1 Top Byte ignored in the address calculation. + TBI1 OFFSET(38) NUMBITS(1) [ + Used = 0, + Ignored = 1 + ], + /// Top Byte ignored - indicates whether the top byte of an address is /// used for address match for the TTBR0_EL1 region, or ignored and used /// for tagged addresses. Defined values are: @@ -33,6 +44,20 @@ register_bitfields! {u64, Ignored = 1 ], + /// ASID Size. Defined values are: + /// + /// 0 8 bit - the upper 8 bits of TTBR0_EL1 and TTBR1_EL1 are ignored by + /// hardware for every purpose except reading back the register, and are + /// treated as if they are all zeros for when used for allocation and matching entries in the TLB. + /// 1 16 bit - the upper 16 bits of TTBR0_EL1 and TTBR1_EL1 are used for + /// allocation and matching in the TLB. + /// + /// If the implementation has only 8 bits of ASID, this field is RES0. + AS OFFSET(36) NUMBITS(1) [ + Bits_8 = 0, + Bits_16 = 1 + ], + /// Intermediate Physical Address Size. /// /// 000 32 bits, 4GiB. @@ -67,6 +92,116 @@ register_bitfields! {u64, Bits_52 = 0b110 ], + /// Granule size for the TTBR1_EL1. + /// + /// 01 16KiB + /// 10 4KiB + /// 11 64KiB + /// + /// Other values are reserved. + /// + /// If the value is programmed to either a reserved value, or a size + /// that has not been implemented, then the hardware will treat the + /// field as if it has been programmed to an IMPLEMENTATION DEFINED + /// choice of the sizes that has been implemented for all purposes other + /// than the value read back from this register. + /// + /// It is IMPLEMENTATION DEFINED whether the value read back is the + /// value programmed or the value that corresponds to the size chosen. + TG1 OFFSET(30) NUMBITS(2) [ + KiB_4 = 0b10, + KiB_16 = 0b01, + KiB_64 = 0b11 + ], + + /// Shareability attribute for memory associated with translation table + /// walks using TTBR1_EL1. + /// + /// 00 Non-shareable + /// 10 Outer Shareable + /// 11 Inner Shareable + /// + /// Other values are reserved. + SH1 OFFSET(28) NUMBITS(2) [ + None = 0b00, + Outer = 0b10, + Inner = 0b11 + ], + + /// Outer cacheability attribute for memory associated with translation + /// table walks using TTBR1_EL1. + /// + /// 00 Normal memory, Outer Non-cacheable + /// + /// 01 Normal memory, Outer Write-Back Read-Allocate Write-Allocate + /// Cacheable + /// + /// 10 Normal memory, Outer Write-Through Read-Allocate No + /// Write-Allocate Cacheable + /// + /// 11 Normal memory, Outer Write-Back Read-Allocate No Write-Allocate + /// Cacheable + ORGN1 OFFSET(26) NUMBITS(2) [ + NonCacheable = 0b00, + WriteBack_ReadAlloc_WriteAlloc_Cacheable = 0b01, + WriteThrough_ReadAlloc_NoWriteAlloc_Cacheable = 0b10, + WriteBack_ReadAlloc_NoWriteAlloc_Cacheable = 0b11 + ], + + /// Inner cacheability attribute for memory associated with translation + /// table walks using TTBR1_EL1. + /// + /// 00 Normal memory, Inner Non-cacheable + /// + /// 01 Normal memory, Inner Write-Back Read-Allocate Write-Allocate + /// Cacheable + /// + /// 10 Normal memory, Inner Write-Through Read-Allocate No + /// Write-Allocate Cacheable + /// + /// 11 Normal memory, Inner Write-Back Read-Allocate No Write-Allocate + /// Cacheable + IRGN1 OFFSET(24) NUMBITS(2) [ + NonCacheable = 0b00, + WriteBack_ReadAlloc_WriteAlloc_Cacheable = 0b01, + WriteThrough_ReadAlloc_NoWriteAlloc_Cacheable = 0b10, + WriteBack_ReadAlloc_NoWriteAlloc_Cacheable = 0b11 + ], + + /// Translation table walk disable for translations using + /// TTBR1_EL1. This bit controls whether a translation table walk is + /// performed on a TLB miss, for an address that is translated using + /// TTBR1_EL1. The encoding of this bit is: + /// + /// 0 Perform translation table walks using TTBR1_EL1. + /// + /// 1 A TLB miss on an address that is translated using TTBR1_EL1 + /// generates a Translation fault. No translation table walk is + /// performed. + EPD1 OFFSET(23) NUMBITS(1) [ + EnableTTBR1Walks = 0, + DisableTTBR1Walks = 1 + ], + + /// Selects whether TTBR0_EL1 or TTBR1_EL1 defines the ASID. The encoding + /// of this bit is: + /// + /// 0 TTBR0_EL1.ASID defines the ASID. + /// + /// 1 TTBR1_EL1.ASID defines the ASID. + A1 OFFSET(22) NUMBITS(1) [ + UseTTBR0ASID = 0b0, + UseTTBR1ASID = 0b1 + ], + + /// The size offset of the memory region addressed by TTBR1_EL1. The + /// region size is 2^(64-T1SZ) bytes. + /// + /// The maximum and minimum possible values for T1SZ depend on the level + /// of translation table and the memory translation granule size, as + /// described in the AArch64 Virtual Memory System Architecture chapter. + T1SZ OFFSET(16) NUMBITS(6) [], + /// Granule size for the TTBR0_EL1. /// /// 00 4KiB diff --git a/crate/aarch64/src/regs/ttbr0_el1.rs b/crate/aarch64/src/regs/ttbr0_el1.rs index c111256..a29ff79 100644 --- a/crate/aarch64/src/regs/ttbr0_el1.rs +++ b/crate/aarch64/src/regs/ttbr0_el1.rs @@ -47,6 +47,11 @@ impl RegisterReadWrite for Reg { } impl Reg { + #[inline] + pub fn get_baddr(&self) -> u64 { + self.read(TTBR0_EL1::BADDR) << 1 + } + #[inline] pub fn set_baddr(&self, addr: u64) { self.write(TTBR0_EL1::BADDR.val(addr >> 1)); diff --git a/crate/aarch64/src/regs/ttbr1_el1.rs b/crate/aarch64/src/regs/ttbr1_el1.rs new file mode 100644 index 0000000..7df383c --- /dev/null +++ b/crate/aarch64/src/regs/ttbr1_el1.rs @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2018 by the author(s) + * + * ============================================================================= + * + * Licensed under either of + * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + * - MIT License (http://opensource.org/licenses/MIT) + * at your option. + * + * ============================================================================= + * + * Author(s): + * - Andre Richter + */ + +//! Translation Table Base Register 1 - EL1 +//! +//! Holds the base address of the translation table for the initial lookup for +//! stage 1 of the translation of an address from the upper VA range in the +//! EL1&0 translation regime, and other information for this translation regime. + +use register::cpu::RegisterReadWrite; + +register_bitfields! {u64, + TTBR1_EL1 [ + /// An ASID for the translation table base address. The TCR_EL1.A1 field + /// selects either TTBR0_EL1.ASID or TTBR1_EL1.ASID. + /// + /// If the implementation has only 8 bits of ASID, then the upper 8 bits + /// of this field are RES 0. + ASID OFFSET(48) NUMBITS(16) [], + + /// Translation table base address + BADDR OFFSET(1) NUMBITS(47) [], + + /// Common not Private + CnP OFFSET(0) NUMBITS(1) [] + ] +} + +pub struct Reg; + +impl RegisterReadWrite for Reg { + sys_coproc_read_raw!(u64, "TTBR1_EL1"); + sys_coproc_write_raw!(u64, "TTBR1_EL1"); +} + +impl Reg { + #[inline] + pub fn get_baddr(&self) -> u64 { + self.read(TTBR1_EL1::BADDR) << 1 + } + + #[inline] + pub fn set_baddr(&self, addr: u64) { + self.write(TTBR1_EL1::BADDR.val(addr >> 1)); + } +} + +pub static TTBR1_EL1: Reg = Reg {}; diff --git a/kernel/src/arch/aarch64/interrupt/context.rs b/kernel/src/arch/aarch64/interrupt/context.rs index df2b5b3..5bff366 100644 --- a/kernel/src/arch/aarch64/interrupt/context.rs +++ b/kernel/src/arch/aarch64/interrupt/context.rs @@ -63,12 +63,12 @@ extern { struct ContextData { x19to29: [usize; 11], lr: usize, - ttbr0: usize, + ttbr1: usize, } impl ContextData { - fn new(ttbr0: usize) -> Self { - ContextData { lr: __trapret as usize, ttbr0, ..ContextData::default() } + fn new(ttbr1: usize) -> Self { + ContextData { lr: __trapret as usize, ttbr1, ..ContextData::default() } } } @@ -98,7 +98,7 @@ impl Context { stp x25, x26, [x8], #16 stp x27, x28, [x8], #16 stp x29, lr, [x8], #16 - mrs x9, ttbr0_el1 + mrs x9, ttbr1_el1 str x9, [x8], #8 ldr x8, [x1] @@ -111,7 +111,8 @@ impl Context { ldr x9, [x8], #8 mov sp, x8 - msr ttbr0_el1, x9 // set new page directory + msr ttbr1_el1, x9 // set new page directory + // TODO: with ASID we needn't flush TLB dsb ishst // ensure write has completed tlbi vmalle1is // invalidate the TLB entry for the entry that changes dsb ish // ensure TLB invalidation is complete @@ -126,21 +127,21 @@ impl Context { Context(0) } - pub unsafe fn new_kernel_thread(entry: extern fn(usize) -> !, arg: usize, kstack_top: usize, ttbr0: usize) -> Self { + pub unsafe fn new_kernel_thread(entry: extern fn(usize) -> !, arg: usize, kstack_top: usize, ttbr: usize) -> Self { InitStack { - context: ContextData::new(ttbr0), + context: ContextData::new(ttbr), tf: TrapFrame::new_kernel_thread(entry, arg, kstack_top), }.push_at(kstack_top) } - pub unsafe fn new_user_thread(entry_addr: usize, ustack_top: usize, kstack_top: usize, is32: bool, ttbr0: usize) -> Self { + pub unsafe fn new_user_thread(entry_addr: usize, ustack_top: usize, kstack_top: usize, is32: bool, ttbr: usize) -> Self { InitStack { - context: ContextData::new(ttbr0), + context: ContextData::new(ttbr), // TODO: set ASID tf: TrapFrame::new_user_thread(entry_addr, ustack_top), }.push_at(kstack_top) } - pub unsafe fn new_fork(tf: &TrapFrame, kstack_top: usize, ttbr0: usize) -> Self { + pub unsafe fn new_fork(tf: &TrapFrame, kstack_top: usize, ttbr: usize) -> Self { InitStack { - context: ContextData::new(ttbr0), + context: ContextData::new(ttbr), // TODO: set ASID tf: { let mut tf = tf.clone(); tf.x0 = 0; diff --git a/kernel/src/arch/aarch64/memory.rs b/kernel/src/arch/aarch64/memory.rs index 3f93e31..1c486d2 100644 --- a/kernel/src/arch/aarch64/memory.rs +++ b/kernel/src/arch/aarch64/memory.rs @@ -38,14 +38,25 @@ pub fn init_mmu_early() { // Configure various settings of stage 1 of the EL1 translation regime. let ips = ID_AA64MMFR0_EL1.read(ID_AA64MMFR0_EL1::PARange); TCR_EL1.write( - TCR_EL1::TBI0::Ignored - + TCR_EL1::IPS.val(ips) - + TCR_EL1::TG0::KiB_4 // 4 KiB granule - + TCR_EL1::SH0::Inner - + TCR_EL1::ORGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable - + TCR_EL1::IRGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable - + TCR_EL1::EPD0::EnableTTBR0Walks - + TCR_EL1::T0SZ.val(16), // Start walks at level 2 + TCR_EL1::TBI1::Ignored + + TCR_EL1::TBI0::Ignored + + TCR_EL1::AS::Bits_16 + + TCR_EL1::IPS.val(ips) + + + TCR_EL1::TG1::KiB_4 + + TCR_EL1::SH1::Inner + + TCR_EL1::ORGN1::WriteBack_ReadAlloc_WriteAlloc_Cacheable + + TCR_EL1::IRGN1::WriteBack_ReadAlloc_WriteAlloc_Cacheable + + TCR_EL1::EPD1::EnableTTBR1Walks + + TCR_EL1::A1::UseTTBR1ASID + + TCR_EL1::T1SZ.val(16) + + + TCR_EL1::TG0::KiB_4 + + TCR_EL1::SH0::Inner + + TCR_EL1::ORGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable + + TCR_EL1::IRGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable + + TCR_EL1::EPD0::EnableTTBR0Walks + + TCR_EL1::T0SZ.val(16), ); // Switch the MMU on. @@ -63,7 +74,7 @@ pub fn init_mmu_early() { fn init_frame_allocator() { use bit_allocator::BitAlloc; use core::ops::Range; - use consts::{MEMORY_OFFSET}; + use consts::MEMORY_OFFSET; let (start, end) = memory_map().expect("failed to find memory map"); let mut ba = FRAME_ALLOCATOR.lock(); @@ -116,7 +127,7 @@ fn remap_the_kernel() { /// /// This function is expected to return `Some` under all normal cirumstances. fn memory_map() -> Option<(usize, usize)> { - let binary_end = unsafe { _end as u32 }; + let binary_end = _end as u32; let mut atags: Atags = Atags::get(); while let Some(atag) = atags.next() { diff --git a/kernel/src/arch/aarch64/paging.rs b/kernel/src/arch/aarch64/paging.rs index c7c4e99..09606a6 100644 --- a/kernel/src/arch/aarch64/paging.rs +++ b/kernel/src/arch/aarch64/paging.rs @@ -5,7 +5,7 @@ use memory::{active_table, alloc_frame, alloc_stack, dealloc_frame}; use ucore_memory::memory_set::*; use ucore_memory::PAGE_SIZE; use ucore_memory::paging::*; -use aarch64::asm::{tlb_invalidate, tlb_invalidate_all, ttbr0_el1_read, ttbr0_el1_write}; +use aarch64::asm::{tlb_invalidate, tlb_invalidate_all, ttbr_el1_read, ttbr_el1_write}; use aarch64::{PhysAddr, VirtAddr}; use aarch64::paging::{Mapper, PageTable as Aarch64PageTable, PageTableEntry, PageTableFlags as EF, RecursivePageTable}; use aarch64::paging::{FrameAllocator, FrameDeallocator, Page, PageRange, PhysFrame as Frame, Size4KiB, Size2MiB}; @@ -122,7 +122,7 @@ pub fn setup_page_table(frame_lvl4: Frame, frame_lvl3: Frame, frame_lvl2: Frame) // } // } - ttbr0_el1_write(frame_lvl4); + ttbr_el1_write(0, frame_lvl4); tlb_invalidate_all(); } @@ -291,9 +291,9 @@ impl InactivePageTable for InactivePageTable0 { type Active = ActivePageTable; fn new() -> Self { - let mut pt = Self::new_bare(); - pt.map_kernel(); - pt + // When the new InactivePageTable is created for the user MemorySet, it's use ttbr1 as the + // TTBR. And the kernel TTBR ttbr0 will never changed, so we needn't call map_kernel() + Self::new_bare() } fn new_bare() -> Self { @@ -308,7 +308,7 @@ impl InactivePageTable for InactivePageTable0 { } fn edit(&mut self, f: impl FnOnce(&mut Self::Active)) { - active_table().with_temporary_map(&ttbr0_el1_read().0, |active_table, p4_table: &mut Aarch64PageTable| { + active_table().with_temporary_map(&ttbr_el1_read(0), |active_table, p4_table: &mut Aarch64PageTable| { let backup = p4_table[RECURSIVE_INDEX].clone(); // overwrite recursive mapping @@ -325,33 +325,34 @@ impl InactivePageTable for InactivePageTable0 { } unsafe fn activate(&self) { - let old_frame = ttbr0_el1_read().0; + let old_frame = ttbr_el1_read(0); let new_frame = self.p4_frame.clone(); - debug!("switch table {:?} -> {:?}", old_frame, new_frame); + debug!("switch TTBR0 {:?} -> {:?}", old_frame, new_frame); if old_frame != new_frame { - ttbr0_el1_write(new_frame); + ttbr_el1_write(0, new_frame); tlb_invalidate_all(); } } unsafe fn with(&self, f: impl FnOnce()) { - let old_frame = ttbr0_el1_read().0; + // Just need to switch the user TTBR + let old_frame = ttbr_el1_read(1); let new_frame = self.p4_frame.clone(); - debug!("switch table {:?} -> {:?}", old_frame, new_frame); + debug!("switch TTBR1 {:?} -> {:?}", old_frame, new_frame); if old_frame != new_frame { - ttbr0_el1_write(new_frame); + ttbr_el1_write(1, new_frame); tlb_invalidate_all(); } f(); - debug!("switch table {:?} -> {:?}", new_frame, old_frame); + debug!("switch TTBR1 {:?} -> {:?}", new_frame, old_frame); if old_frame != new_frame { - ttbr0_el1_write(old_frame); + ttbr_el1_write(1, old_frame); tlb_invalidate_all(); } } fn token(&self) -> usize { - self.p4_frame.start_address().as_u64() as usize // as CR3 + self.p4_frame.start_address().as_u64() as usize // as TTBRx_EL1 } fn alloc_frame() -> Option { diff --git a/kernel/src/consts.rs b/kernel/src/consts.rs index e879173..487b6a6 100644 --- a/kernel/src/consts.rs +++ b/kernel/src/consts.rs @@ -136,7 +136,7 @@ mod aarch64 { pub const KERNEL_PML4: usize = 0; pub const KERNEL_HEAP_SIZE: usize = 8 * 1024 * 1024; pub const MEMORY_OFFSET: usize = 0; - pub const USER_STACK_OFFSET: usize = 0x3000_0000; + pub const USER_STACK_OFFSET: usize = 0xffff_ffff_0000_0000; pub const USER_STACK_SIZE: usize = 1 * 1024 * 1024; pub const USER32_STACK_OFFSET: usize = USER_STACK_OFFSET; } diff --git a/kernel/src/process/context.rs b/kernel/src/process/context.rs index f20d209..088b1b8 100644 --- a/kernel/src/process/context.rs +++ b/kernel/src/process/context.rs @@ -142,6 +142,10 @@ fn memory_set_from<'a>(elf: &'a ElfFile<'a>) -> MemorySet { ProgramHeader::Ph32(ph) => (ph.virtual_addr as usize, ph.mem_size as usize, ph.flags), ProgramHeader::Ph64(ph) => (ph.virtual_addr as usize, ph.mem_size as usize, ph.flags), }; + + #[cfg(target_arch = "aarch64")] + assert_eq!((virt_addr >> 48), 0xffff, "Segment Fault"); + set.push(MemoryArea::new(virt_addr, virt_addr + mem_size, memory_attr_from(flags), "")); } set From 33d4b6975b54bdc77a14c1133cd6793cc976139e Mon Sep 17 00:00:00 2001 From: equation314 Date: Fri, 23 Nov 2018 15:23:00 +0800 Subject: [PATCH 09/22] aarch64/mmu: use DSB instead of TLB flush after modifying PTE --- crate/aarch64/src/barrier.rs | 4 ++++ crate/aarch64/src/paging/recursive.rs | 6 +++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/crate/aarch64/src/barrier.rs b/crate/aarch64/src/barrier.rs index 0e48209..3211480 100644 --- a/crate/aarch64/src/barrier.rs +++ b/crate/aarch64/src/barrier.rs @@ -52,8 +52,12 @@ macro_rules! dmb_dsb { } pub struct SY; +pub struct ISH; +pub struct ISHST; dmb_dsb!(SY); +dmb_dsb!(ISH); +dmb_dsb!(ISHST); impl sealed::Isb for SY { #[inline(always)] diff --git a/crate/aarch64/src/paging/recursive.rs b/crate/aarch64/src/paging/recursive.rs index ee61cba..31039d5 100644 --- a/crate/aarch64/src/paging/recursive.rs +++ b/crate/aarch64/src/paging/recursive.rs @@ -1,13 +1,13 @@ #![cfg(target_arch = "aarch64")] -use asm::tlb_invalidate; use paging::{ frame_alloc::FrameAllocator, page_table::{FrameError, PageTable, PageTableEntry, PageTableFlags}, NotGiantPageSize, Page, PageSize, PhysFrame, Size4KiB, }; use paging::page_table::PageTableFlags as Flags; -use asm::ttbr_el1_read; +use asm::{ttbr_el1_read, tlb_invalidate}; +use barrier; use ux::u9; use addr::{PhysAddr, VirtAddr}; @@ -233,7 +233,7 @@ impl<'a> RecursivePageTable<'a> { let page_table_ptr = next_table_page.start_address().as_mut_ptr(); let page_table: &mut PageTable = unsafe { &mut *(page_table_ptr) }; if created { - tlb_invalidate(next_table_page.start_address()); + unsafe { barrier::dsb(barrier::ISHST); } page_table.zero(); } Ok(page_table) From de6354ddd338060726d6cce659b3b7796ed97146 Mon Sep 17 00:00:00 2001 From: equation314 Date: Fri, 23 Nov 2018 23:25:06 +0800 Subject: [PATCH 10/22] aarch64/mmu: use ASID to avoid flush TLB frequently in context switch --- crate/aarch64/src/asm.rs | 9 ++ kernel/src/arch/aarch64/interrupt/context.rs | 109 ++++++++++++++----- kernel/src/arch/aarch64/paging.rs | 10 +- 3 files changed, 92 insertions(+), 36 deletions(-) diff --git a/crate/aarch64/src/asm.rs b/crate/aarch64/src/asm.rs index a5b16d5..1c1ee6d 100644 --- a/crate/aarch64/src/asm.rs +++ b/crate/aarch64/src/asm.rs @@ -177,3 +177,12 @@ pub fn ttbr_el1_write(which: u8, frame: PhysFrame) { _ => {} }; } + +pub fn ttbr_el1_write_asid(which: u8, asid: u16, frame: PhysFrame) { + let baddr = frame.start_address().as_u64(); + match which { + 0 => TTBR0_EL1.write(TTBR0_EL1::ASID.val(asid as u64) + TTBR0_EL1::BADDR.val(baddr >> 1)), + 1 => TTBR1_EL1.write(TTBR1_EL1::ASID.val(asid as u64) + TTBR1_EL1::BADDR.val(baddr >> 1)), + _ => {} + }; +} diff --git a/kernel/src/arch/aarch64/interrupt/context.rs b/kernel/src/arch/aarch64/interrupt/context.rs index 5bff366..fe93839 100644 --- a/kernel/src/arch/aarch64/interrupt/context.rs +++ b/kernel/src/arch/aarch64/interrupt/context.rs @@ -1,12 +1,20 @@ //! TrapFrame and context definitions for aarch64. +extern crate aarch64; + +use spin::{Mutex}; +use aarch64::barrier; +use aarch64::addr::PhysAddr; +use aarch64::paging::PhysFrame; +use aarch64::asm::{tlb_invalidate_all, ttbr_el1_write_asid}; + #[repr(C)] #[derive(Default, Debug, Copy, Clone)] pub struct TrapFrame { pub elr: usize, pub spsr: usize, pub sp: usize, - pub tpidr: usize, + pub tpidr: usize, // currently unused // pub q0to31: [u128; 32], // disable SIMD/FP registers pub x1to29: [usize; 29], pub __reserved: usize, @@ -47,10 +55,14 @@ pub struct InitStack { } impl InitStack { - unsafe fn push_at(self, stack_top: usize) -> Context { + unsafe fn push_at(self, stack_top: usize, ttbr: usize) -> Context { let ptr = (stack_top as *mut Self).offset(-1); *ptr = self; - Context(ptr as usize) + Context { + stack_top: ptr as usize, + ttbr: PhysFrame::containing_address(PhysAddr::new(ttbr as u64)), + asid: Asid::default(), + } } } @@ -63,18 +75,20 @@ extern { struct ContextData { x19to29: [usize; 11], lr: usize, - ttbr1: usize, } impl ContextData { - fn new(ttbr1: usize) -> Self { - ContextData { lr: __trapret as usize, ttbr1, ..ContextData::default() } + fn new() -> Self { + ContextData { lr: __trapret as usize, ..ContextData::default() } } } - #[derive(Debug)] -pub struct Context(usize); +pub struct Context { + stack_top: usize, + ttbr: PhysFrame, + asid: Asid, +} impl Context { /// Switch to another kernel thread. @@ -86,10 +100,10 @@ impl Context { /// Pop all callee-saved registers, then return to the target. #[naked] #[inline(never)] - pub unsafe extern fn switch(&mut self, target: &mut Self) { + unsafe extern fn __switch(self_stack: &mut usize, target_stack: &mut usize) { asm!( " - mov x10, #-(13 * 8) + mov x10, #-(12 * 8) add x8, sp, x10 str x8, [x0] stp x19, x20, [x8], #16 // store callee-saved registers @@ -98,8 +112,6 @@ impl Context { stp x25, x26, [x8], #16 stp x27, x28, [x8], #16 stp x29, lr, [x8], #16 - mrs x9, ttbr1_el1 - str x9, [x8], #8 ldr x8, [x1] ldp x19, x20, [x8], #16 // restore callee-saved registers @@ -108,45 +120,88 @@ impl Context { ldp x25, x26, [x8], #16 ldp x27, x28, [x8], #16 ldp x29, lr, [x8], #16 - ldr x9, [x8], #8 mov sp, x8 - msr ttbr1_el1, x9 // set new page directory - // TODO: with ASID we needn't flush TLB - dsb ishst // ensure write has completed - tlbi vmalle1is // invalidate the TLB entry for the entry that changes - dsb ish // ensure TLB invalidation is complete - isb // synchronize context on this processor - str xzr, [x1] ret" : : : : "volatile" ); } + pub unsafe fn switch(&mut self, target: &mut Self) { + target.asid = ASID_ALLOCATOR.lock().alloc(target.asid); + + // with ASID we needn't flush TLB frequently + ttbr_el1_write_asid(1, target.asid.value, target.ttbr); + barrier::dsb(barrier::ISH); + Self::__switch(&mut self.stack_top, &mut target.stack_top); + } + pub unsafe fn null() -> Self { - Context(0) + Context { + stack_top: 0, + ttbr: PhysFrame::containing_address(PhysAddr::new(0)), + asid: Asid::default(), + } } pub unsafe fn new_kernel_thread(entry: extern fn(usize) -> !, arg: usize, kstack_top: usize, ttbr: usize) -> Self { InitStack { - context: ContextData::new(ttbr), + context: ContextData::new(), tf: TrapFrame::new_kernel_thread(entry, arg, kstack_top), - }.push_at(kstack_top) + }.push_at(kstack_top, ttbr) } pub unsafe fn new_user_thread(entry_addr: usize, ustack_top: usize, kstack_top: usize, is32: bool, ttbr: usize) -> Self { InitStack { - context: ContextData::new(ttbr), // TODO: set ASID + context: ContextData::new(), tf: TrapFrame::new_user_thread(entry_addr, ustack_top), - }.push_at(kstack_top) + }.push_at(kstack_top, ttbr) } pub unsafe fn new_fork(tf: &TrapFrame, kstack_top: usize, ttbr: usize) -> Self { InitStack { - context: ContextData::new(ttbr), // TODO: set ASID + context: ContextData::new(), tf: { let mut tf = tf.clone(); tf.x0 = 0; tf }, - }.push_at(kstack_top) + }.push_at(kstack_top, ttbr) } } + +const ASID_MASK: u16 = 0xffff; + +#[derive(Debug, Copy, Clone, Default)] +#[repr(C)] +struct Asid { + value: u16, + generation: u16, +} + +struct AsidAllocator(Asid); + +impl AsidAllocator { + fn new() -> Self { + AsidAllocator(Asid { value: 0, generation: 1 }) + } + + fn alloc(&mut self, old_asid: Asid) -> Asid { + if self.0.generation == old_asid.generation { + return old_asid; + } + + if self.0.value == ASID_MASK { + self.0.value = 0; + self.0.generation = self.0.generation.wrapping_add(1); + if self.0.generation == 0 { + self.0.generation += 1; + } + tlb_invalidate_all(); + } + self.0.value += 1; + return self.0; + } +} + +lazy_static! { + static ref ASID_ALLOCATOR: Mutex = Mutex::new(AsidAllocator::new()); +} diff --git a/kernel/src/arch/aarch64/paging.rs b/kernel/src/arch/aarch64/paging.rs index 09606a6..f1df051 100644 --- a/kernel/src/arch/aarch64/paging.rs +++ b/kernel/src/arch/aarch64/paging.rs @@ -259,15 +259,7 @@ impl Entry for PageEntry { fn set_swapped(&mut self, value: bool) { self.as_flags().set(EF::SWAPPED, value); } fn set_user(&mut self, value: bool) { self.as_flags().set(EF::USER_ACCESSIBLE, value); - if value { - let mut addr = self as *const _ as usize; - for _ in 0..3 { - // Upper level entry - addr = ((addr >> 9) & 0o777_777_777_7770) | (RECURSIVE_INDEX << 39); - // set USER_ACCESSIBLE - unsafe { (*(addr as *mut EF)).insert(EF::USER_ACCESSIBLE) }; - } - } + self.as_flags().set(EF::NONE_GLOBAL, value); // set non-global to use ASID } fn execute(&self) -> bool { !self.0.flags().contains(EF::UXN) } fn set_execute(&mut self, value: bool) { self.as_flags().set(EF::UXN, !value); } From f9e47b2fd8f142cbbbb4bd98f826a85149ff96ef Mon Sep 17 00:00:00 2001 From: equation314 Date: Sun, 25 Nov 2018 00:29:39 +0800 Subject: [PATCH 11/22] aarch64/mmu: refactor PageTableFlags --- crate/aarch64/src/paging/page_table.rs | 172 ++++++++++++++++++------- crate/aarch64/src/paging/recursive.rs | 7 +- kernel/src/arch/aarch64/paging.rs | 57 ++++---- kernel/src/consts.rs | 2 +- 4 files changed, 163 insertions(+), 75 deletions(-) diff --git a/crate/aarch64/src/paging/page_table.rs b/crate/aarch64/src/paging/page_table.rs index c06123e..2047e6b 100644 --- a/crate/aarch64/src/paging/page_table.rs +++ b/crate/aarch64/src/paging/page_table.rs @@ -1,12 +1,27 @@ use core::fmt; use core::ops::{Index, IndexMut}; -use super::{PageSize, PhysFrame, Size4KiB}; +use super::PhysFrame; use addr::PhysAddr; use usize_conversions::usize_from; use ux::*; +use register::FieldValue; +use register::cpu::RegisterReadWrite; + +/// Memory attribute fields mask +const MEMORY_ATTR_MASK: u64 = (MEMORY_ATTRIBUTE::SH.mask << MEMORY_ATTRIBUTE::SH.shift) + | (MEMORY_ATTRIBUTE::AttrIndx.mask << MEMORY_ATTRIBUTE::AttrIndx.shift); +/// Output address mask +const ADDR_MASK: u64 = 0x0000_ffff_ffff_f000; +/// Other flags mask +const FLAGS_MASK: u64 = !(MEMORY_ATTR_MASK | ADDR_MASK); + +/// Memory attribute fields +type PageTableAttributeFieldValue = FieldValue; +pub struct PageTableAttribute(PageTableAttributeFieldValue); + /// The error returned by the `PageTableEntry::frame` method. #[derive(Debug, Clone, Copy, PartialEq)] pub enum FrameError { @@ -24,6 +39,18 @@ pub struct PageTableEntry { pub entry: u64, } +impl RegisterReadWrite for PageTableEntry { + #[inline] + fn get(&self) -> u64 { + self.entry + } + + #[inline] + fn set(&self, value: u64) { + unsafe { *(&self.entry as *const u64 as *mut u64) = value } + } +} + impl PageTableEntry { /// Returns whether this entry is zero. pub fn is_unused(&self) -> bool { @@ -42,7 +69,16 @@ impl PageTableEntry { /// Returns the physical address mapped by this entry, might be zero. pub fn addr(&self) -> PhysAddr { - PhysAddr::new(self.entry & 0x0000_ffff_ffff_f000) + PhysAddr::new(self.entry & ADDR_MASK) + } + + /// Returns the memory attribute fields of this entry. + pub fn attr(&self) -> PageTableAttribute { + PageTableAttribute(PageTableAttributeFieldValue::new( + MEMORY_ATTR_MASK, + 0, + self.entry & MEMORY_ATTR_MASK, + )) } /// Returns the physical frame mapped by this entry. @@ -53,30 +89,36 @@ impl PageTableEntry { /// - `FrameError::HugeFrame` if the entry has the `HUGE_PAGE` flag set (for huge pages the /// `addr` function must be used) pub fn frame(&self) -> Result { - if !self.flags().contains(PageTableFlags::PRESENT) { + if !self.flags().contains(PageTableFlags::VALID) { Err(FrameError::FrameNotPresent) - } else if self.flags().contains(PageTableFlags::HUGE_PAGE) { + } else if !self.flags().contains(PageTableFlags::TABLE_OR_PAGE) { + // is a huge page (block) Err(FrameError::HugeFrame) } else { Ok(PhysFrame::containing_address(self.addr())) } } + /// Map the entry to the specified physical frame with the specified flags. + pub fn set_frame(&mut self, frame: PhysFrame, flags: PageTableFlags) { + // is not a huge page (block) + assert!(flags.contains(PageTableFlags::TABLE_OR_PAGE)); + self.set(frame.start_address().as_u64() | flags.bits()); + } + /// Map the entry to the specified physical address with the specified flags. - pub fn set_addr(&mut self, addr: PhysAddr, flags: PageTableFlags) { - assert!(addr.is_aligned(Size4KiB::SIZE)); - self.entry = (addr.as_u64()) | flags.bits(); + pub fn modify_addr(&mut self, addr: PhysAddr) { + self.entry = (self.entry & !ADDR_MASK) | addr.as_u64(); } - /// Map the entry to the specified physical frame with the specified flags. - pub fn set_frame(&mut self, frame: PhysFrame, flags: PageTableFlags) { - assert!(!flags.contains(PageTableFlags::HUGE_PAGE)); - self.set_addr(frame.start_address(), flags) + /// Sets the flags of this entry. + pub fn modify_flags(&mut self, flags: PageTableFlags) { + self.entry = (self.entry & !FLAGS_MASK) | flags.bits(); } /// Sets the flags of this entry. - pub fn set_flags(&mut self, flags: PageTableFlags) { - self.entry = self.addr().as_u64() | flags.bits(); + pub fn modify_attr(&mut self, attr: PageTableAttribute) { + self.entry = (self.entry & !MEMORY_ATTR_MASK) | attr.0.value; } } @@ -90,42 +132,78 @@ impl fmt::Debug for PageTableEntry { } } +register_bitfields! {u64, + // Memory attribute fields in the VMSAv8-64 translation table format descriptors (Page 2148~2152) + MEMORY_ATTRIBUTE [ + /// Shareability field + SH OFFSET(8) NUMBITS(2) [ + NonShareable = 0b00, + OuterShareable = 0b10, + InnerShareable = 0b11 + ], + + /// Memory attributes index into the MAIR_EL1 register + AttrIndx OFFSET(2) NUMBITS(3) [] + ] +} + bitflags! { /// Possible flags for a page table entry. pub struct PageTableFlags: u64 { - const ALL = 0xffffffff_ffffffff; - const TYPE_MASK = 3 << 0; - // const TYPE_FAULT = 0 << 0; - const TYPE_PAGE = 3 << 0; - const TABLE_BIT = 1 << 1; - // const BLOCK_BIT = 0 << 1; - const PAGE_BIT = 1 << 1; - - const PRESENT = 1 << 0; - const USER_ACCESSIBLE = 1 << 6; /* AP[1] */ - const RDONLY = 1 << 7; /* AP[2] */ - const SHARED = 3 << 8; /* SH[1:0], inner shareable */ - const BIT_8 = 1 << 8; - const BIT_9 = 1 << 9; + // const SHARED = 3 << 8; /* SH[1:0], inner shareable */ + // const BIT_8 = 1 << 8; + // const BIT_9 = 1 << 9; // pub const ATTRIB_SH_NON_SHAREABLE: usize = 0x0 << 8; - const OUTER_SHAREABLE = 0b10 << 8; - const INNER_SHAREABLE = 0b11 << 8; - - const ACCESSED = 1 << 10; /* AF, Access Flag */ - const NONE_GLOBAL = 1 << 11; /* None Global */ - const GLOBAL = (!(1 << 11)); - const DBM = 1 << 51; /* Dirty Bit Management */ - const WRITE = 1 << 51; /* DBM */ - const CONT = 1 << 52; /* Contiguous range */ - const PXN = 1 << 53; /* Privileged XN */ - const UXN = 1 << 54; /* User XN */ - const HYP_XN = 1 << 54; /* HYP XN */ + // const OUTER_SHAREABLE = 0b10 << 8; + // const INNER_SHAREABLE = 0b11 << 8; + + /// identifies whether the descriptor is valid + const VALID = 1 << 0; + /// the descriptor type + /// 0, Block + /// 1, Table/Page + const TABLE_OR_PAGE = 1 << 1; + /// Access permission: accessable at EL0 + const AP_EL0 = 1 << 6; + /// Access permission: read-only + const AP_RO = 1 << 7; + /// Access flag + const AF = 1 << 10; + /// not global bit + const nG = 1 << 11; + /// Dirty Bit Modifier + const DBM = 1 << 51; + + /// A hint bit indicating that the translation table entry is one of a contiguous set or + /// entries + const Contiguous = 1 << 52; + /// Privileged Execute-never + const PXN = 1 << 53; + /// Execute-never/Unprivileged execute-never + const XN = 1 << 54; + + /// Software Dirty Bit Modifier + const WRITE = 1 << 51; + /// Software dirty bit const DIRTY = 1 << 55; + /// Software swapped bit const SWAPPED = 1 << 56; - const HUGE_PAGE = 1 << 57; - const PROT_NONE = 1 << 58; + /// Software writable shared bit for COW + const WRITABLE_SHARED = 1 << 57; + /// Software readonly shared bit for COW + const READONLY_SHARED = 1 << 58; + + /// Privileged Execute-never for table descriptors + const PXNTable = 1 << 59; + /// Execute-never/Unprivileged execute-never for table descriptors + const XNTable = 1 << 60; + } +} +impl Default for PageTableFlags { + fn default() -> Self { + Self::VALID | Self::TABLE_OR_PAGE | Self::AF | Self::WRITE | Self::PXN | Self::XN } } @@ -151,12 +229,12 @@ impl PageTable { } } - /// Setup identity map: VirtPage at pagenumber -> PhysFrame at pagenumber - /// pn: pagenumber = addr>>12 in riscv32. - pub fn map_identity(&mut self, p4num: usize, flags: PageTableFlags) { - let entry = self.entries[p4num].clone(); - self.entries[p4num].set_addr(entry.addr(), flags); - } + // Setup identity map: VirtPage at pagenumber -> PhysFrame at pagenumber + // pn: pagenumber = addr>>12 in riscv32. + // pub fn map_identity(&mut self, p4num: usize, flags: PageTableFlags) { + // let entry = self.entries[p4num].clone(); + // self.entries[p4num].set_addr(entry.addr(), flags); + // } } impl Index for PageTable { diff --git a/crate/aarch64/src/paging/recursive.rs b/crate/aarch64/src/paging/recursive.rs index 31039d5..e878f98 100644 --- a/crate/aarch64/src/paging/recursive.rs +++ b/crate/aarch64/src/paging/recursive.rs @@ -218,7 +218,7 @@ impl<'a> RecursivePageTable<'a> { if entry.is_unused() { if let Some(frame) = allocator.alloc() { - entry.set_frame(frame, Flags::PRESENT | Flags::WRITE | Flags::ACCESSED | Flags::PAGE_BIT); + entry.set_frame(frame, Flags::default()); created = true; } else { return Err(MapToError::FrameAllocationFailed); @@ -226,7 +226,8 @@ impl<'a> RecursivePageTable<'a> { } else { created = false; } - if entry.flags().contains(Flags::HUGE_PAGE) { + // is a huge page (block) + if !entry.flags().contains(Flags::TABLE_OR_PAGE) { return Err(MapToError::ParentEntryHugePage); } @@ -382,7 +383,7 @@ impl<'a> Mapper for RecursivePageTable<'a> { return Err(FlagUpdateError::PageNotMapped); } - p1[page.p1_index()].set_flags(flags); + p1[page.p1_index()].modify_flags(flags); Ok(MapperFlush::new(page)) } diff --git a/kernel/src/arch/aarch64/paging.rs b/kernel/src/arch/aarch64/paging.rs index f1df051..9986faa 100644 --- a/kernel/src/arch/aarch64/paging.rs +++ b/kernel/src/arch/aarch64/paging.rs @@ -8,7 +8,7 @@ use ucore_memory::paging::*; use aarch64::asm::{tlb_invalidate, tlb_invalidate_all, ttbr_el1_read, ttbr_el1_write}; use aarch64::{PhysAddr, VirtAddr}; use aarch64::paging::{Mapper, PageTable as Aarch64PageTable, PageTableEntry, PageTableFlags as EF, RecursivePageTable}; -use aarch64::paging::{FrameAllocator, FrameDeallocator, Page, PageRange, PhysFrame as Frame, Size4KiB, Size2MiB}; +use aarch64::paging::{FrameAllocator, FrameDeallocator, Page, PhysFrame as Frame, Size4KiB, Size2MiB}; register_bitfields! {u64, // AArch64 Reference Manual page 2150 @@ -167,7 +167,7 @@ impl PageTable for ActivePageTable { type Entry = PageEntry; fn map(&mut self, addr: usize, target: usize) -> &mut PageEntry { - let flags = EF::PRESENT | EF::WRITE | EF::ACCESSED | EF::UXN | EF::PAGE_BIT; + let flags = EF::default(); self.0.map_to(Page::of_addr(addr), Frame::of_addr(target), flags, &mut FrameAllocatorForAarch64) .unwrap().flush(); self.get_entry(addr) @@ -224,49 +224,58 @@ impl Entry for PageEntry { tlb_invalidate(addr); } - fn present(&self) -> bool { self.0.flags().contains(EF::PRESENT) } - fn accessed(&self) -> bool { self.0.flags().contains(EF::ACCESSED) } + fn present(&self) -> bool { self.0.flags().contains(EF::VALID) } + fn accessed(&self) -> bool { self.0.flags().contains(EF::AF) } fn writable(&self) -> bool { self.0.flags().contains(EF::WRITE) } fn dirty(&self) -> bool { self.hw_dirty() && self.sw_dirty() } - fn clear_accessed(&mut self) { self.as_flags().remove(EF::ACCESSED); } + fn clear_accessed(&mut self) { self.as_flags().remove(EF::AF); } fn clear_dirty(&mut self) { self.as_flags().remove(EF::DIRTY); - self.as_flags().insert(EF::RDONLY); + self.as_flags().insert(EF::AP_RO); } fn set_writable(&mut self, value: bool) { - self.as_flags().set(EF::RDONLY, !value); + self.as_flags().set(EF::AP_RO, !value); self.as_flags().set(EF::WRITE, value); } - fn set_present(&mut self, value: bool) { self.as_flags().set(EF::PRESENT, value); } + fn set_present(&mut self, value: bool) { self.as_flags().set(EF::VALID, value); } fn target(&self) -> usize { self.0.addr().as_u64() as usize } fn set_target(&mut self, target: usize) { - let flags = self.0.flags(); - self.0.set_addr(PhysAddr::new(target as u64), flags); + self.0.modify_addr(PhysAddr::new(target as u64)); } - fn writable_shared(&self) -> bool { self.0.flags().contains(EF::BIT_9) } - fn readonly_shared(&self) -> bool { self.0.flags().contains(EF::BIT_9) } + fn writable_shared(&self) -> bool { self.0.flags().contains(EF::WRITABLE_SHARED) } + fn readonly_shared(&self) -> bool { self.0.flags().contains(EF::READONLY_SHARED) } fn set_shared(&mut self, writable: bool) { let flags = self.as_flags(); - flags.set(EF::BIT_8, writable); - flags.set(EF::BIT_9, writable); + flags.set(EF::WRITABLE_SHARED, writable); + flags.set(EF::READONLY_SHARED, !writable); } - fn clear_shared(&mut self) { self.as_flags().remove(EF::BIT_8 | EF::BIT_9); } - fn user(&self) -> bool { self.0.flags().contains(EF::USER_ACCESSIBLE) } + fn clear_shared(&mut self) { self.as_flags().remove(EF::WRITABLE_SHARED | EF::READONLY_SHARED); } + fn user(&self) -> bool { self.0.flags().contains(EF::AP_EL0) } fn swapped(&self) -> bool { self.0.flags().contains(EF::SWAPPED) } fn set_swapped(&mut self, value: bool) { self.as_flags().set(EF::SWAPPED, value); } fn set_user(&mut self, value: bool) { - self.as_flags().set(EF::USER_ACCESSIBLE, value); - self.as_flags().set(EF::NONE_GLOBAL, value); // set non-global to use ASID + self.as_flags().set(EF::AP_EL0, value); + self.as_flags().set(EF::nG, value); // set non-global to use ASID + } + fn execute(&self) -> bool { + match self.user() { + true => !self.0.flags().contains(EF::XN), + false => !self.0.flags().contains(EF::PXN), + } + } + fn set_execute(&mut self, value: bool) { + match self.user() { + true => self.as_flags().set(EF::XN, !value), + false => self.as_flags().set(EF::PXN, !value), + } } - fn execute(&self) -> bool { !self.0.flags().contains(EF::UXN) } - fn set_execute(&mut self, value: bool) { self.as_flags().set(EF::UXN, !value); } } impl PageEntry { - fn read_only(&self) -> bool { self.0.flags().contains(EF::RDONLY) } + fn read_only(&self) -> bool { self.0.flags().contains(EF::AP_RO) } fn hw_dirty(&self) -> bool { self.writable() && !self.read_only() } fn sw_dirty(&self) -> bool { self.0.flags().contains(EF::DIRTY) } fn as_flags(&mut self) -> &mut EF { @@ -294,7 +303,7 @@ impl InactivePageTable for InactivePageTable0 { active_table().with_temporary_map(&frame, |_, table: &mut Aarch64PageTable| { table.zero(); // set up recursive mapping for the table - table[RECURSIVE_INDEX].set_frame(frame.clone(), EF::PRESENT | EF::WRITE | EF::ACCESSED | EF::PAGE_BIT); + table[RECURSIVE_INDEX].set_frame(frame.clone(), EF::default()); }); InactivePageTable0 { p4_frame: frame } } @@ -304,7 +313,7 @@ impl InactivePageTable for InactivePageTable0 { let backup = p4_table[RECURSIVE_INDEX].clone(); // overwrite recursive mapping - p4_table[RECURSIVE_INDEX].set_frame(self.p4_frame.clone(), EF::PRESENT | EF::WRITE | EF::ACCESSED | EF::PAGE_BIT); + p4_table[RECURSIVE_INDEX].set_frame(self.p4_frame.clone(), EF::default()); tlb_invalidate_all(); // execute f in the new context @@ -367,7 +376,7 @@ impl InactivePageTable0 { assert!(!e0.is_unused()); self.edit(|_| { - table[KERNEL_PML4].set_addr(e0.addr(), e0.flags() & EF::GLOBAL); + table[KERNEL_PML4].set_frame(Frame::containing_address(e0.addr()), EF::default()); }); } } diff --git a/kernel/src/consts.rs b/kernel/src/consts.rs index 487b6a6..b2be26c 100644 --- a/kernel/src/consts.rs +++ b/kernel/src/consts.rs @@ -136,7 +136,7 @@ mod aarch64 { pub const KERNEL_PML4: usize = 0; pub const KERNEL_HEAP_SIZE: usize = 8 * 1024 * 1024; pub const MEMORY_OFFSET: usize = 0; - pub const USER_STACK_OFFSET: usize = 0xffff_ffff_0000_0000; + pub const USER_STACK_OFFSET: usize = 0xffff_8000_0000_0000; pub const USER_STACK_SIZE: usize = 1 * 1024 * 1024; pub const USER32_STACK_OFFSET: usize = USER_STACK_OFFSET; } From a8b334123adf8f3b56e097efbfde194c70e8d499 Mon Sep 17 00:00:00 2001 From: equation314 Date: Sun, 25 Nov 2018 17:58:17 +0800 Subject: [PATCH 12/22] aarch64/mmu: add memory region attribute config --- crate/aarch64/src/paging/memory_attribute.rs | 64 ++++++++++ crate/aarch64/src/paging/mod.rs | 6 + crate/aarch64/src/paging/page_table.rs | 51 ++++---- crate/aarch64/src/paging/recursive.rs | 11 +- crate/aarch64/src/regs/mair_el1.rs | 67 ++++------- crate/aarch64/src/regs/mod.rs | 2 +- kernel/src/arch/aarch64/board/raspi3/mod.rs | 2 + kernel/src/arch/aarch64/memory.rs | 13 +- kernel/src/arch/aarch64/paging.rs | 119 ++++--------------- 9 files changed, 155 insertions(+), 180 deletions(-) create mode 100644 crate/aarch64/src/paging/memory_attribute.rs diff --git a/crate/aarch64/src/paging/memory_attribute.rs b/crate/aarch64/src/paging/memory_attribute.rs new file mode 100644 index 0000000..beb9b69 --- /dev/null +++ b/crate/aarch64/src/paging/memory_attribute.rs @@ -0,0 +1,64 @@ +//!Memory region attributes (D4.5, page 2174) + +use super::{PageTableAttribute, MEMORY_ATTRIBUTE}; +use regs::*; + +pub trait MairType { + const INDEX: u64; + + #[inline] + fn config_value() -> u64; + + #[inline] + fn attr_value() -> PageTableAttribute; +} + +pub enum MairDevice {} +pub enum MairNormal {} +pub enum MairNormalNonCacheable {} + +impl MairType for MairDevice { + const INDEX: u64 = 0; + + #[inline] + fn config_value() -> u64 { + (MAIR_ATTR::Attr_HIGH::Device + MAIR_ATTR::Attr_LOW_DEVICE::Device_nGnRE).value + } + + #[inline] + fn attr_value() -> PageTableAttribute { + MEMORY_ATTRIBUTE::SH::OuterShareable + MEMORY_ATTRIBUTE::AttrIndx.val(Self::INDEX) + } +} + +impl MairType for MairNormal { + const INDEX: u64 = 1; + + #[inline] + fn config_value() -> u64 { + (MAIR_ATTR::Attr_HIGH::Memory_OuterWriteBack_NonTransient_ReadAlloc_WriteAlloc + + MAIR_ATTR::Attr_LOW_MEMORY::InnerWriteBack_NonTransient_ReadAlloc_WriteAlloc) + .value + } + + #[inline] + fn attr_value() -> PageTableAttribute { + MEMORY_ATTRIBUTE::SH::InnerShareable + MEMORY_ATTRIBUTE::AttrIndx.val(Self::INDEX) + } +} + +impl MairType for MairNormalNonCacheable { + const INDEX: u64 = 2; + + #[inline] + fn config_value() -> u64 { + (MAIR_ATTR::Attr_HIGH::Memory_OuterNonCacheable + + MAIR_ATTR::Attr_LOW_MEMORY::InnerNonCacheable) + .value + } + + #[inline] + fn attr_value() -> PageTableAttribute { + MEMORY_ATTRIBUTE::SH::NonShareable + MEMORY_ATTRIBUTE::AttrIndx.val(Self::INDEX) + } +} diff --git a/crate/aarch64/src/paging/mod.rs b/crate/aarch64/src/paging/mod.rs index c73d5b4..552d6f6 100644 --- a/crate/aarch64/src/paging/mod.rs +++ b/crate/aarch64/src/paging/mod.rs @@ -18,6 +18,8 @@ mod frame_alloc; mod page_table; mod recursive; +pub mod memory_attribute; + /// Trait for abstracting over the three possible block/page sizes on aarch64, 4KiB, 2MiB, 1GiB. pub trait PageSize: Copy + Eq + PartialOrd + Ord { /// The page size in bytes. @@ -366,6 +368,10 @@ impl PhysFrame { pub fn of_addr(address: usize) -> Self { Self::containing_address(PhysAddr::new(address as u64)) } + + pub fn range_of(begin: usize, end: usize) -> PhysFrameRange { + Self::range(Self::of_addr(begin), Self::of_addr(end - 1) + 1) + } } impl fmt::Debug for PhysFrame { diff --git a/crate/aarch64/src/paging/page_table.rs b/crate/aarch64/src/paging/page_table.rs index 2047e6b..6d33fa6 100644 --- a/crate/aarch64/src/paging/page_table.rs +++ b/crate/aarch64/src/paging/page_table.rs @@ -1,7 +1,7 @@ use core::fmt; use core::ops::{Index, IndexMut}; -use super::PhysFrame; +use super::{PhysFrame, PageSize}; use addr::PhysAddr; use usize_conversions::usize_from; @@ -19,8 +19,7 @@ const ADDR_MASK: u64 = 0x0000_ffff_ffff_f000; const FLAGS_MASK: u64 = !(MEMORY_ATTR_MASK | ADDR_MASK); /// Memory attribute fields -type PageTableAttributeFieldValue = FieldValue; -pub struct PageTableAttribute(PageTableAttributeFieldValue); +pub type PageTableAttribute = FieldValue; /// The error returned by the `PageTableEntry::frame` method. #[derive(Debug, Clone, Copy, PartialEq)] @@ -53,32 +52,33 @@ impl RegisterReadWrite for PageTableEntry { impl PageTableEntry { /// Returns whether this entry is zero. + #[inline] pub fn is_unused(&self) -> bool { self.entry == 0 } /// Sets this entry to zero. + #[inline] pub fn set_unused(&mut self) { self.entry = 0; } /// Returns the flags of this entry. + #[inline] pub fn flags(&self) -> PageTableFlags { PageTableFlags::from_bits_truncate(self.entry) } /// Returns the physical address mapped by this entry, might be zero. + #[inline] pub fn addr(&self) -> PhysAddr { PhysAddr::new(self.entry & ADDR_MASK) } /// Returns the memory attribute fields of this entry. + #[inline] pub fn attr(&self) -> PageTableAttribute { - PageTableAttribute(PageTableAttributeFieldValue::new( - MEMORY_ATTR_MASK, - 0, - self.entry & MEMORY_ATTR_MASK, - )) + PageTableAttribute::new(MEMORY_ATTR_MASK, 0, self.entry & MEMORY_ATTR_MASK) } /// Returns the physical frame mapped by this entry. @@ -99,11 +99,19 @@ impl PageTableEntry { } } - /// Map the entry to the specified physical frame with the specified flags. - pub fn set_frame(&mut self, frame: PhysFrame, flags: PageTableFlags) { - // is not a huge page (block) + /// Map the entry to the specified physical frame with the specified flags and memory attribute. + pub fn set_frame(&mut self, frame: PhysFrame, flags: PageTableFlags, attr: PageTableAttribute) { + // is not a block assert!(flags.contains(PageTableFlags::TABLE_OR_PAGE)); - self.set(frame.start_address().as_u64() | flags.bits()); + self.set(frame.start_address().as_u64() | flags.bits() | attr.value); + } + + /// The descriptor gives the base address of a block of memory, and the attributes for that + /// memory region. + pub fn set_block(&mut self, addr: PhysAddr, flags: PageTableFlags, attr: PageTableAttribute) { + // is a block + assert!(!flags.contains(PageTableFlags::TABLE_OR_PAGE)); + self.set(addr.align_down(S::SIZE).as_u64() | flags.bits() | attr.value); } /// Map the entry to the specified physical address with the specified flags. @@ -118,7 +126,7 @@ impl PageTableEntry { /// Sets the flags of this entry. pub fn modify_attr(&mut self, attr: PageTableAttribute) { - self.entry = (self.entry & !MEMORY_ATTR_MASK) | attr.0.value; + self.entry = (self.entry & !MEMORY_ATTR_MASK) | attr.value; } } @@ -128,6 +136,7 @@ impl fmt::Debug for PageTableEntry { f.field("value", &self.entry); f.field("addr", &self.addr()); f.field("flags", &self.flags()); + f.field("attr", &self.attr().value); f.finish() } } @@ -150,14 +159,6 @@ register_bitfields! {u64, bitflags! { /// Possible flags for a page table entry. pub struct PageTableFlags: u64 { - // const SHARED = 3 << 8; /* SH[1:0], inner shareable */ - // const BIT_8 = 1 << 8; - // const BIT_9 = 1 << 9; - - // pub const ATTRIB_SH_NON_SHAREABLE: usize = 0x0 << 8; - // const OUTER_SHAREABLE = 0b10 << 8; - // const INNER_SHAREABLE = 0b11 << 8; - /// identifies whether the descriptor is valid const VALID = 1 << 0; /// the descriptor type @@ -202,6 +203,7 @@ bitflags! { } impl Default for PageTableFlags { + #[inline] fn default() -> Self { Self::VALID | Self::TABLE_OR_PAGE | Self::AF | Self::WRITE | Self::PXN | Self::XN } @@ -228,13 +230,6 @@ impl PageTable { entry.set_unused(); } } - - // Setup identity map: VirtPage at pagenumber -> PhysFrame at pagenumber - // pn: pagenumber = addr>>12 in riscv32. - // pub fn map_identity(&mut self, p4num: usize, flags: PageTableFlags) { - // let entry = self.entries[p4num].clone(); - // self.entries[p4num].set_addr(entry.addr(), flags); - // } } impl Index for PageTable { diff --git a/crate/aarch64/src/paging/recursive.rs b/crate/aarch64/src/paging/recursive.rs index e878f98..a1633ff 100644 --- a/crate/aarch64/src/paging/recursive.rs +++ b/crate/aarch64/src/paging/recursive.rs @@ -5,7 +5,7 @@ use paging::{ page_table::{FrameError, PageTable, PageTableEntry, PageTableFlags}, NotGiantPageSize, Page, PageSize, PhysFrame, Size4KiB, }; -use paging::page_table::PageTableFlags as Flags; +use paging::{page_table::PageTableFlags as Flags, PageTableAttribute, memory_attribute::*}; use asm::{ttbr_el1_read, tlb_invalidate}; use barrier; use ux::u9; @@ -46,6 +46,7 @@ pub trait Mapper { page: Page, frame: PhysFrame, flags: PageTableFlags, + attr: PageTableAttribute, allocator: &mut A, ) -> Result, MapToError> where @@ -71,6 +72,7 @@ pub trait Mapper { &mut self, frame: PhysFrame, flags: PageTableFlags, + attr: PageTableAttribute, allocator: &mut A, ) -> Result, MapToError> where @@ -79,7 +81,7 @@ pub trait Mapper { Self: Mapper, { let page = Page::containing_address(VirtAddr::new(frame.start_address().as_u64())); - self.map_to(page, frame, flags, allocator) + self.map_to(page, frame, flags, attr, allocator) } } @@ -218,7 +220,7 @@ impl<'a> RecursivePageTable<'a> { if entry.is_unused() { if let Some(frame) = allocator.alloc() { - entry.set_frame(frame, Flags::default()); + entry.set_frame(frame, Flags::default(), MairNormal::attr_value()); created = true; } else { return Err(MapToError::FrameAllocationFailed); @@ -289,6 +291,7 @@ impl<'a> Mapper for RecursivePageTable<'a> { page: Page, frame: PhysFrame, flags: PageTableFlags, + attr: PageTableAttribute, allocator: &mut A, ) -> Result, MapToError> where @@ -309,7 +312,7 @@ impl<'a> Mapper for RecursivePageTable<'a> { if !p1[page.p1_index()].is_unused() { return Err(MapToError::PageAlreadyMapped); } - p1[page.p1_index()].set_frame(frame, flags); + p1[page.p1_index()].set_frame(frame, flags, attr); Ok(MapperFlush::new(page)) } diff --git a/crate/aarch64/src/regs/mair_el1.rs b/crate/aarch64/src/regs/mair_el1.rs index 2c7c7da..dbd7f9d 100644 --- a/crate/aarch64/src/regs/mair_el1.rs +++ b/crate/aarch64/src/regs/mair_el1.rs @@ -24,72 +24,49 @@ use register::cpu::RegisterReadWrite; register_bitfields! {u64, MAIR_EL1 [ - // TODO: Macrofy this - /// Attribute 7 - Attr7_HIGH OFFSET(60) NUMBITS(4) [], - Attr7_LOW_DEVICE OFFSET(56) NUMBITS(4) [], - Attr7_LOW_MEMORY OFFSET(56) NUMBITS(4) [], + Attr7 OFFSET(56) NUMBITS(8) [], /// Attribute 6 - Attr6_HIGH OFFSET(52) NUMBITS(4) [], - Attr6_LOW_DEVICE OFFSET(48) NUMBITS(4) [], - Attr6_LOW_MEMORY OFFSET(48) NUMBITS(4) [], + Attr6 OFFSET(48) NUMBITS(8) [], /// Attribute 5 - Attr5_HIGH OFFSET(44) NUMBITS(4) [], - Attr5_LOW_DEVICE OFFSET(40) NUMBITS(4) [], - Attr5_LOW_MEMORY OFFSET(40) NUMBITS(4) [], + Attr5 OFFSET(40) NUMBITS(8) [], /// Attribute 4 - Attr4_HIGH OFFSET(36) NUMBITS(4) [], - Attr4_LOW_DEVICE OFFSET(32) NUMBITS(4) [], - Attr4_LOW_MEMORY OFFSET(32) NUMBITS(4) [], + Attr4 OFFSET(32) NUMBITS(8) [], /// Attribute 3 - Attr3_HIGH OFFSET(28) NUMBITS(4) [], - Attr3_LOW_DEVICE OFFSET(24) NUMBITS(4) [], - Attr3_LOW_MEMORY OFFSET(24) NUMBITS(4) [], + Attr3 OFFSET(24) NUMBITS(8) [], /// Attribute 2 - Attr2_HIGH OFFSET(20) NUMBITS(4) [ - Device = 0b0000, - Memory_OuterNonCacheable = 0b0100, - Memory_OuterWriteBack_NonTransient_ReadAlloc_WriteAlloc = 0b1111 - ], - Attr2_LOW_DEVICE OFFSET(16) NUMBITS(4) [ - Device_nGnRE = 0b0100 - ], - Attr2_LOW_MEMORY OFFSET(16) NUMBITS(4) [ - InnerNonCacheable = 0b0100, - InnerWriteBack_NonTransient_ReadAlloc_WriteAlloc = 0b1111 - ], + Attr2 OFFSET(16) NUMBITS(8) [], /// Attribute 1 - Attr1_HIGH OFFSET(12) NUMBITS(4) [ - Device = 0b0000, - Memory_OuterNonCacheable = 0b0100, - Memory_OuterWriteBack_NonTransient_ReadAlloc_WriteAlloc = 0b1111 - ], - Attr1_LOW_DEVICE OFFSET(8) NUMBITS(4) [ - Device_nGnRE = 0b0100 - ], - Attr1_LOW_MEMORY OFFSET(8) NUMBITS(4) [ - InnerNonCacheable = 0b0100, - InnerWriteBack_NonTransient_ReadAlloc_WriteAlloc = 0b1111 - ], + Attr1 OFFSET(8) NUMBITS(8) [], /// Attribute 0 - Attr0_HIGH OFFSET(4) NUMBITS(4) [ + Attr0 OFFSET(0) NUMBITS(8) [] + ] +} + +register_bitfields! {u64, + MAIR_ATTR [ + Attr_HIGH OFFSET(4) NUMBITS(4) [ Device = 0b0000, Memory_OuterNonCacheable = 0b0100, + Memory_OuterWriteThrough_NonTransient_ReadAlloc_WriteAlloc = 0b1011, Memory_OuterWriteBack_NonTransient_ReadAlloc_WriteAlloc = 0b1111 ], - Attr0_LOW_DEVICE OFFSET(0) NUMBITS(4) [ - Device_nGnRE = 0b0100 + Attr_LOW_DEVICE OFFSET(0) NUMBITS(4) [ + Device_nGnRnE = 0b0000, + Device_nGnRE = 0b0100, + Device_nGRE = 0b1000, + Device_GRE = 0b1100 ], - Attr0_LOW_MEMORY OFFSET(0) NUMBITS(4) [ + Attr_LOW_MEMORY OFFSET(0) NUMBITS(4) [ InnerNonCacheable = 0b0100, + InnerWriteThrough_NonTransient_ReadAlloc_WriteAlloc = 0b1011, InnerWriteBack_NonTransient_ReadAlloc_WriteAlloc = 0b1111 ] ] diff --git a/crate/aarch64/src/regs/mod.rs b/crate/aarch64/src/regs/mod.rs index 5a7e556..f69f0c4 100644 --- a/crate/aarch64/src/regs/mod.rs +++ b/crate/aarch64/src/regs/mod.rs @@ -40,7 +40,7 @@ pub use self::daif::DAIF; pub use self::elr_el2::ELR_EL2; pub use self::hcr_el2::HCR_EL2; pub use self::id_aa64mmfr0_el1::ID_AA64MMFR0_EL1; -pub use self::mair_el1::MAIR_EL1; +pub use self::mair_el1::{MAIR_EL1, MAIR_ATTR}; pub use self::mpidr_el1::MPIDR_EL1; pub use self::sctlr_el1::SCTLR_EL1; pub use self::sp::SP; diff --git a/kernel/src/arch/aarch64/board/raspi3/mod.rs b/kernel/src/arch/aarch64/board/raspi3/mod.rs index 2d13315..0db5133 100644 --- a/kernel/src/arch/aarch64/board/raspi3/mod.rs +++ b/kernel/src/arch/aarch64/board/raspi3/mod.rs @@ -6,6 +6,8 @@ pub mod irq; pub mod timer; pub mod serial; +pub const IO_BASE: usize = bcm2837::IO_BASE; + pub fn init() { assert_has_not_been_called!("board::init must be called only once"); diff --git a/kernel/src/arch/aarch64/memory.rs b/kernel/src/arch/aarch64/memory.rs index 1c486d2..7536dc4 100644 --- a/kernel/src/arch/aarch64/memory.rs +++ b/kernel/src/arch/aarch64/memory.rs @@ -3,7 +3,8 @@ use ucore_memory::PAGE_SIZE; use memory::{FRAME_ALLOCATOR, init_heap, MemoryArea, MemoryAttr, MemorySet, Stack}; use super::atags::atags::Atags; -use aarch64::{barrier, regs::*, addr::*, paging::PhysFrame as Frame}; +use aarch64::{barrier, regs::*, addr::*}; +use aarch64::paging::{PhysFrame as Frame, memory_attribute::*}; /// Memory initialization. pub fn init() { @@ -28,13 +29,11 @@ pub fn init_mmu_early() { // device. MAIR_EL1.write( - // Attribute 1 - MAIR_EL1::Attr1_HIGH::Device - + MAIR_EL1::Attr1_LOW_DEVICE::Device_nGnRE - // Attribute 0 - + MAIR_EL1::Attr0_HIGH::Memory_OuterWriteBack_NonTransient_ReadAlloc_WriteAlloc - + MAIR_EL1::Attr0_LOW_MEMORY::InnerWriteBack_NonTransient_ReadAlloc_WriteAlloc, + MAIR_EL1::Attr0.val(MairDevice::config_value()) + + MAIR_EL1::Attr1.val(MairNormal::config_value()) + + MAIR_EL1::Attr2.val(MairNormalNonCacheable::config_value()), ); + // Configure various settings of stage 1 of the EL1 translation regime. let ips = ID_AA64MMFR0_EL1.read(ID_AA64MMFR0_EL1::PARange); TCR_EL1.write( diff --git a/kernel/src/arch/aarch64/paging.rs b/kernel/src/arch/aarch64/paging.rs index 9986faa..f11820c 100644 --- a/kernel/src/arch/aarch64/paging.rs +++ b/kernel/src/arch/aarch64/paging.rs @@ -8,62 +8,8 @@ use ucore_memory::paging::*; use aarch64::asm::{tlb_invalidate, tlb_invalidate_all, ttbr_el1_read, ttbr_el1_write}; use aarch64::{PhysAddr, VirtAddr}; use aarch64::paging::{Mapper, PageTable as Aarch64PageTable, PageTableEntry, PageTableFlags as EF, RecursivePageTable}; -use aarch64::paging::{FrameAllocator, FrameDeallocator, Page, PhysFrame as Frame, Size4KiB, Size2MiB}; - -register_bitfields! {u64, - // AArch64 Reference Manual page 2150 - STAGE1_DESCRIPTOR [ - /// Execute-never - XN OFFSET(54) NUMBITS(1) [ - False = 0, - True = 1 - ], - - /// Various address fields, depending on use case - LVL4_OUTPUT_ADDR_4KiB OFFSET(39) NUMBITS(9) [], // [47:39] - LVL3_OUTPUT_ADDR_4KiB OFFSET(30) NUMBITS(18) [], // [47:30] - LVL2_OUTPUT_ADDR_4KiB OFFSET(21) NUMBITS(27) [], // [47:21] - NEXT_LVL_TABLE_ADDR_4KiB OFFSET(12) NUMBITS(36) [], // [47:12] - - /// Access flag - AF OFFSET(10) NUMBITS(1) [ - False = 0, - True = 1 - ], - - /// Shareability field - SH OFFSET(8) NUMBITS(2) [ - OuterShareable = 0b10, - InnerShareable = 0b11 - ], - - /// Access Permissions - AP OFFSET(6) NUMBITS(2) [ - RW_EL1 = 0b00, - RW_EL1_EL0 = 0b01, - RO_EL1 = 0b10, - RO_EL1_EL0 = 0b11 - ], - - /// Memory attributes index into the MAIR_EL1 register - AttrIndx OFFSET(2) NUMBITS(3) [], - - TYPE OFFSET(1) NUMBITS(1) [ - Block = 0, - Table = 1 - ], - - VALID OFFSET(0) NUMBITS(1) [ - False = 0, - True = 1 - ] - ] -} - -mod mair { - pub const NORMAL: u64 = 0; - pub const DEVICE: u64 = 1; -} +use aarch64::paging::{FrameAllocator, FrameDeallocator, Page, PhysFrame as Frame, Size4KiB, Size2MiB, Size1GiB}; +use aarch64::paging::memory_attribute::*; // need 3 page pub fn setup_page_table(frame_lvl4: Frame, frame_lvl3: Frame, frame_lvl2: Frame) { @@ -74,34 +20,24 @@ pub fn setup_page_table(frame_lvl4: Frame, frame_lvl3: Frame, frame_lvl2: Frame) p3.zero(); p2.zero(); - // Fill the rest of the LVL2 (2MiB) entries as block - // descriptors. Differentiate between normal and device mem. - const MMIO_BASE: u64 = 0x3F000000; - let mmio_base: u64 = MMIO_BASE >> 21; - let mut common = STAGE1_DESCRIPTOR::VALID::True - + STAGE1_DESCRIPTOR::TYPE::Block - + STAGE1_DESCRIPTOR::AP::RW_EL1 - + STAGE1_DESCRIPTOR::AF::True; - // + STAGE1_DESCRIPTOR::XN::True; - - for i in 0..512 { - let j: u64 = i as u64; - - let mem_attr = if j >= mmio_base { - STAGE1_DESCRIPTOR::SH::OuterShareable + STAGE1_DESCRIPTOR::AttrIndx.val(mair::DEVICE) - } else { - STAGE1_DESCRIPTOR::SH::InnerShareable + STAGE1_DESCRIPTOR::AttrIndx.val(mair::NORMAL) - }; + let (start_addr, end_addr) = (0, 0x40000000); + let block_flags = EF::VALID | EF::AF | EF::WRITE | EF::XN; + for page in Page::::range_of(start_addr, end_addr) { + let paddr = PhysAddr::new(page.start_address().as_u64()); - p2[i].entry = (common + mem_attr + STAGE1_DESCRIPTOR::LVL2_OUTPUT_ADDR_4KiB.val(j)).value; + use arch::board::IO_BASE; + if paddr.as_u64() >= IO_BASE as u64 { + p2[page.p2_index()].set_block::(paddr, block_flags | EF::PXN, MairDevice::attr_value()); + } else { + p2[page.p2_index()].set_block::(paddr, block_flags, MairNormal::attr_value()); + } } - common = common + STAGE1_DESCRIPTOR::SH::InnerShareable + STAGE1_DESCRIPTOR::AttrIndx.val(mair::NORMAL); + p3[0].set_frame(frame_lvl2, EF::default(), MairNormal::attr_value()); + p3[1].set_block::(PhysAddr::new(0x40000000), block_flags | EF::PXN, MairDevice::attr_value()); - p3[0].entry = (common + STAGE1_DESCRIPTOR::TYPE::Table + STAGE1_DESCRIPTOR::NEXT_LVL_TABLE_ADDR_4KiB.val(frame_lvl2.start_address().as_u64() >> 12)).value; - p3[1].entry = (common + STAGE1_DESCRIPTOR::LVL3_OUTPUT_ADDR_4KiB.val(1)).value; - p4[0].entry = (common + STAGE1_DESCRIPTOR::TYPE::Table + STAGE1_DESCRIPTOR::NEXT_LVL_TABLE_ADDR_4KiB.val(frame_lvl3.start_address().as_u64() >> 12)).value; - p4[RECURSIVE_INDEX].entry = (common + STAGE1_DESCRIPTOR::TYPE::Table + STAGE1_DESCRIPTOR::NEXT_LVL_TABLE_ADDR_4KiB.val(frame_lvl4.start_address().as_u64() >> 12)).value; + p4[0].set_frame(frame_lvl3, EF::default(), MairNormal::attr_value()); + p4[RECURSIVE_INDEX].set_frame(frame_lvl4, EF::default(), MairNormal::attr_value()); // warn!("p2"); // for i in 0..512 { @@ -129,18 +65,10 @@ pub fn setup_page_table(frame_lvl4: Frame, frame_lvl3: Frame, frame_lvl2: Frame) /// map the range [start, end) as device memory, insert to the MemorySet pub fn remap_device_2mib(ms: &mut MemorySet, start_addr: usize, end_addr: usize) { ms.edit(|active_table| { - let common = STAGE1_DESCRIPTOR::VALID::True - + STAGE1_DESCRIPTOR::TYPE::Block - + STAGE1_DESCRIPTOR::AP::RW_EL1 - + STAGE1_DESCRIPTOR::AF::True - + STAGE1_DESCRIPTOR::XN::True; - - let mem_attr = STAGE1_DESCRIPTOR::SH::OuterShareable + STAGE1_DESCRIPTOR::AttrIndx.val(mair::DEVICE); - - type Page2MiB = Page; - for page in Page2MiB::range_of(start_addr, end_addr) { + for page in Page::::range_of(start_addr, end_addr) { + let paddr = PhysAddr::new(page.start_address().as_u64()); let p2 = unsafe { &mut *active_table.0.p2_ptr(page) }; - p2[page.p2_index()].entry = (common + mem_attr + STAGE1_DESCRIPTOR::LVL2_OUTPUT_ADDR_4KiB.val(page.start_address().as_u64() >> 21)).value; + p2[page.p2_index()].set_block::(paddr, EF::default() - EF::TABLE_OR_PAGE, MairDevice::attr_value()); } // let p2 = unsafe { &mut *(0o777_777_000_000_0000 as *mut Aarch64PageTable) }; @@ -168,7 +96,8 @@ impl PageTable for ActivePageTable { fn map(&mut self, addr: usize, target: usize) -> &mut PageEntry { let flags = EF::default(); - self.0.map_to(Page::of_addr(addr), Frame::of_addr(target), flags, &mut FrameAllocatorForAarch64) + let attr = MairNormal::attr_value(); + self.0.map_to(Page::of_addr(addr), Frame::of_addr(target), flags, attr, &mut FrameAllocatorForAarch64) .unwrap().flush(); self.get_entry(addr) } @@ -303,7 +232,7 @@ impl InactivePageTable for InactivePageTable0 { active_table().with_temporary_map(&frame, |_, table: &mut Aarch64PageTable| { table.zero(); // set up recursive mapping for the table - table[RECURSIVE_INDEX].set_frame(frame.clone(), EF::default()); + table[RECURSIVE_INDEX].set_frame(frame.clone(), EF::default(), MairNormal::attr_value()); }); InactivePageTable0 { p4_frame: frame } } @@ -313,7 +242,7 @@ impl InactivePageTable for InactivePageTable0 { let backup = p4_table[RECURSIVE_INDEX].clone(); // overwrite recursive mapping - p4_table[RECURSIVE_INDEX].set_frame(self.p4_frame.clone(), EF::default()); + p4_table[RECURSIVE_INDEX].set_frame(self.p4_frame.clone(), EF::default(), MairNormal::attr_value()); tlb_invalidate_all(); // execute f in the new context @@ -376,7 +305,7 @@ impl InactivePageTable0 { assert!(!e0.is_unused()); self.edit(|_| { - table[KERNEL_PML4].set_frame(Frame::containing_address(e0.addr()), EF::default()); + table[KERNEL_PML4].set_frame(Frame::containing_address(e0.addr()), EF::default(), MairNormal::attr_value()); }); } } From 264600f145465bce0411b3600303c412db05e1fc Mon Sep 17 00:00:00 2001 From: equation314 Date: Sun, 25 Nov 2018 18:27:31 +0800 Subject: [PATCH 13/22] aarch64/mmu: add io remap --- crate/memory/src/memory_set.rs | 8 +++- crate/memory/src/paging/mod.rs | 2 + kernel/src/arch/aarch64/board/raspi3/mod.rs | 3 +- kernel/src/arch/aarch64/memory.rs | 7 ++- kernel/src/arch/aarch64/paging.rs | 52 ++------------------- kernel/src/arch/riscv32/paging.rs | 2 + kernel/src/arch/x86_64/paging.rs | 4 +- 7 files changed, 24 insertions(+), 54 deletions(-) diff --git a/crate/memory/src/memory_set.rs b/crate/memory/src/memory_set.rs index f67c956..316c28b 100644 --- a/crate/memory/src/memory_set.rs +++ b/crate/memory/src/memory_set.rs @@ -98,6 +98,7 @@ pub struct MemoryAttr { user: bool, readonly: bool, execute: bool, + mmio: bool, hide: bool, } @@ -114,6 +115,10 @@ impl MemoryAttr { self.execute = true; self } + pub fn mmio(mut self) -> Self { + self.mmio = true; + self + } pub fn hide(mut self) -> Self { self.hide = true; self @@ -122,8 +127,9 @@ impl MemoryAttr { if self.user { entry.set_user(true); } if self.readonly { entry.set_writable(false); } if self.execute { entry.set_execute(true); } + if self.mmio { entry.set_mmio(false); } if self.hide { entry.set_present(false); } - if self.user || self.readonly || self.execute || self.hide { entry.update(); } + if self.user || self.readonly || self.execute || self.mmio || self.hide { entry.update(); } } } diff --git a/crate/memory/src/paging/mod.rs b/crate/memory/src/paging/mod.rs index 0a3d4c0..0f38ef1 100644 --- a/crate/memory/src/paging/mod.rs +++ b/crate/memory/src/paging/mod.rs @@ -57,4 +57,6 @@ pub trait Entry { fn set_user(&mut self, value: bool); fn execute(&self) -> bool; fn set_execute(&mut self, value: bool); + fn mmio(&self) -> bool; + fn set_mmio(&mut self, value: bool); } diff --git a/kernel/src/arch/aarch64/board/raspi3/mod.rs b/kernel/src/arch/aarch64/board/raspi3/mod.rs index 0db5133..98e59e0 100644 --- a/kernel/src/arch/aarch64/board/raspi3/mod.rs +++ b/kernel/src/arch/aarch64/board/raspi3/mod.rs @@ -6,7 +6,8 @@ pub mod irq; pub mod timer; pub mod serial; -pub const IO_BASE: usize = bcm2837::IO_BASE; +pub const IO_REMAP_BASE: usize = bcm2837::IO_BASE; +pub const IO_REMAP_END: usize = 0x40001000; pub fn init() { assert_has_not_been_called!("board::init must be called only once"); diff --git a/kernel/src/arch/aarch64/memory.rs b/kernel/src/arch/aarch64/memory.rs index 7536dc4..29f3009 100644 --- a/kernel/src/arch/aarch64/memory.rs +++ b/kernel/src/arch/aarch64/memory.rs @@ -25,7 +25,7 @@ pub fn init_mmu_early() { let frame_lvl4 = Frame::containing_address(PhysAddr::new(&PAGE_TABLE_LVL4 as *const _ as u64)); let frame_lvl3 = Frame::containing_address(PhysAddr::new(&PAGE_TABLE_LVL3 as *const _ as u64)); let frame_lvl2 = Frame::containing_address(PhysAddr::new(&PAGE_TABLE_LVL2 as *const _ as u64)); - super::paging::setup_page_table(frame_lvl4, frame_lvl3, frame_lvl2); + super::paging::setup_temp_page_table(frame_lvl4, frame_lvl3, frame_lvl2); // device. MAIR_EL1.write( @@ -111,9 +111,8 @@ fn remap_the_kernel() { ms.push(MemoryArea::new_identity(srodata as usize, erodata as usize, MemoryAttr::default().readonly(), "rodata")); ms.push(MemoryArea::new_identity(sbss as usize, ebss as usize, MemoryAttr::default(), "bss")); - // ensure the level 2 page table exists - ms.push(MemoryArea::new_identity(0x40000000, 0x40200000, MemoryAttr::default(), "arm_control")); - super::paging::remap_device_2mib(&mut ms, 0x3F000000, 0x40200000); + use arch::board::{IO_REMAP_BASE, IO_REMAP_END}; + ms.push(MemoryArea::new_identity(IO_REMAP_BASE, IO_REMAP_END, MemoryAttr::default().mmio(), "io_remap")); unsafe { ms.activate(); } use core::mem::forget; diff --git a/kernel/src/arch/aarch64/paging.rs b/kernel/src/arch/aarch64/paging.rs index f11820c..eab53f4 100644 --- a/kernel/src/arch/aarch64/paging.rs +++ b/kernel/src/arch/aarch64/paging.rs @@ -12,7 +12,7 @@ use aarch64::paging::{FrameAllocator, FrameDeallocator, Page, PhysFrame as Frame use aarch64::paging::memory_attribute::*; // need 3 page -pub fn setup_page_table(frame_lvl4: Frame, frame_lvl3: Frame, frame_lvl2: Frame) { +pub fn setup_temp_page_table(frame_lvl4: Frame, frame_lvl3: Frame, frame_lvl2: Frame) { let p4 = unsafe { &mut *(frame_lvl4.start_address().as_u64() as *mut Aarch64PageTable) }; let p3 = unsafe { &mut *(frame_lvl3.start_address().as_u64() as *mut Aarch64PageTable) }; let p2 = unsafe { &mut *(frame_lvl2.start_address().as_u64() as *mut Aarch64PageTable) }; @@ -25,8 +25,8 @@ pub fn setup_page_table(frame_lvl4: Frame, frame_lvl3: Frame, frame_lvl2: Frame) for page in Page::::range_of(start_addr, end_addr) { let paddr = PhysAddr::new(page.start_address().as_u64()); - use arch::board::IO_BASE; - if paddr.as_u64() >= IO_BASE as u64 { + use arch::board::IO_REMAP_BASE; + if paddr.as_u64() >= IO_REMAP_BASE as u64 { p2[page.p2_index()].set_block::(paddr, block_flags | EF::PXN, MairDevice::attr_value()); } else { p2[page.p2_index()].set_block::(paddr, block_flags, MairNormal::attr_value()); @@ -39,54 +39,10 @@ pub fn setup_page_table(frame_lvl4: Frame, frame_lvl3: Frame, frame_lvl2: Frame) p4[0].set_frame(frame_lvl3, EF::default(), MairNormal::attr_value()); p4[RECURSIVE_INDEX].set_frame(frame_lvl4, EF::default(), MairNormal::attr_value()); - // warn!("p2"); - // for i in 0..512 { - // if p2[i].flags().bits() != 0 { - // info!("{:x?} {:x?} {:x?}",i, &p2[i] as *const _ as usize, p2[i]); - // } - // } - // warn!("p3"); - // for i in 0..512 { - // if p3[i].flags().bits() != 0 { - // info!("{:x?} {:x?} {:x?}",i, &p3[i] as *const _ as usize, p3[i]); - // } - // } - // warn!("p4"); - // for i in 0..512 { - // if p4[i].flags().bits() != 0 { - // info!("{:x?} {:x?} {:x?}",i, &p4[i] as *const _ as usize, p4[i]); - // } - // } - ttbr_el1_write(0, frame_lvl4); tlb_invalidate_all(); } -/// map the range [start, end) as device memory, insert to the MemorySet -pub fn remap_device_2mib(ms: &mut MemorySet, start_addr: usize, end_addr: usize) { - ms.edit(|active_table| { - for page in Page::::range_of(start_addr, end_addr) { - let paddr = PhysAddr::new(page.start_address().as_u64()); - let p2 = unsafe { &mut *active_table.0.p2_ptr(page) }; - p2[page.p2_index()].set_block::(paddr, EF::default() - EF::TABLE_OR_PAGE, MairDevice::attr_value()); - } - - // let p2 = unsafe { &mut *(0o777_777_000_000_0000 as *mut Aarch64PageTable) }; - // for i in 0..512 { - // if p2[i].flags().bits() != 0 { - // info!("{:x?} {:x?} {:x?}",i, &p2[i] as *const _ as usize, p2[i]); - // } - // } - - // let p2 = unsafe { &mut *(0o777_777_000_001_0000 as *mut Aarch64PageTable) }; - // for i in 0..512 { - // if p2[i].flags().bits() != 0 { - // info!("{:x?} {:x?} {:x?}",i, &p2[i] as *const _ as usize, p2[i]); - // } - // } - }); -} - pub struct ActivePageTable(RecursivePageTable<'static>); pub struct PageEntry(PageTableEntry); @@ -201,6 +157,8 @@ impl Entry for PageEntry { false => self.as_flags().set(EF::PXN, !value), } } + fn mmio(&self) -> bool { self.0.attr().value == MairDevice::attr_value().value } + fn set_mmio(&mut self, value: bool) { self.0.modify_attr(MairDevice::attr_value()); } } impl PageEntry { diff --git a/kernel/src/arch/riscv32/paging.rs b/kernel/src/arch/riscv32/paging.rs index 680448b..ef0710f 100644 --- a/kernel/src/arch/riscv32/paging.rs +++ b/kernel/src/arch/riscv32/paging.rs @@ -126,6 +126,8 @@ impl Entry for PageEntry { fn set_user(&mut self, value: bool) { self.as_flags().set(EF::USER, value); } fn execute(&self) -> bool { self.0.flags().contains(EF::EXECUTABLE) } fn set_execute(&mut self, value: bool) { self.as_flags().set(EF::EXECUTABLE, value); } + fn mmio(&self) -> bool { unimplemented!() } + fn set_mmio(&mut self, value: bool) { unimplemented!() } } impl PageEntry { diff --git a/kernel/src/arch/x86_64/paging.rs b/kernel/src/arch/x86_64/paging.rs index 8717703..9e164bb 100644 --- a/kernel/src/arch/x86_64/paging.rs +++ b/kernel/src/arch/x86_64/paging.rs @@ -138,6 +138,8 @@ impl Entry for PageEntry { } fn execute(&self) -> bool { !self.0.flags().contains(EF::NO_EXECUTE) } fn set_execute(&mut self, value: bool) { self.as_flags().set(EF::NO_EXECUTE, !value); } + fn mmio(&self) -> bool { unimplemented!() } + fn set_mmio(&mut self, value: bool) { unimplemented!() } } impl PageEntry { @@ -261,4 +263,4 @@ impl FrameDeallocator for FrameAllocatorForX86 { fn dealloc(&mut self, frame: Frame) { dealloc_frame(frame.start_address().as_u64() as usize); } -} \ No newline at end of file +} From d30c7e59cabfc2a1d7b3e5e2bf1b22d1539b6b35 Mon Sep 17 00:00:00 2001 From: equation314 Date: Sun, 25 Nov 2018 20:01:13 +0800 Subject: [PATCH 14/22] aarch64: minor modify --- crate/aarch64/Cargo.lock | 108 ++++++++++++++++ crate/aarch64/src/asm.rs | 152 ++++++++--------------- kernel/Cargo.lock | 18 --- kernel/Cargo.toml | 2 - kernel/src/arch/aarch64/interrupt/mod.rs | 3 +- kernel/src/arch/aarch64/mod.rs | 1 - kernel/src/arch/aarch64/paging.rs | 26 ++-- 7 files changed, 183 insertions(+), 127 deletions(-) create mode 100644 crate/aarch64/Cargo.lock diff --git a/crate/aarch64/Cargo.lock b/crate/aarch64/Cargo.lock new file mode 100644 index 0000000..1038287 --- /dev/null +++ b/crate/aarch64/Cargo.lock @@ -0,0 +1,108 @@ +[[package]] +name = "aarch64" +version = "0.1.0" +dependencies = [ + "bare-metal 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", + "bit_field 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "os_bootinfo 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "register 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "usize_conversions 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "ux 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "bare-metal" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "bit_field" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "bitflags" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "cfg-if" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "log" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "os_bootinfo" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "register" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "tock-registers 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "tock-registers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "usize_conversions" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "ux" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[metadata] +"checksum bare-metal 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "a3caf393d93b2d453e80638d0674597020cef3382ada454faacd43d1a55a735a" +"checksum bit_field 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ed8765909f9009617974ab6b7d332625b320b33c326b1e9321382ef1999b5d56" +"checksum bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "228047a76f468627ca71776ecdebd732a3423081fcf5125585bcd7c49886ce12" +"checksum cfg-if 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "082bb9b28e00d3c9d39cc03e64ce4cea0f1bb9b3fde493f0cbc008472d22bdf4" +"checksum log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c84ec4b527950aa83a329754b01dbe3f58361d1c5efacd1f6d68c494d08a17c6" +"checksum os_bootinfo 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "66481dbeb5e773e7bd85b63cd6042c30786f834338288c5ec4f3742673db360a" +"checksum register 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e10f31b6d2299e5620986ad9fcdd66463e125ad72af4f403f9aedf7592d5ccdb" +"checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +"checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +"checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" +"checksum tock-registers 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3a385d94f3f62e60445a0adb9ff8d9621faa272234530d4c0f848ec98f88e316" +"checksum usize_conversions 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f70329e2cbe45d6c97a5112daad40c34cd9a4e18edb5a2a18fefeb584d8d25e5" +"checksum ux 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "53d8df5dd8d07fedccd202de1887d94481fadaea3db70479f459e8163a1fab41" diff --git a/crate/aarch64/src/asm.rs b/crate/aarch64/src/asm.rs index 1c1ee6d..4cd37d7 100644 --- a/crate/aarch64/src/asm.rs +++ b/crate/aarch64/src/asm.rs @@ -1,31 +1,9 @@ +//! Miscellaneous assembly instructions and functions + use paging::PhysFrame; use addr::{PhysAddr, VirtAddr}; use regs::*; -#[inline(always)] -pub fn tlb_invalidate_all() { - unsafe { - asm!( - "dsb ishst - tlbi vmalle1is - dsb ish - isb" - ); - } -} - -#[inline(always)] -pub fn tlb_invalidate(vaddr: VirtAddr) { - unsafe { - asm!( - "dsb ishst - tlbi vaae1is, $0 - dsb ish - isb" :: "r"(vaddr.as_u64() >> 12) - ); - } -} - /// Returns the current stack pointer. #[inline(always)] pub fn sp() -> *const u8 { @@ -37,91 +15,32 @@ pub fn sp() -> *const u8 { ptr as *const u8 } +/// Returns the current point counter. #[inline(always)] pub unsafe fn get_pc() -> usize { let pc: usize; - asm!("ADR $0, ." : "=r"(pc)); + asm!("adr $0, ." : "=r"(pc)); pc } -/// Returns the current exception level. -/// -/// # Safety -/// This function should only be called when EL is >= 1. -#[inline(always)] -pub unsafe fn current_el() -> u8 { - let el_reg: u64; - asm!("mrs $0, CurrentEL" : "=r"(el_reg)); - ((el_reg & 0b1100) >> 2) as u8 -} - -#[inline(always)] -pub unsafe fn get_far() -> usize { - let far: usize; - asm!("mrs $0, far_el1" : "=r"(far)); - far -} - -#[inline(always)] -pub unsafe fn get_ttbr0() -> usize { - let ttbr0: usize; - asm!("mrs $0, ttbr0_el1" : "=r"(ttbr0)); - ttbr0 -} - -#[inline(always)] -pub unsafe fn get_ttbr1() -> usize { - let ttbr0: usize; - asm!("mrs $0, ttbr1_el1" : "=r"(ttbr0)); - ttbr0 -} - -#[inline(always)] -pub fn address_translate(vaddr: usize) -> usize { - let paddr: usize; - unsafe { - asm!("at S1E1R, $1; mrs $0, par_el1" : "=r"(paddr) : "r"(vaddr)); - } - paddr -} - -/// Returns the SPSel value. -#[inline(always)] -pub fn sp_sel() -> u8 { - let ptr: u32; - unsafe { - asm!("mrs $0, SPSel" : "=r"(ptr)); - } - - (ptr & 1) as u8 -} - -/// Returns the core currently executing. -/// -/// # Safety -/// -/// This function should only be called when EL is >= 1. -pub unsafe fn affinity() -> usize { - let x: usize; - asm!("mrs $0, mpidr_el1 - and $0, $0, #3" - : "=r"(x)); - - x -} +/// The classic no-op +#[inline] +pub fn nop() { + match () { + #[cfg(target_arch = "aarch64")] + () => unsafe { asm!("nop" :::: "volatile") }, -pub fn wfi() { - unsafe { - asm!("wfi" :::: "volatile"); + #[cfg(not(target_arch = "aarch64"))] + () => unimplemented!(), } } -/// The classic no-op +/// Wait For Interrupt #[inline] -pub fn nop() { +pub fn wfi() { match () { #[cfg(target_arch = "aarch64")] - () => unsafe { asm!("nop" :::: "volatile") }, + () => unsafe { asm!("wfi" :::: "volatile") }, #[cfg(not(target_arch = "aarch64"))] () => unimplemented!(), @@ -160,7 +79,44 @@ pub fn eret() -> ! { } } -pub fn ttbr_el1_read(which: u8) -> (PhysFrame) { +/// Invalidate all TLB entries. +#[inline(always)] +pub fn tlb_invalidate_all() { + unsafe { + asm!( + "dsb ishst + tlbi vmalle1is + dsb ish + isb" + ); + } +} + +/// Invalidate TLB entries that would be used to translate the specified address. +#[inline(always)] +pub fn tlb_invalidate(vaddr: VirtAddr) { + unsafe { + asm!( + "dsb ishst + tlbi vaae1is, $0 + dsb ish + isb" :: "r"(vaddr.as_u64() >> 12) + ); + } +} + +/// Address Translate. +#[inline(always)] +pub fn address_translate(vaddr: usize) -> usize { + let paddr: usize; + unsafe { + asm!("at S1E1R, $1; mrs $0, par_el1" : "=r"(paddr) : "r"(vaddr)); + } + paddr +} + +/// Read TTBRx_EL1 as PhysFrame +pub fn ttbr_el1_read(which: u8) -> PhysFrame { let baddr = match which { 0 => TTBR0_EL1.get_baddr(), 1 => TTBR1_EL1.get_baddr(), @@ -169,6 +125,7 @@ pub fn ttbr_el1_read(which: u8) -> (PhysFrame) { PhysFrame::containing_address(PhysAddr::new(baddr)) } +/// Write TTBRx_EL1 from PhysFrame pub fn ttbr_el1_write(which: u8, frame: PhysFrame) { let baddr = frame.start_address().as_u64(); match which { @@ -178,6 +135,7 @@ pub fn ttbr_el1_write(which: u8, frame: PhysFrame) { }; } +/// write TTBRx_EL1 from PhysFrame and ASID pub fn ttbr_el1_write_asid(which: u8, asid: u16, frame: PhysFrame) { let baddr = frame.start_address().as_u64(); match which { diff --git a/kernel/Cargo.lock b/kernel/Cargo.lock index 7795f1f..4e2b6dc 100644 --- a/kernel/Cargo.lock +++ b/kernel/Cargo.lock @@ -5,7 +5,6 @@ dependencies = [ "bare-metal 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", "bit_field 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", "os_bootinfo 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "register 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "usize_conversions 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -177,14 +176,6 @@ name = "redox_syscall" version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "register" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "tock-registers 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "register" version = "0.2.1" @@ -247,11 +238,6 @@ dependencies = [ "remove_dir_all 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "tock-registers" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - [[package]] name = "tock-registers" version = "0.2.0" @@ -279,13 +265,11 @@ dependencies = [ "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", "bootloader 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)", - "cortex-a 2.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "linked_list_allocator 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", "once 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", - "register 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "riscv 0.3.0", "simple-filesystem 0.0.1 (git+https://github.com/wangrunji0408/SimpleFileSystem-Rust)", "spin 0.4.9 (registry+https://github.com/rust-lang/crates.io-index)", @@ -399,7 +383,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum pulldown-cmark 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "8361e81576d2e02643b04950e487ec172b687180da65c731c03cf336784e6c07" "checksum rand 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "8356f47b32624fef5b3301c1be97e5944ecdd595409cc5da11d05f211db6cfbd" "checksum redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "c214e91d3ecf43e9a4e41e578973adeb14b474f2bee858742d127af75a0112b1" -"checksum register 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0e985243ba7e1c336b62444ef2a10d7bd87cf41a222285ae3de605c859006479" "checksum register 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e10f31b6d2299e5620986ad9fcdd66463e125ad72af4f403f9aedf7592d5ccdb" "checksum remove_dir_all 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3488ba1b9a2084d38645c4c08276a1752dcbf2c7130d74f1569681ad5d2799c5" "checksum simple-filesystem 0.0.1 (git+https://github.com/wangrunji0408/SimpleFileSystem-Rust)" = "" @@ -407,7 +390,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum spin 0.4.9 (registry+https://github.com/rust-lang/crates.io-index)" = "37b5646825922b96b5d7d676b5bb3458a54498e96ed7b0ce09dc43a07038fea4" "checksum static_assertions 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "c19be23126415861cb3a23e501d34a708f7f9b2183c5252d690941c2e69199d5" "checksum tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" -"checksum tock-registers 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2acc33f980e23cee18d234a32d0637fbc1ea55e13ab04012fa857b899fa1b7a9" "checksum tock-registers 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3a385d94f3f62e60445a0adb9ff8d9621faa272234530d4c0f848ec98f88e316" "checksum uart_16550 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "269f953d8de3226f7c065c589c7b4a3e83d10a419c7c3b5e2e0f197e6acc966e" "checksum unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "882386231c45df4700b275c7ff55b6f3698780a650026380e72dabe76fa46526" diff --git a/kernel/Cargo.toml b/kernel/Cargo.toml index 06ea088..20b696f 100644 --- a/kernel/Cargo.toml +++ b/kernel/Cargo.toml @@ -32,7 +32,6 @@ bit-allocator = { path = "../crate/bit-allocator" } ucore-memory = { path = "../crate/memory" } ucore-process = { path = "../crate/process" } simple-filesystem = { git = "https://github.com/wangrunji0408/SimpleFileSystem-Rust" } -register="0.1.0" [target.'cfg(target_arch = "x86_64")'.dependencies] bootloader = "0.3" @@ -45,7 +44,6 @@ riscv = { path = "../crate/riscv" } bbl = { path = "../crate/bbl" } [target.'cfg(target_arch = "aarch64")'.dependencies] -cortex-a = "2.2.1" aarch64 = { path = "../crate/aarch64" } atags = { path = "../crate/atags" } bcm2837 = { path = "../crate/bcm2837", features = ["use_generic_timer"] } diff --git a/kernel/src/arch/aarch64/interrupt/mod.rs b/kernel/src/arch/aarch64/interrupt/mod.rs index 140e9b8..bba925a 100644 --- a/kernel/src/arch/aarch64/interrupt/mod.rs +++ b/kernel/src/arch/aarch64/interrupt/mod.rs @@ -4,7 +4,8 @@ mod handler; mod context; mod syndrome; -use super::cortex_a::regs::*; +use aarch64::regs::*; + pub use self::context::*; pub use self::handler::*; diff --git a/kernel/src/arch/aarch64/mod.rs b/kernel/src/arch/aarch64/mod.rs index ccd6230..685072e 100644 --- a/kernel/src/arch/aarch64/mod.rs +++ b/kernel/src/arch/aarch64/mod.rs @@ -1,7 +1,6 @@ //! Entrance and initialization for aarch64. extern crate atags; -extern crate cortex_a; pub mod io; pub mod paging; diff --git a/kernel/src/arch/aarch64/paging.rs b/kernel/src/arch/aarch64/paging.rs index eab53f4..0093385 100644 --- a/kernel/src/arch/aarch64/paging.rs +++ b/kernel/src/arch/aarch64/paging.rs @@ -146,19 +146,29 @@ impl Entry for PageEntry { self.as_flags().set(EF::nG, value); // set non-global to use ASID } fn execute(&self) -> bool { - match self.user() { - true => !self.0.flags().contains(EF::XN), - false => !self.0.flags().contains(EF::PXN), + if self.user() { + !self.0.flags().contains(EF::XN) + } else { + !self.0.flags().contains(EF::PXN) } } fn set_execute(&mut self, value: bool) { - match self.user() { - true => self.as_flags().set(EF::XN, !value), - false => self.as_flags().set(EF::PXN, !value), + if self.user() { + self.as_flags().set(EF::XN, !value) + } else { + self.as_flags().set(EF::PXN, !value) + } + } + fn mmio(&self) -> bool { + self.0.attr().value == MairDevice::attr_value().value + } + fn set_mmio(&mut self, value: bool) { + if value { + self.0.modify_attr(MairDevice::attr_value()) + } else { + self.0.modify_attr(MairNormal::attr_value()) } } - fn mmio(&self) -> bool { self.0.attr().value == MairDevice::attr_value().value } - fn set_mmio(&mut self, value: bool) { self.0.modify_attr(MairDevice::attr_value()); } } impl PageEntry { From 0eb33b661f152c5de3a97c9bd0e3f0aaf82bf6aa Mon Sep 17 00:00:00 2001 From: equation314 Date: Fri, 30 Nov 2018 14:13:27 +0800 Subject: [PATCH 15/22] aarch64/mmu: fix some bugs --- crate/aarch64/Cargo.lock | 108 ------------------- crate/aarch64/src/paging/memory_attribute.rs | 2 +- crate/memory/src/memory_set.rs | 2 +- kernel/src/lib.rs | 4 - 4 files changed, 2 insertions(+), 114 deletions(-) delete mode 100644 crate/aarch64/Cargo.lock diff --git a/crate/aarch64/Cargo.lock b/crate/aarch64/Cargo.lock deleted file mode 100644 index 1038287..0000000 --- a/crate/aarch64/Cargo.lock +++ /dev/null @@ -1,108 +0,0 @@ -[[package]] -name = "aarch64" -version = "0.1.0" -dependencies = [ - "bare-metal 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", - "bit_field 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", - "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", - "os_bootinfo 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "register 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "usize_conversions 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "ux 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "bare-metal" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "bit_field" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "bitflags" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "cfg-if" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "log" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "cfg-if 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "os_bootinfo" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "register" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "tock-registers 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "rustc_version" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "semver" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "semver-parser" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "tock-registers" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "usize_conversions" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "ux" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[metadata] -"checksum bare-metal 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "a3caf393d93b2d453e80638d0674597020cef3382ada454faacd43d1a55a735a" -"checksum bit_field 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ed8765909f9009617974ab6b7d332625b320b33c326b1e9321382ef1999b5d56" -"checksum bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "228047a76f468627ca71776ecdebd732a3423081fcf5125585bcd7c49886ce12" -"checksum cfg-if 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "082bb9b28e00d3c9d39cc03e64ce4cea0f1bb9b3fde493f0cbc008472d22bdf4" -"checksum log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c84ec4b527950aa83a329754b01dbe3f58361d1c5efacd1f6d68c494d08a17c6" -"checksum os_bootinfo 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "66481dbeb5e773e7bd85b63cd6042c30786f834338288c5ec4f3742673db360a" -"checksum register 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e10f31b6d2299e5620986ad9fcdd66463e125ad72af4f403f9aedf7592d5ccdb" -"checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" -"checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" -"checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" -"checksum tock-registers 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3a385d94f3f62e60445a0adb9ff8d9621faa272234530d4c0f848ec98f88e316" -"checksum usize_conversions 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f70329e2cbe45d6c97a5112daad40c34cd9a4e18edb5a2a18fefeb584d8d25e5" -"checksum ux 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "53d8df5dd8d07fedccd202de1887d94481fadaea3db70479f459e8163a1fab41" diff --git a/crate/aarch64/src/paging/memory_attribute.rs b/crate/aarch64/src/paging/memory_attribute.rs index beb9b69..4a24071 100644 --- a/crate/aarch64/src/paging/memory_attribute.rs +++ b/crate/aarch64/src/paging/memory_attribute.rs @@ -59,6 +59,6 @@ impl MairType for MairNormalNonCacheable { #[inline] fn attr_value() -> PageTableAttribute { - MEMORY_ATTRIBUTE::SH::NonShareable + MEMORY_ATTRIBUTE::AttrIndx.val(Self::INDEX) + MEMORY_ATTRIBUTE::SH::OuterShareable + MEMORY_ATTRIBUTE::AttrIndx.val(Self::INDEX) } } diff --git a/crate/memory/src/memory_set.rs b/crate/memory/src/memory_set.rs index 316c28b..b47a6c2 100644 --- a/crate/memory/src/memory_set.rs +++ b/crate/memory/src/memory_set.rs @@ -127,7 +127,7 @@ impl MemoryAttr { if self.user { entry.set_user(true); } if self.readonly { entry.set_writable(false); } if self.execute { entry.set_execute(true); } - if self.mmio { entry.set_mmio(false); } + if self.mmio { entry.set_mmio(true); } if self.hide { entry.set_present(false); } if self.user || self.readonly || self.execute || self.mmio || self.hide { entry.update(); } } diff --git a/kernel/src/lib.rs b/kernel/src/lib.rs index 05cc181..063b1d0 100644 --- a/kernel/src/lib.rs +++ b/kernel/src/lib.rs @@ -38,10 +38,6 @@ extern crate aarch64; use linked_list_allocator::LockedHeap; - -#[macro_use] -extern crate register; - #[macro_use] // print! pub mod logging; mod memory; From d6a54496f0aaa479ae06ac0b80a7d85f98dc17ae Mon Sep 17 00:00:00 2001 From: equation314 Date: Sat, 1 Dec 2018 19:43:49 +0800 Subject: [PATCH 16/22] arch64/mmu: invalidate all icaches in InactivePageTable::with() --- crate/aarch64/src/asm.rs | 12 ++++++++++++ kernel/src/arch/aarch64/paging.rs | 3 ++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/crate/aarch64/src/asm.rs b/crate/aarch64/src/asm.rs index 4cd37d7..bada049 100644 --- a/crate/aarch64/src/asm.rs +++ b/crate/aarch64/src/asm.rs @@ -105,6 +105,18 @@ pub fn tlb_invalidate(vaddr: VirtAddr) { } } +/// Invalidate all instruction caches in Inner Shareable domain to Point of Unification. +#[inline(always)] +pub fn flush_icache_all() { + unsafe { + asm!( + "ic ialluis + dsb ish + isb" + ); + } +} + /// Address Translate. #[inline(always)] pub fn address_translate(vaddr: usize) -> usize { diff --git a/kernel/src/arch/aarch64/paging.rs b/kernel/src/arch/aarch64/paging.rs index 0093385..0b7546e 100644 --- a/kernel/src/arch/aarch64/paging.rs +++ b/kernel/src/arch/aarch64/paging.rs @@ -5,7 +5,7 @@ use memory::{active_table, alloc_frame, alloc_stack, dealloc_frame}; use ucore_memory::memory_set::*; use ucore_memory::PAGE_SIZE; use ucore_memory::paging::*; -use aarch64::asm::{tlb_invalidate, tlb_invalidate_all, ttbr_el1_read, ttbr_el1_write}; +use aarch64::asm::{tlb_invalidate, tlb_invalidate_all, flush_icache_all, ttbr_el1_read, ttbr_el1_write}; use aarch64::{PhysAddr, VirtAddr}; use aarch64::paging::{Mapper, PageTable as Aarch64PageTable, PageTableEntry, PageTableFlags as EF, RecursivePageTable}; use aarch64::paging::{FrameAllocator, FrameDeallocator, Page, PhysFrame as Frame, Size4KiB, Size2MiB, Size1GiB}; @@ -246,6 +246,7 @@ impl InactivePageTable for InactivePageTable0 { if old_frame != new_frame { ttbr_el1_write(1, old_frame); tlb_invalidate_all(); + flush_icache_all(); } } From a59a7fbe9a3e4878bcd8abbc913e87b46c6c717d Mon Sep 17 00:00:00 2001 From: equation314 Date: Sat, 1 Dec 2018 20:00:35 +0800 Subject: [PATCH 17/22] aarch64/mmu: simply handle page fault --- crate/aarch64/src/regs/far_el1.rs | 31 ++++++++++++++++++++ crate/aarch64/src/regs/mod.rs | 2 ++ kernel/src/arch/aarch64/interrupt/handler.rs | 28 +++++++++++++++--- 3 files changed, 57 insertions(+), 4 deletions(-) create mode 100644 crate/aarch64/src/regs/far_el1.rs diff --git a/crate/aarch64/src/regs/far_el1.rs b/crate/aarch64/src/regs/far_el1.rs new file mode 100644 index 0000000..fc809fa --- /dev/null +++ b/crate/aarch64/src/regs/far_el1.rs @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2018 by the author(s) + * + * ============================================================================= + * + * Licensed under either of + * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + * - MIT License (http://opensource.org/licenses/MIT) + * at your option. + * + * ============================================================================= + * + * Author(s): + * - Andre Richter + */ + +//! Fault Address Register - EL1 +//! +//! Holds the faulting Virtual Address for all synchronous Instruction or Data +//! Abort, PC alignment fault and Watchpoint exceptions that are taken to EL1. + +use register::cpu::RegisterReadWrite; + +pub struct Reg; + +impl RegisterReadWrite for Reg { + sys_coproc_read_raw!(u64, "FAR_EL1"); + sys_coproc_write_raw!(u64, "FAR_EL1"); +} + +pub static FAR_EL1: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/mod.rs b/crate/aarch64/src/regs/mod.rs index f69f0c4..11a6d3f 100644 --- a/crate/aarch64/src/regs/mod.rs +++ b/crate/aarch64/src/regs/mod.rs @@ -12,6 +12,7 @@ mod cntvoff_el2; mod currentel; mod daif; mod elr_el2; +mod far_el1; mod hcr_el2; mod id_aa64mmfr0_el1; mod mair_el1; @@ -38,6 +39,7 @@ pub use self::cntvoff_el2::CNTVOFF_EL2; pub use self::currentel::CurrentEL; pub use self::daif::DAIF; pub use self::elr_el2::ELR_EL2; +pub use self::far_el1::FAR_EL1; pub use self::hcr_el2::HCR_EL2; pub use self::id_aa64mmfr0_el1::ID_AA64MMFR0_EL1; pub use self::mair_el1::{MAIR_EL1, MAIR_ATTR}; diff --git a/kernel/src/arch/aarch64/interrupt/handler.rs b/kernel/src/arch/aarch64/interrupt/handler.rs index 787cbf5..21142f9 100644 --- a/kernel/src/arch/aarch64/interrupt/handler.rs +++ b/kernel/src/arch/aarch64/interrupt/handler.rs @@ -2,7 +2,9 @@ use arch::board::irq::handle_irq; use super::context::TrapFrame; -use super::syndrome::Syndrome; +use super::syndrome::{Fault, Syndrome}; + +use aarch64::regs::*; global_asm!(include_str!("trap.S")); global_asm!(include_str!("vector.S")); @@ -46,7 +48,14 @@ pub extern "C" fn rust_trap(info: Info, esr: u32, tf: &mut TrapFrame) { // syndrome is only valid with sync match syndrome { Syndrome::Brk(brk) => handle_break(brk, tf), - Syndrome::Svc(_) => handle_syscall(tf), + Syndrome::Svc(svc) => handle_syscall(svc, tf), + Syndrome::DataAbort { kind, level: _ } + | Syndrome::InstructionAbort { kind, level: _ } => match kind { + Fault::Translation | Fault::AccessFlag | Fault::Permission => { + handle_page_fault(tf) + } + _ => ::trap::error(tf), + }, _ => ::trap::error(tf), } } @@ -57,12 +66,16 @@ pub extern "C" fn rust_trap(info: Info, esr: u32, tf: &mut TrapFrame) { trace!("Interrupt end"); } -fn handle_break(num: u16, tf: &mut TrapFrame) { +fn handle_break(_num: u16, tf: &mut TrapFrame) { // Skip the current brk instruction (ref: J1.1.2, page 6147) tf.elr += 4; } -fn handle_syscall(tf: &mut TrapFrame) { +fn handle_syscall(num: u16, tf: &mut TrapFrame) { + if num != 0 { + ::trap::error(tf); + } + // svc instruction has been skipped in syscall (ref: J1.1.2, page 6152) let ret = ::syscall::syscall( tf.x1to29[7] as usize, @@ -78,3 +91,10 @@ fn handle_syscall(tf: &mut TrapFrame) { ); tf.x0 = ret as usize; } + +fn handle_page_fault(tf: &mut TrapFrame) { + let addr = FAR_EL1.get(); + trace!("\nEXCEPTION: Page Fault @ {:#x}", addr); + + ::trap::error(tf); +} From 5a83d6a20f1f2214c2ccf700e4add752fc335a42 Mon Sep 17 00:00:00 2001 From: equation314 Date: Fri, 30 Nov 2018 13:36:17 +0800 Subject: [PATCH 18/22] aarch64: add user program SFS image --- kernel/src/fs.rs | 2 +- user/user-aarch64.img | Bin 0 -> 2097152 bytes 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 user/user-aarch64.img diff --git a/kernel/src/fs.rs b/kernel/src/fs.rs index 6810916..89e32aa 100644 --- a/kernel/src/fs.rs +++ b/kernel/src/fs.rs @@ -24,7 +24,7 @@ global_asm!(r#" .global _user_img_start .global _user_img_end _user_img_start: - .incbin "../user/user-riscv.img" + .incbin "../user/user-aarch64.img" _user_img_end: "#); diff --git a/user/user-aarch64.img b/user/user-aarch64.img new file mode 100644 index 0000000000000000000000000000000000000000..ba673f0526d82a59d91e1596bc28e41ddadf4ca9 GIT binary patch literal 2097152 zcmeFa3w#{amFIuEt1SskkZjq=U_fPT$BvS*4A@9E50_s77MNh8`G>)~t8Ta2cE@^m z%a17YrwQ3lGUM=HckrwUWS5q0993sBCp zV^RO#xm6`~mo2l8*_qkwulSU2-E+=8=iKjo)UC!L4c~gGQH7OSrj$x0JG$Ew=B6b6 zNbO9e6CDupA^{R00TLhq5+DH*AOR8}0TLhq68PVrK%K(pe^{YV2JQ&%D{wdBUW64TdB z`0*tyNq_`MfCNZ@1W14cNPq-LfCNZ@1W14cNPq-LfCNZ@1W14cNPq-LfCNZ@1W14c zNPq-LfCNZ@1W14cNPq-LfCNZ@1W14cNPq-LfCNZ@1W14cNPq-LfCNZ@1W14cNPq-L zfCNZ@1W14cNPq-LfCNZ@1W14cNPq-LfCNZ@1W14cNPq-LfCNZ@1W14cNPq-LfCNZ@ z1W14cNPq-LfCNZ@1W14cNPq-LfCNZ@1W14cNPq-LfCNZ@1W14cNPq-LfCNZ@1W14c zNPq-LfCNZ@1W14cNPq-LfCNZ@1W14cNPq-LfCNZ@1W14cNPq-LfCNZ@1W14cNPq-L zfCNZ@1W14cNPq-LfCNZ@1W14cNPq-LfCNZ@1W14cNPq-LfCNZ@1W14cNPq-LfCPU3 z2~0;9XW*WR`!d{hxM$(M95*i#AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq z5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH* zAOR8}0TLjAe;fi;A^ku99}jj(pk9M|!LW)Wb5-`PklF)7`OgK6Ut|4G+GB>wj!sp@ zQ6rq+w>q5N|8Rx9|G`jp-&|EZYJ?gFARAbqiX(=~7b2PI`#-FTxeaOvP7bL;DEn-y z8hYZfO8d*Vs^Z;B?Kx1l#vXW3W%pIDwGY&V?S0jTT`+3%OQF+kRBiU7cs4_s=_g&; zr1?mEiRl$3d@D_Ec?J}mHptAEA7DrA^TuG7>ugy za|gaK{n-_2=thh~ZiO{8*8lvtAMqvESpQ)if6y?pwh_(_&W+e7Dl*e;FdqbSVV!SW zW9J(~@Vypy*cPAp#tJ-F+5@|)bgcakUt~+XCmy?4=lA7SRg|0%Lg#tKcn&`-%fVS4 z{?Tg~`;lE0_EC≈`?U!2H!4tzzN9ko{U6=EU`C$Z1kTBlYf_cop-cV1yfvVUA$D z^2Z*ow2xyh42MrQ%`sJWq*@hMDXVxCeK3vFP3BF=_;{+Fna^*?K4el?gXvtGt-*m3rb^*=YxMD&)Y z8w%*a665u(EqxXrQP1wLO6+v|kn7tnkxN^-{P-EqCKs^L?Za;SF9vj_?S~E5W_Q5G z%TKSN%q6bPXG?7S_&HCPDbe}-U_!qj=7QPB+!78CxyPje;?w z;TgzBP+|{i_@;gq``U(4>@9~;*U$H?w8#1<*0rCn#0z`lI*M50|1HEq=OxAiVYLVS-y1=_MVlq)fAtD;6WQ!uj6A(y{L ze@pcPS|8E+A?V)-@OaILG{_$0DC2eEv8C#TMt{?u$$dir!rHw*ta?rx6%B7F)pFFh40B{A@|@L>39oo5RiS!fjsrWZ2o{G_?&7R( za}51aw{3481+%v|tg=tdmi_bj!KkW#={Wj%%&5x$0{wgwx%%&y;?{`$=qzhUa(fH8 zNPh1~esRNo8~MGPnqp(14;}w;B>P2W*=I4Hxw>*#OR6oj$wrbzQ@0Qo+AlA>(SIK9b zNqn_k+`7hoH?u-$!c^E|-PcL}cTH0* z%jceMlH8>al7B7s_CLdUL;5ZGtin4~;thChzFMnVuDGha&#!y-2e;#Vfqeb`N?-kF zRqgiu;C5^1zCe3JSwj)*m&c8|hNGAhnA2G6m*sE3eC+n(zHuqy)|JP7264|^4q4*_ ze+6HDOg*>I{=OOI?WB*tI;?u`xl|pFVm{@XRPmK7tWVxqt@QjW$oVCtmVB+rDhh{3 z)aj;Km1x852#724J4ja8bZ$wT{ zUs|5Guolj5j<}=%igdGadWI1pnWnT@7?cf9ouL zdS-b(wJI}o3~Ox-*5y5KI^U47_qpdVuZ2DIKM=i)`JckizE_>4XD%!E^8ni5fm&iqyENO>tpmXxWnWwDH;y``J`?SA1VE=TWep{f9GhSJ|pS)LIe;svy{Qu@X z=bU_B2-v?As8_+i-~N34y(tiXFi>BDI`#v7jr9+?-y`1%W$o1=e2*CR2?O6FIP(wQ zf$x!zsiDCItMQ#T1@{{Ly(8az@~v^OzQR^GvyAn>Fn*qX0DB?Ea#=Vc-y)gy4=S~^ z0(fwEw{*7Uu@jc&&NG)_KrDwdvJH-?!w)PTh8t6V-vYc3pt;n zz`1m~{x%yttIw(9-@p_48Csu%{x!TS(B_Ejlm8Nau}|aE6+X)JGIfpaJL)e!W zLneE**qIT$6C*8i@c2%@C`vCSh(ZBbuv-j)hm-woOAKz|OZbo(v_I9y(8#Z!&m7esa61`@3 ztTP!mV`+1KYWe)O><_ayk`qIlM+M^b9P^m_o zskq&LcI&4a{46r!T&}J<=O5qhAFkY}7OO|bc|>00l?%O6q;gqa{a&x^4(MGg%wA8v+bajXa@Z>iUU||h-CvO$XO><` zjJ_S+Uu@gvh;?g=4G*s@wUV!KW<-Vsyh)+Zb~NF%tUW*SFfko7;7`T zd%NO^R7y*do!esVNyuWov5rJK;oD(&&8|&mN1~&vcPA{py4N(}HikTv*xZrmOvlp6 zu1?ISzV@^m$oHFQ-(+sWM0BM}c@E5E%G{c4#{~5h?v$4~+t(|`on2%d8zqGJ+}M|rY$w`lMOUxcmh4T$({5ix58GI&P&+OW#Cy9^ zDKEO&p6uKzjd1mKCOf*@6Vhjp-IR!>-F&4ln`7-}Dv{8p>8>ubBi6Z7cOJ2N6Q(RZ zbxuBJti3l8YuhP{HkBUdI-aleN|H--bf?j^g`Iuv?H^J~kEuD|$EW6dE#q5~?QQ0@ zz1;wG;WV2(=sYp_3~(zIFmYjA)?uvK%YBP z?#A-bmABuy+-zCY;_WkTlk3Lgi#wTI;AM7g9TzJt*T&bD$^AUfR~ibtwZ0suRBzy5 zy`x4XK{{y2|+U#{Jk`1j?eZZ{EkD(V;VKQCY}^$Xcs0ee60 zk%0Uox_&MPU;apdPb2Iv6!&h8dZJ%Vun&WeFL(V;aLWYj$MX!xZ+XACctMtq<86Li zPkvI=xP1Q>hPc$;mq#a&XC{&3TyS1L!;{DhlgI(}ynf6{>$T@v>()TV=|<;|-rMU87NhF0Xmo?I7@KY?51{<`qxU-jhv{viJi zaJinuE%yF=cnUvY$n_jU$t{%P|1-K&>P?hl@6VHFPkzRe zn_ix#CqM4V{e8pF(?Gu@&xkgh$RGc{Rn@@%Jl%d`{!~LQah2!a&vTY1j|Sp?*NdC; zus_m zpAEi$Z#}kYfA~WVtT+$O+o8HPexgwy6ZPj|vNM^EceN!Jbt!#z7n?TuOiXs+G2Y#m zj&D(^ovCf%fv+2Q#*g+yY#X)Tnl=?6GdZC!rExEm6Fa`(pD4=6|f5jb<2X_w_dJLe`r| zl}fu4V_XxcsyWbzr9O_K^v~TpYTWip?=$1gz4zus9iJ{*rl+NdP5A@+;Y_#vB~mT9 z^ztaFm&Ps;E5H5ji(ILsu^PAgUUL7x8$=`O@z^5puBv)Be$*{|u}5XRinVrB7e7*yZ+-du>Xt5* zvaX)6u~ai`Ed1U2DcaWktMC(gY3DY>-*?ylJ0|@7d9U>U{=Mgx1qqM<36KB@kN^pg z011!)36KB@kN^pg011!)36KB@kN^pg011!)36KB@kN^pg011!)36KB@kN^pg011!) z36KB@kN^pg011!)36KB@kN^pg011!)36KB@kN^pg011!)36KB@kN^pg011!)36KB@ zkN^pg011!)36KB@kN^pg011!)36KB@kN^pg011!)36KB@kN^pg011!)36KB@kN^pg z011!)36KB@kN^pg011!)36KB@kN^pg011!)36KB@kN^pg011!)36KB@kN^pg011!) z36KB@kN^pg011!)36KB@kN^pg011!)36KB@kN^pg011!)36KB@kN^pg011!)36KB@ zkN^pg011!)36KB@kN^pg011!)36KB@kN^pg011!)36Q}5Ap{nppAEP_jQbQmtE4Al9-5Pt~L6zNC zz1BWZ7q<6R8+O5{%`b%xZ4N0*Wj~5%GnAQr!j(bnUcr1MzQpx&$|0AW5wQ!wJPPG` z;J<+|X&$vDdCbG}=v1pX*8jc1VWah>vHowB@CfC1Lmv%WLn@-CMZ;>ywf(lXm5j&w zziA&set6uitSMPxxp1CkA6TujAG~sYHO%aFY^?33#{U?4_n1o8m(eTsh<`w zHwzdeIszqWH45LBfyy`f0Vdy2tR_OoYtkW`5YZd1Vo4bxf zcNEX7@GSPfM18FPSsU}K!Bm;`GIqm`vv;ijxp5|;fYKnIo>uV-!Pv-pU5c7Ii3 z=ldTvT>pb5I={Vc8*=rz^XSd6tNUU==ePIQNahmP?%$T!`SEj}?sFx&K>G%(#%+Ip ziH)=uF5iF*_E_&LvLAP!^*-_x7>XbN`eva1Vxa!RKwZ|`G0iXBp!cDBLiSr=iSfyQ z?>%SlD4sK+(@phi##ResT`*=eya4$KO6(~OFY0HpuWcB`UNMZie!gd=J=Q<5PW}8O zUf3I_QN$AeZy~;d40CPwoZ%H&%!|EI#Jt9p9g?{{XIN!_26+y0#Li2ML|E-X|My0) zr(q9h2&4Th5n5IkJpSygX}4eGF}%RTdH1o zRPua#Ls+=vPt8`vchJwbu~&aBqSnZ*eti}7os^_#((eQ>+Ek}*Zuy4*p zp0gS<;T12XDpXI*abRZ^!QzlIE3>}MG4w~>w!L{2%--Ix%04w)_QU4~qpJR;hI6Rtr7dtS=Nx`_7-xH{N9oL;)eY;@_RQm#m2-qbbNg%`$c8hXEC0+ zx+;4XWK&+dH2cH~Rm>RHvWF{GwtkQ5F)_CFrtz~(L@haL%tY=p^25kGdYRQM`jyB% z7PgknQkm(e(5H$>X8ml8kBsH5SUcwD)w)krz*kGCCuyjb9Q0*=-fV3?i+7Otvalz7 zsZ#wYGuoY-)z3KA&GVFJ&@Gg#3Sj%oi&ZEYa`JW($pQGd}{e1>9u{k)! zwfR2k4#rhvr}jO5g|z#wXH!w(+B}DP6#C|2wNCU^C}mHnpRalrdRz^oa(47;t66kA zO7yp*uTAebHwaT<`T!5>Kcw>PGC-BtzVYE z0rRoji~Gil(&;U{_3#mx#v=KIEwj{Yf{Bm zuCP9NXSLGvuOR1unYE@?F7}nYvtjl}ebiN^D?{m*#UJHBZe;|4p z^FM{3eXlx8&s zdjR7pYeU8_3w#z*^W3X=c;KdFh+MxRV{P9Ie7*)JF%|tPM*+XE??vA8KVDV z=r@$=FF9B5@hCQx*nL3!l*AD~+e-Bpo$JTrR-9jAC;0~bEGpGko$IHzR4?rV{gL&I z@heU(v6FX}>@{NR_gmItCv0-7F?PetcZpnN$&z+=VO)awWddy$2kJZCr~Tal`=SpSgw zees=8)?OXL_l04fFz|hWGyLEk_`dj<8X8=%8sBYGaIewdFY*m1-wp@sD{O@`%UJ&l zVKw7Ak<0vgdEgATl%8GpzOEKC4_!_77l>^R3ROFO}#uyJMZnxEV{E^Ha;`w=KWe z#D#`(Y5n$CZ)dV|^Z6`R{n4$<&BTsm+T0XNwkO&aUF^0?rQ4F7%a!WtPIQ_#>Dn!x z$#Qdk!Z*|Y8{3l`Q;mHcv1I3>xZ7NzMOR;X`9yovf(|Oxh%*(p`_FFuRD+)lMx4vl zRp;+6@PBfPhO``6sF0u? zR<5X5-{FRJG?UIm5Z;v?)oK5 zZ)j>>7TXwaOKkGv_I0MZ6Y=DxWTMSX^!9f3dWwy)HnY38E1pQDv?STNE!LidEY=(A zNTd_K9fsHJ+GKVlI=Xsy!qTgIO%rZo$Ww{U9f{6#ES>D?#C+;&PrHG9zlruu<|a%; zSE`ifz)Ysht;u#wP*34bd6~0)y<*(iWv2S#Tg&aKi2S6^qcqq{vJ zeFoW0iCEgrSNgI!)^4T}32mD0>M}cGojY~s5vw<0%F2a>( z`AV-OxkN{I8eLo1+1K9wA*J+~n)7{pYQEQUdn}pmPPQ$?eeFlg56Vu^8t-aLe8{}{ zW^=Lm!4K-qL5sA+?Uhoo6gD1j5M|%Mrl38_e$$w0Pb9kCtpIg%{^BKVW=AR!PmOOI zN}FI7r97EipWkNgK$2#v9fMe6?ME#y4HfbPj#H{Ja1ec@S|k&4v5ctSmD!0!mHcrY zKfb(1>nHs9ax);Giu#58&kNW~{X+Iuz}}C0Bp{dn7OOf>e|`B=0S=9@zEIq|HPVUx zHfir6<;z{a6WlTZ`|&&j@>||7E?#z}<8_-K*OQ+VH7?&j-m|zS^7Q4=N#vPHu$Q>LJcqhme%x*x9?uuIMKAb? ze0=-Gm!98VdFKGp%JoUyAw>4&FKVg#-%OT_UJTg#{q@fkp1J;SEQ|e9wIZ3w=d-9? zh(jij=kE8h9}d|2{jCY`_w#%?kdH5q;!s_h=Od_12MN=g4_AsB*FM}L_vfcpLni-X zL*)LrJcSQtxentNdDL5X_+3w5^RT~&{D`;D?C|7&jZ);Mt~-wJ|J$D2@9!F|SL&CZ zJm=-<+vA&D`|;=J9?$-7J-Mapj`JB${&$|--#7gJj(Bo^T$(-mUwd*4CN9S@rNH$j z?i%DjPuC{K9UliepPa|R_aF8+SOGszdG;Akt_#S%@5%k~{S4}IeFe90^XK`UTB_8= zo_~LT@cqJ ztpEpn*X#UUvx#y4k|)>88u|G7>w^`JDD!rxu8p5))W;V68I$ZxrsV*%s7vWHrr5N} z=S;E-kMZulbbO0S?M%gE?d=eFPpS`%QT0E9#LK2OoL-Zi=}jGJX!K)eUx#}VYfYu0 z+6=!^h#&9B1b?mZ_O4Vyc_%Zq&37WPS~qp|ZdGl4-PgG-cDfN&YpYIHn`l?!xw9Py zNNJ=`k_Z{=P3yXJLO;Nf+MGzIlO2gxIiP8wCs5nsZe%!@4@j+(r1XOv6!BkxPwnhT z$2Ou&_qyd4|0$i=kyeX(yV_#um^XI+GsSE)#?ULhZI3&W^&wLHfTs7WaRt^;`85Ys zvDC*2l>Yhfk!rVnVSHk^?c9CPeU?8<+fGQ8+A+gzf3Md-+}w=YZ||>*iy+fewzM&x zk6h`tU+Wg*+CWvmPR`eU=GCqxj`(F)uzi{Gf1o1#-HFxS{y)&1O-O(QNPq-LfCNZ@ z1W14cNPq-LfCT8w|+xTHM#+z8?1y-1z(U>IU5M?~3I2&1JapR|nLM zxNpKO|L%tu36KB@kN^pg011!)36KB@kN^pgz(0Qi_piP4qv6oR-zo^JC-vWnn5R>B z9Lwv%i{9TykgA1W148(-LQT=O`mgBeC(n5N@3lEjbfzowOaJe6_?9e5fCNZ@1W14c zNPq-LfCNZ@1W14cNPq-LfCNZ@1W14cNPq-LfCNZ@1W14cNPq-LfCNZ@1W14cNPq-L zfCNZ@1W14cNPq-LfCNZ@1W14cNPq-LfCNZ@1W14cNPq-LfCNZ@1W14cNPq-LfCNZ@ z1W14cNPq-LfCNZ@1W14cNPq-LfCNZ@1W14cNPq-LfCNZ@1W14cNPq-LfCNZ@1W14c zNPq-LfCNZ@1W14cNPq-LfCNZ@1W14cNPq-LfCNZ@1W14cNPq-LfCNZ@1W14cNPq-L zfCNZ@1W14cNPq-LfCNZ@1W14cNPq-LfCNZ@1W14cNPq-LfCNZ@1W14cNPq-LfCNZ@ z1W14cNPq-LfCNZ@1W14cNPq-LfCNZ@1W14cNPq-LfCNZ@1pdz;a5K8O0=I>GCGJ(Y zSL0rTn->X?011!)36KB@kN^pg011!)36KB@kN^pg011!)36KB@kN^pg011!)36KB@ zkN^pg011!)36KB@kN^pg011!)36KB@kN^pg011!)36KB@kN^pg011!)36Q`)5`mDK zQ5q={s8^w0Fs$OpT$NoJQhQ)qWdq&;S+?C4Zg95urEeXGOS{SQ~z`yUKt_svzs zqeiG<0J4DvsyJe(d?AvVzW>9jnA@O+;A^+CR%D-TRYOlaR%w6vR#m)PsXYhk*4P6N zs_ee%wf2F!u)VL^unR_QekpWlb4Zz?>__ozhBDJnxH5>{E0~YOm$-gTIpmTvB6cB| z$8dQbi!P8yZAl*U@H{%zDvtGkZ*bUXeQB)!TO~X~`5g4our;J2YFad`hFsfkYg@^9 ztpA(#G319wu$Yn+mJ8=u_JP$Z`@t($+Jg&1_Q85E7**Nl4t!zyvn$k4BgP@O!WtUu ze}3GL_>ya^|FDifXc$@B2xkZ9M(h(6ndvr|4}!U{&Nr^H^Nk_+UW+?yi_d&x1)eMI zfn8NP*8YbtvL)UVk6o60(m?}F` zt%|FZRXmD5n8xWQ^CskR@=v}1-$fJhLZd34sJ4a*^)gRkx4<7frO_&el=^7_ zbF+XkLT*_ZBg0hFjvL3eRHyOVr2upS3Z+8cda0FJm|CID5zXpBrZ)ddt%d1$1DE@p{&lK8ug2XZKen zcDjAY^=+5PrLA0k{ETOl3)uMjVZ-(D$a(bs8p&MZ+WgxR8^3+d(|xW)=g0TjH&``p z`}<35q`h$Y24t{5g#XY;_T%oe-bbDSL-FHZ-wf2jvrPYdpf2m}nC2I5(EHFmA^R<` z#Q5aD_nxzN6wjH^>85%$W2=R+E*LW!UVwZACH9nt7xlB)*EWn|uNX#MKi{*`9_yc2 zr+$7CFYJxeC}N5Kw-8@JhPifVPEKVpFZMHgxImi(^FEQ4I)gJVJ zZv^eI2Q-9HuEaR48dlki70Uh%a`}7ow^To%^&zeQF7$5%c)Vsr8e~s#l<~Uo*i!Yv zqmt*_8^XdRe`>ZWzJq?gjlKG75w))Fp~}a89ab$*VJ+StRz0VUiiS6oYB_3LhJAA; z@|@L>39oo5RiS!fjsrWZ2o{Hws>=E{$Iu^j+xF&BFnfE$D*M!I*$#Vj-#K) zjH>)E(9buKtG_=Nw?^zoXIVp%+gr#*@_R?}iyQXa$nV|M6dMD7==gQv>=%_~pT&6Q z>ZRHvWF{GwtkQ5F)_CFrtz~(L@haL%tY=p^25kGdYRQM`jyB% z7PgknQkm(e(5H$>X8ml8kBsH5SUcwD)w)krz*kGCCuyjb9Q0*=-fV3?i+7Otvalz7 zsZ#wYGuoY-)z3KA&GVFJ&@Gg#3Sj%oi&ZEYa`JW($pQGd}{e1>9u{k)! zwfR2k4#rhvr}jO5g|z#wXH!w(+B}DP6#C|2wNCU^C}lsYpRalrdRz^oa(47;t66kA zO7yp*uTAebHwaT<`FSCWSOn0juZ{e3ga+esgPby)S>bE!HU#eB*& zsp2bFSf9MJTIu;$kn>1LE%{oLRTK`7sMAfeD%HCBN_DNwVe$XW6t&c!zcI|;Qx$5R zv>SNOSt?_B*9k}4iH-8c9X5J>-iVx@zO+1VVJ)0zA^$&z{!C4|Uiwv!cU|N_=Dx?H zs^|0A5A+%@M6&ATw$m!C4!?r6X<+RM!?;n$pL-7TTG&JX z1JTQv|0(?Jd(~Nb=CX1>4?w=^ch1t~v&!YtPUb~F^q>9>;#ZgJBc475y^P&&upbVr zz`lpr0~k+P8!~=b=qu5;(c$VW&SOOx>k+I2?Cp8U`FrT&sj#|3#;gNlE4dc1zn1n# z`XXZ|I@vesu(oBNlJO4x+|l!7JL0UItES<<)75j=RMj%qo0Df?vlHtI@8k(R=JGY( zo+0`_hJHh-{*rU`9*<&EiQNapPe~l{v#nHr(Yby+ZpHZ}c9L(<&!SR&)wzCZOZCz| z&>vaP7{B7w5<7Wk$zCJ2e!pcccETpN8e=!Se3!^YmMm#^7se%+UnbCIaiG5AecIn0 zuzxyGzb#P5nXN3xpS)LIe;svy{Bb5YCtsXr%G$pas8_+i-~N34y(tiXFi>BDI`#v7 zjr9+?-xuErW$o1=d|w#$2?O63IKvO#f$xitsiDCItMT171@{{L{UYCR^6hZ2zQR^G zvyAn>Fn&IM0DB?Ea#=Vc-xitm4=S~^0(fwEw{+oU2NRa&*xh< zo^Y9~XVNL|>SZ-3Ps!$IR^Gc$V)x$+hGRc}Nv%vX8iSGvur% z_N$`PvIE#})bOvf^Z>Ie>wkJ2H8v8n8$<9S_x4A-#uDSM&@%Z9*nXcWe*6MOA{@kFSyiTE*HC>(^XR%jW zUU{!qZt=>DSNbg<@#=@X^2LDOw|m)>pYZkg@j>Fv^2%nfyv-}`^~!Fq{G3-l;+2QI z@@21FeC>7DFIjp+Q}eRe#&}y|lOMOQGu54lCpRS%ZDyjkx2xAvY>c&;-MwA$L@K2v z$o$&22yk^%Xvm?>b)w>gxUfpY&a2rFON^I^(bf#nJWLGEVQ(t@9 z4dnYxv~MyuVIsOxr91~_GG%T}wqt^N3U|uOobBrs zl}5PwI+GpU?Fs2K$Zkr+(r&)em(8(uGnGha({xvt*%9m9sXLEYy$Ms6o;oKVGuGal zh_&sMMVm^Ga~;oDdL_vvI=a*7+QQDh_Vy1crN`8q@8eVRy_UC|u?{o6B_RWf92c1% z?dshsdxgm2TaxWVYwu1UZyRy<2Ql_`hH*cBy!Cyq zpUL9+cIF*Gc5Y$|dlPB%W<56+b|>3D^da-+o6W^a#ZuT9y-6nf8a6H6W0^!I+Vu5y zrhvmwQ+a;2$C7Ed5mz5E|L~9Qywi6miu1LTu3_MIf?p~%zAL(8AN-KT{>R3`Efn@Nm1kS z{abj^m)iUC=p^#YB=X!O^5IG3g-PUqdft4@N#s^Qem7cQXk4NJd%wS#N#wgHk>@6n z9}39*`BS48z{Gs``1|BJ9PrZ$dznAJybX1^{QMVBk=lfRKkmXnTzRoxh|iILpB~st zTwk6;T`oUvHx9|?i~IRNT;G22rRTSAo>ngZB&s1q_T?{%8kgTM<)1@b@wnVS2YKRg z`N>+5Oz`=3FNu^ee8z=_I{o<`q$-&xN$d7pY%nog>)PM5qOD$N7vdl{)Ilea_9E{J(l~3=S^GF{QxuGu)D= zzrOr@Mj@|(e4ZGUjPDuL#NOX${Co_rzgEEibDn+1ll%UE=*j)@tyDH0E@0UKn;p{VShw9q+iAMPxb3d1ooyl~(t1YppOX;($*tE%KQ?d(>@$SBK ze2Yr$OvPjE?GSiRst?~4CPRYs6^rnt9H2Sf#ufsj@wWiWgZH8Yd#E*Lv zgS*yvdsix%Pv7sbzAAQmw5zZ*kq(j+3f5&_`Cp zjP<5mH?lemhd_w%@Jy_lY*{zC2Mc zZ95@VDmv3`U*k29d2}OgzrDZzTm+e(nx&2L+cAOR8}0TLhq z5+DH*AOR8}0TTGVB(P&q2!C%Oq~xz@-HQ7*+_&TY7r5`hEq}k}F5G{J`(wE8#=Q>r zdfY9zd657KkN^pg011!)36KB@kN^pg015o_CvgATJ3kr@P5f>CuzFJe?-<X4qKjBK)rqMC$%eohI}X|9c?ES$MVQKz=yTSE-#ZV@U!eKmsH{0wh2J zBtQZrKmsH{0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{0wh2JBtQZr zKmsH{0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{ z0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{0wh2J zBtQZrKmsH{0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{0wh2JBtQZr zKmsH{0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{ z0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{0wh2JBtQZrKmsK2`%d5@dw)a&VcEPC4-wvH_Q<$5@f4^Xl8Olsw?B)cqdj<26_!8I8 zDTiEgM#L@z^T?FvvFZYOEGWsN8PB6rt>Re!_XdZJ)|bZmzg5B`lz$%jXxJK35j8Cu zRzt4sx3#TgJl6l$+J66ce1@U?h!JkMMrGDV&(=2=bpA&EQODVPuAlFW`w?GqjrD)i zK8F6{akmVtXA0%Ub)g9ToAGk)`RJ&%074C3)7!np@vdmm|I~DjrBi& z9)E{*euIXQwT*CgaBjptQIVN$V+;o|hB9{f#x-`nF$CXhaffa3nQyGXbEQ47t4hb( z|L{e&#Czhgi*u&t-geT%p`EQTEueu8HLiq!A7ulEx z4F~Hl)-0s|I(DWad;NIqAC$$0UOB77KY9&wd1P0GeH8PpP_2dvm_x7LXcY?&hV0ks zFvqV~Lr#+#8mV{Z_^X)r1&rY_%z4be{IQ2C?c=szXV;{DPuQXc4kWxP_V67D}=a5@g=A2=w zX~&Sm{Yv#NhB5EP`kx!;B6`cy4Fz<G{I=?;QHTXH}KEw9Y0XrScix;#nz<%2W z?7hA;ST$~bTT5&uKZ&^yGPuWnI5#`jeb)Qz0gQ}sznoEpbpUJeI{*2>%{VQk_0z+Gm@3-`NkUd7?j@}cp-@;g7{PTV9IeSO(oC%$7 zs#i0%;@!|vFlIEQARj@AJ-Fdh`dRF28%D7=wWF?|?^$V&^-t_;e$KKVhP`&Zf_JZWzQ3tterX0dD)*qo`W2*^AclzMC~~YSp+uN;~K&!S7IKm z%FM+anU{S4x%@r)SgIe;`gvNv6Z$s-{9ZF64YFrC%6MIPY^i$TQOWb|4PoJuKQ&tw z-$6g$#+>|GM6IiPsPeI2hgHi{SnKzPRnKXoqTvmtT8t|y;W$bSaW!9UYSL;4i z0beblo}{5#a?qFgd9$_oEZ*_r%fcS@rAqaq%xJ6jJ^XHYeGX#%9Bd??afb2Lc5&+( z``y%9%tQBGEc*SoTE*zS(H(K6YNCVYE(OM|5$uzftClkhuph6Gz4G$>GVH&raMp=c zSj%oi&ZEYa`JW($pQGd}{e1>9u{k)!wfR2k4(5T#PVIaA3TgLU&!(cnwRsNpDD=(4 zYMtn-P|E&TKVS7M^tc*EZV;xz_AlW}^1o}EYFR$_bd%&R zeUSWXu@C$i&S%nZ$w%gl#2fJ1e6?1!Tya%-pI`Uv4{pbK4Eg%~mA?AVs@m=Q!R^-2 zeS!9dvW6nz%=+U-9rn-}*!O2*Z@4Vqfce|)#eL&a#H}lj`wZfqxg4^_3H}Pc{Fr)f zq5XX`%G*gFe|1>(+;gcq9L0RfHL2n&S6H9Cvs&r-r_W0vwd89}R#7-SqE0u>s#NRh zE7i3!hsFOhQ`Ay_{>CtWPgSUO(r(~AXQ_= zh5Y{<`ZG1gBf6Dy$B_g0*QtCJf_+>#{z_ zvp7Fq4WDx|YTb$*p`P2%;QbAoLZv$C-dI^Ql#t zp<`HUYp^cwdDHoZjJ?l2hj}gRq5pyCWz7E+e)hfUEIo5sxt|9hU-dg@>GE0Sa%m^? zq96KC{|51^%k>dYpMqY-?l;)a23BC7MC<{Kr>qSbzby2X=-X)Kx-8D8MH%Z6tOL9Y z@{;oy`gkg=?vOF-z}QNz1?&N3(oL^!m`3C(gD%Dq=>!-FUk@bx6D^4x3lXsl#HDc@cTh?MHY;vnHcEihe ziCkpKl6H4tT!Q&!0&Nxt>O0=2{oMikrvvrd0`**={*(90bABClfBgUEJ?ET!UkKR0 z6sT9hzu*3R{kin*s2^bu`kkC{LMpV&tg+7I zK5N*|!e<=&k(~eJ+`aFZ%0^~bwu-ET|B#)lbbS{b=cIhU;r*F?Z4Bo!oNM53S?&(` zZv3u}C;P31chuRk{ygPjwjC07} z;$1S<|DulZ0QPpw&1GSnf8-4Lpi)aK@GeV&d`3UEc zA4{Hdw`^=Swp`PUcVu?I<1F=U!tgy8_qP$JRT*1znDr4AD{^ne%ribBgg$i$tp}piQe#K9#3T?&Ne<}L%WAPzpQGG7Q zH=eAe>+xX&&d_8TVhNn%(_uMjqdDd$G2@9>xBq zLTc&Zzgua4qfXABIEU!pDBSOR$@3>+CFj!J%Dg@+?O#G$IqweRJK`xI$Hu*010 z`3ly9HAS`TbF5{pm*DJ+dh4fD4?bjC?mx4l2j4vU9W1_IMw=Y$#82+~*EjpVaQ>h6 zAuBq!H-!GlnEhP)lHUnVH~j+7BA54a`R@&XF}@yhnENtr1K$Db{VKZ@^P~Qx>dE1q zdl$y0{y&F$hA&2WGSrj7d$0bq>QVUC5N?`Me**k;-g;g9bbSBH;QOZj zH<+Us<2h%guQ;cJ!!Z2(-~-|d`v>IzpEpnaT&7Ddh#$jyeE{Ec-^M#s#u4A|doyR8 zCBpF)wD}6&w<7-!=zKs1nSD}{P zf3fO$4!#EP`;aif03X7geG&Te_3O9juiuvud{6z~@Grb2)@Sjozxz+QIc)cG=tC*L zKS&<;e;O<+5bqe+qkrbh*Pr(LE_40Ih%009+^D0!6MfweMGh__5tobXj}r4=gUj&m zTrT)+r%w8~Q)IGVSnApb-f>#?LxwrN`!IgT!umb32kRDdFdtH%UcX8;XOr-K7~kQC ztF5BEx8N7Qe}wa`$W7LV^y#zkEx%Lz85zF(L8wjsWCtjzVLC-z3{R-ya0DiN~Ko$if;qV;#k3Qye?>S%pI@X+me|ODI?DPBa zyYBw?oF(oWFdF6*mgqH*M_%$<$paO_*d3E1{N?c5t-@aPTj=mzv0M0~o4ezM>$9uO zCE1l%!#1-d8%BG~v-~WKnY8&syQF z*egXU*KJ-s>XkbJde=&+&w27kyz-D&zUY-Fyz;bHR$VDA9mn*_X0N=>E2CcdIj>xN z?RD2LS$acL^Rn2+cw1tVAGfbF)t!hZHzgBoW}>&ZtJhO(jJ28FyX}2$;hi$A>s2!IG;=Ns|lo#D>Pj+sVM!5Pq zlO5gd3F$M)Zc4<`Zobl&&9Qbfl}KpQbXS+z5$oKkJC9hs2~(DyIwv19*4~?lwe6He zn@W#!9nV*KCCMc^y3^>|!p^?-_75qg$JCtf<5Tm!mgYqt&@29?Pu=pt8@TS;;#hp*N_qk6@F~5oG^qOl=w87v`DT+|36@ zZRQ_fJGofR_cjiM(y1;t$BD#xH}?s@lq5Q#Pwm7s+TmN@q&vN-tG&HzJ3w#TX>RZ8 zZM%3<`&-`EUS@u80>jlQS#Ij>>M%Da(&?_PiB2RE_eeH+Ap26W&57kgYD2uM10&h? z2Qqa3%G*xePEYcup|G>-%W+Cg4E%7F!4Jl=zRTr!b7=BGIewY9&SyUb16wYah!OR> zGQVPhZ!uiw=Htt2w0!()r-`%fdn8eDdLd zd@Zall)r_kRqE$ss7e05+~qfs&tgouQhPt2yC;!%Pa@wviTuzc^1>u?g{gkNe9TGY zcTXbE1mquw*9&pW1?+te!;{F5Od>A?B4)UUxarEVFqQ>Rt znF-kYabFI^?LeCgaWL_Na%tQhs7YL3o)a}LKkjhA{sHL3-j^@b))V$81NMHNH(f4< z6YI|Rf2jKWa-}!2bL}$$x$pl(!2d&V8$zbO{6z_a%eOz|oeN#*gxo(r2IV>LeCEp^ z#lwX-9|`2?_qS%6csRFTizkt10`mRu6ZcTS-j7?DULJQ4)ew3x{yx(APK;v>KGM#Y z=k7`5!vXnW?Z4zb81Ns}UylP+#>ha*He0(DfMel?(gS%JAmwA&yVH3Uu2!j^(JodV|w||(^940_2dO_p3Fmg zxgx$FU2_~i&x<^{Ki_=&>7LwQzkb{+A)hADOmPIx-h8;mv-kJ6Jvt7~#hyH>>yGmo zPrlfb`|GROlgp1>(l3|qM4tHje_CG9iQ_xWvtQ}Shdn-NPky^E_vZ7Lv|g#ZJ-Por z^10myc@5&O(d{SZ&pW7V&YoO{$8*f{M>ug+dEFi^Yd4pJm=-_bJ*v}yLE>q#_KDde6c6@?eX_Hb^fl~iE%&d$^HG) zUw7Yuq4d|CeiQckLk?d-^LD7Njh|@LpGEp-lVoQy9q(#OEb3DFn^tVv^v^F{c#L=V zrQ=)FCbvlKOvPjE?T~vy;R<}zhQfc`T(}{O@6)gVXc^w9?xh!i<1D<|ZSU{DzS=z}_%)@L;ZbDc?Jd#$x3~Wnur6zF{`a*n?Yp$} zukrjZaV7E^l(WB||18h&B9!B~don2&Hn{zN#>3#N0#&@L&fosnhh0ng?kc^4?Mroh z{--1GQG`xyQJqS)xBt^+!N?>)0wh2JBtQZrKmsH{0wh2JBtQcHO9|ljF8N3PwZ^}~ zE&pD&6?YVO4EILdaolaV6S(Ey!*0gC1@|X#^CAHfAOR8}0TLhq5+DH*AOR8}0TTG< zPvHKwcYZV+n)urpVfCc`do}ZP>W*W1UC78^Q1mWKHN(cj-`$y_ZQZ|WGohdOd!qP{ z0@S*V{L=sXJ-#oNBtQZrKmsH{0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{0wh2JBtQZr zKmsH{0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{ z0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{0wh2J zBtQZrKmsH{0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{0wh2JBtQZr zKmsH{0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{ z0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{0wh2J zBtQZrKmz|~5J;k%TXDDJ?!euNy9+lj5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH* zAOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8} z0TLhq5+DH*AOR8}fqy&#CzYB}8ttt>{WR(Y!zzxHU_KIG;`%w|kW0>p*o9yo*3D&kEV@7*wIzAX!}I7=t2oyGy}@Cl^`)`?ZGhqNYW|YRI+ywzidw$NIl%A47h4+^wuBSz)n` z2ZK?SeeS>)ra!wv4b@>Bax1K%vHs`B{fIBQ#`+KI_=AR#wT*CgaBjptQIVN$gZUts z3+sI28av+@g73As!?yU$H&)=e(jM4VrDN@X_##{4J@MGZI=?Tss-onC5IWCy&@VTK zAC~3dtPcO^HH`hpt_u4o#-~uNh6-T*>Wx;h@L-ODmzlGimQ}WJc>S;#_1;WCggGQPrd-(MHBKuqbi=L zwuTDzGEZT*z$za5uvL7e(JF?N`e^}kvw$%|Zdn;4!&K9bA&2{w>Uk)nS|monFd8Jz ztBz9_hF)@Ph5j$jIxQo$R&ma-x$8J|NAbK0&tm^e)W`atwK2aMOqE$LV>j$Ld&l~p z8)qVV%hL@7bYO|`de)Xci;t*h_g5u$zW-sv^*>mmD`~Iic;*t<=J666zkSZrJyN0z zv~RF#-1h&r#75dnzy1?su*Z76IlI|?*89MVU?6_{E2J_%R_Mp~J-MuhW4eFg2E9+* z6SChzU%@Xw@}9GI6wjH^>85%$W2=R+C>S#ueg*jmO04;Yf_@hJ+J;fA<(E;{&-bjf z$NDGMksn{;g*}d=h$a5tLVU>yb7A)!^Oh{eZEqAYukmCu_vV-@vOj}72RUNrCC0k2 z+9Tr`K|8F`hA_&N7>8BXE!oWQE!pG9Z*etd;x2s^_#((eQ>+Ek}*Z zFy=Fn=d6ZIc*RSp3e^*H9N1Y!usEbLQ?kCzG4w~>w!L{2%--Ix%04w)_N(UyqpJR; zhGV$tr7dtS=Nx`_7-xH{N9oL;)eY;@_RQm#m2zzKK}o%$bL~- z_F0T)uCB_Sh1gSW{Xq7K6{?sqtYr^Zs%-rp)nj68>rLZlnTT3))R>9fXXJ;Gcl0u= zS@bKBdn{}%o24?-PoYm0k<9wp7#|tSTSJ-k=I7PAPgTHIOQ4 zg!rN=lFvAk_-ebjb&dUQYAxoKyO)c8|E*RrdT(?` zT&bGqpt(zdS!)FQ#^tKz%mVCBD`fAuJiiS4(W;P|5v#D4-H4n=jVtp%K@2}f$yfUO z3}j++aEfaq>%_shitN(|2JgnADcKd#CyESxQpuM52p-4Eh{I^8s@Qmw16RM*NJ7XQyoQA_>#8^ioP zRiV~NyMgzdr81Uxop7|B*eGw@VWZdQjmYWgOUv^X*1~xf^8a(_&(xIbrC;@U7ex+a z?t46{dOnZ+K(Fy=IICW6JFUX%@GDrG2G*W1jORX(^*Nr!d-`hloRd-OR_qA%+6W)*m>8@hp|dG$uQ5J1lgHb zPMmA!>s}lu+<0d0OiMC0bNvx)8AD^_djDN@O5I%oyw*(KtgOi{R_WB)d!N1c`JInC zEgI0S61uUstyNE7QJhZ|rH03`*4ATP-u15aeI9$4dj|8G*+c&e*7KNu4So*1X01AR zWwD=oA#Z-iTGh0mSkCQuUJOA0l{XQ;qF7&|^a<#B?B2wFxPLA7J;dIR@#M9^GjElH;)+#X7*=KEyfy41GM~r`vhVx-qt#YZm)!VSl(UJa(+(eWMy{oA)Um zZ{I7HJx_Ka&blQucP+-~&e_zuM9s-_u-T1ub$_8AbNO1eXR!VY&~Gi&Up8H@c;p%i z>@H_N1##HV&O-gn>3$TqTy23J=j-*eyii|0-A`4ap4)r+!|NI2mz!N+$L}oOYuMK9 zH?PGgY%(nvyOE|nEN5A`pxvDq7jJ$kPn#8+Ed@?sb@U(mp?4d`3=_D^lO zk7!jX_THBHMkAhA65bJ*_dBqbTls9wHcsj0lm9$~ePc^<(t3#ZvUcoYzruSQ=a#wm ze8qYH^M2a{z2T=-e4fF3r!|6lN<$p1V@g$|Up$X9BhK<1$36dz;tXT!M=no~K+n1@ zXovIK5tcWP%t?=8PEu-4x)plf?_6Dw=A@hTIqB!Hr?CDG=v`g2UXi{R=Q);t+V)G- zmk7o6o6&SOT)RJ9*+DDl;i>rSceP#n6m77}9r(G`eo~!6(La}; zEo-kTGpanQ$`@6cRi)eVtg2VebJ}00N>}fejY_^zmF=qRSLMU198~2|RUT92NmZUz zWhH(&!1*?+a>eyG+_-YpO%08!10BK6P?sCGFPi8H1;bt8P^TV>$76A&=m>P`J@HsD zlt|c;aCB!N5{4`g4|Io;A=eJWtH-+Z?of9uz8jXRu9}9N#*imM+qy&1WFQ%iMKPcH zB1tEZ>o*kX(z`Gb9VwA=pobIsj&KAMR4JS(&vUjf&c@N0p6Cm1*E>Ty!$Am~Qjf-x zdSEB40v(P|M_$h%*k?yyg0s!Hw-vFt-WiUEf=Q<@tcPu&P-r_Y5Cr3~L_$T^BjM-{ zZiK5Z8t(3igt*Tj+Z75VoqV}3+X4|i5enI+$yiM94n%j`oky&ANav+T)AG>+k$5Q3 zxtkYlB00r%DqrpuCl~7ONuq1lMf)O=j}zHrs@J;ss8+SSO%HVI!R_HlCzM$GJs8_t z^c56HU}I5(#QOpkz1@QMllrb$d`B{rNMg0`3SjF&iEIIMl=bQgJugNb~(|fW04lT)E>n&n@M# zpUTrCzvZLi;w4r%UK`!GN`8vfxLp4RUW$eGuDtysa=eo-XrH-=eB>hX>_y~&TI7Ed zYM0`yd+go(jf==PdF1Z+_ISqka~~16-Q%YX_B_6>yc2c4+*wm z=_;<9XNi)hRQ|fMzg)=;Prv>L^qh}#<;UeQCAX(P`s?<~J}}_DytJQU9e-YLzpm}$ zc{mzQ24kI}?NCX1)M15$Cs(ZL+FI}BDm4&0p zu5La6*pJb^Zs&y2mPkUi4Su#Uixj(XzL3Mw#&|XsIqk1>iYcw9O1!;Z zsQv$bwPT5+Xwl_uU!;`EpB#bz13h;4j6b=~ONWJk5D)@FKnMr{As_^VfDjM@LO=-o zi4Zt)l@Gte^5J)$MErNl3Eccw#C^DT;@*Xue`n2qYs@WE2^;C;Syrp2GmXeDO0BsK5gCn%TMO2hE6mr^&SpKYVz#~&>- zpSYEB%i;S_^?GyveUv^>vB5l4?KclpXl7Qc8me;Q?xk!)dMTbKW*fQ5fu9YIXl=(Q z2Yyr#-#3(kzTIyOQwhy&_tUUr`(xV{acOekhvqo)!{c5eEzP-Kx708XwNU!cu3l#j zF7ug(YY=;a($5`wVBWK9Y1o3#%vxi3a^U$XKkSRI$$_Wr_=B33HZ^~Ga7l@IvNSc% zL|+HdSMK9b{d#k#-Ur_saQjX6IaFVY=Q4Buo^m_Z!3SrW9PjZ*XW98Z(MCDW2_ft} z|7j|RUl!$HwfJ9p9h^t^l$v9{)Vyp34QIjpwKYaAd!NsIy_#}kH=+*>G(1{EIUbwW zmQXIM`Rm4)L8tkK#vd#*Phbp2{AU{$>y#d?pj&Zt;k|=LB?Pcy7is+y4XVlLOD1=tG@OsVzKqBbN2h zo>xSp+IGB%uzrAkm@|o|UKj`1kHL#n&bINzNp@!z~Fu*FS&97So`6x=P zg}STlXST1Zo4{I|kGlPQ*E(}@AV1GJSH3u&U$L4%EcX8%;^Tk)*n+XzySTj}4XzKh zBj&YA#&ctFdt>^ykY^xA>>-Y^*-!gkgscSZu!idVDA$2=bE+YoqQ>;(ht@;C#~2mr z_uKkLTYnn*x1e`2bAGwKA<|wNxh3t| zjH5raaaZF6n4R9*Y@S)f`^ocz?NoF81o}Cyl@I+r`uVOs-<-WPvfvxxI&6 zIKTHfzo2HGMt&b8W|-*M-V^UOroTmoc^>1LsV+Ac5FMHE)zb9iYblq~jMWd8QMzUy z_39Yg8eRKss)SaKX;&ck`9mYfyZuU|k@f44d%$n3UO=gNXV9n8lGK((7#|+XTYaf5 z`qydmmrCKQ)z=%=s5Jw9k)L)VIs%rPM^5?R(^d;`(C5dIf#ue5RNL zTg=>!_2vhO4VYKXUd#G}w;H+j&$ssni7ML%^*sb;Z6#RO3#s+oGVC8~d2KHoT8;gp z*+=sOrN-(t$azA$dgx`uaC78*xxddq#x{p%I5s~=-NLxC?9743uHtq-RW_xij?HtZ zw?p4JLYrA%j*|C;np*0;PI1+=vPJD5Ga6afU!cDYeQkK(x`~-G+wZ{_=YQv1YHeC_ zwt;i!K5+h3*z5ih?>p`{=hKXL3dh^8+I+W)TCcjMxX*7W`@_5NoN$e(EjfE#qGF{|NIp7-Zh7wYR7!aG*Ir7mXG&=^x7+`rwTj2`&XbmH$2O$K9X9s*T!WmRo>QDR zvu4fw+hx^xJ-$U&E7*AdsJbr2D%h0!pRB0OLnH-PxDAob? z_94#sXXxV@Ki$q_){U{{T(j6;3;V-;;jv>K?;F)v+q_Tlc>7+l?0JIsUG=&pGHxq%;|m< zw_I(39p~%yv%FAWKHX1Mp`P1&`orrP^&iM`0-SPj`ht{-wU+~x;_teYb z-)(=P{@(0~Kj^8iL>>EqeN7GwJLmJC_|j&J59f2uJgMP)jx+G!?Kq!*hK2{1wczu{ z4BYGO^FM#~;IsGPno^T+W|6NrX`dLXISCYowHn`Y^_l)0_n6^KW} z(QOy9X#VW2O?s$5oYcDl;Yg@+`7EblBH0;^HW9^oLQ(x@yLO8*Y0_&$u9+RbJ`(Oo z)c17?oc!tNQPrk%%{Fc(h21H4Mn>2E=)v6N~9d<;e@^;9Ki%t3TMjm zob8LVaWtkU`hwf_&d|1U+S^v`laj*kNqd@`t*Le@{Fgy^&jDX#N)pKbw4<`a>s9;&nb_+%fYaRAkX0z z*j=h$BOd#WZd@gI;^gBV!H4j|adhJzyNLYMMdX#*h5fAX$Tz|4Qu%N4*t`5kJp4cR z5q^3+e%fHq{9U=b5ZwM|J@#(g-+AKlhIy&{QzbAcn9sc`t}D;58kZYapTV{K__}d# zo>MG$?X#B^%U%1E9{VrC%!eMk@)x-oF4z8udT%+>yxe`idF2`P{&3}A#lxjItnhH# z`w{j%9{X>@eja*IsmA4M75K1{yWEcA0|{RzaI?Lk=6Sy@#fKFo*X_Dx*~5e9zg2Q~ z-1pgfB76?E{kZuzD)~>8+@1frlK&EACF1gaQ#g(&VUjAC-KC>X(i>U*E^g z_AUo|m?3*Y$=z|k19iS$#m)B4dbX_fwv^~CB{%H4Wu@$r=qyUM-{j%>o{}F?>)Orp z1toWy=ecE+ywXwr4@dTwyMbDL-cuUdp|v){RJ7KkV9I;Uqpz@Byno#xcm#wTi9 z&PQjW<%!+h$v_9nWZWsYyHCkbf0CBRW1WFyz=8R1`nd?Ld5V`7dEx!Tp7``7dmZJK z&N$`ki}A>o_Hd(g-{Yyvo%V*iKvWZF-%%A5g>CavqLH~y`;96wc5^*$w>^H3Xoc5EhK$VU(FXV0LLhWZPbS!byDZ0GviNHo7N zQr1&{74@uU|dU)`uiLkAx$H4onBOJ`dA z>5NulW=m4@4t|1inXNPoA4e$bOF!F2!;e2&WansdKysbL;!q4b|!z0Mq5<}(l1Aoc{MpF8xxyl2y#d?pj& zZt;k|=LB?Pcy7is+y4XVlLOD1=tG@OsVzKqBbN2hIOfbK=8wJZ@A8@Nfd|HC z=&cW}hbHix@||s{q4_%ujCEF)y1V*}kf70(-@)sN2tXturSF^6S*i zkK_5(cupV|`+pDd2{O#JBa27;Y0Qg<+7a_wCF8lhctlJ87V-?_h&{wHe&wfq=>J0{ zXoo$Z&W~~(#<4l;Pp8_o^skW1@6q2v{eD}o+4`SC|CWcx>sm=2?Yg!EZy*Jd33p^dv5C&29V)@Jj}BHj<5A8eh;pk|&%e?CaeFfmb%oY+#5{uUYLd5nLiy4+j<*^JlcrXOERxs+zCez1(v zHT$Sn$6TnIzS`-4Q4?SuLr0^_z4>|+b5_1rS-muq=1TR5~D z`(v|@<_AiR)oYORgm(4N%ZTCT$oX=ApMi{R4$p9GevZ0@dBCzW2OhhM+x=A8l$JU+ z&!OH9ed7piW_>wI-h*mtsrNd?Rny8AwSUZLWLv*!Q1*v+;k}N0-Trc4-Dj$B z`hIwqF?_eDy++1x3HI9)T6NtR<^M(zMRNS}bAZ~SW+-DH?+(O9e^ZW_E z+?e*4%Q%2YG9A^K|%%D~7{0(6Ko++ix+;0De)+!#$J5O4+9ovu^ci7nLc@1)UdQNfP z%$hkbK>pu^{#<3Tp8HjUcVo$+)ZLG@Q}5TYe_*{HAu3DL?>f(tpPqUZYg2=a83v7c zX_w=9oQFOJpNmtpd2PS1_qKC*zrrS4=8VZP=(t^w=3#%y^M4!dDxn*D+gkPX6~*~f zQEGS`Yi&K&w+XZzP;pG55a7*AdsJbr2D%h0!p?A$cYZ#f?8QLF>J z3x+u7nK)CO@zd=*X5AQD&NYkuxUfIm7alv-@xD=wwaxn!kGJm?%bq8@5NF*In!6Tb zbmwepU83gXIoRyRy23j=ug6@zR_z(A{{r+|3-y;x*DD^mh620G*-t?n_Or84KXbYt z#VuD`V8{7-{VXrkmrwUoRjB9op8oK9#`xuC7ufMTj`te2b^Fb0F$$YZ3&w7wX%EX; z7A|OaC&tB_U&_;Fg{R*C5$!j5?4S13H+t$g;}zxj@`uHBoNuPa7iWlR`M%(>KkliQ z!~fL$xj=t!_QW6b)K{X8{lLB^2Zo)`BR}z_%@!X%k7(ve4WCDF<{!KrpGQ7J!-LCO z@cC*6?)CQP9saDvpEVBGl$wMy&g8%g1?TB!aNhRSEiR##um`=vXB;2-_UM(yxy)BJ z^I7-|Vn5>Z-#E?*2gWH~GT$(%WF7qb%uJc%8_DhE&o`8+PQN~ha~aMx@V7d1JAbbF zsU45^Tb$kJoiFOoQT6%CpYIPoOBVKLsJ=ygzT*D)&4UllFb|?Xk3U*!K7n({f5*FI za^OWfh7ac+?5C?oh*t6$^4j@EBjG*wW6N6QoP)4`6`Y6e#k&A=d9~kem)de4(W+9s zds@&A&nvkt*8L9bWvv%#U;O!$hOR^3d4C$i9t0*w(A{+GynO~9^5Okij(wN+C--yf zV#u&R+h=zk;K_l%vw5hq@X*SuY2Gi;znaUa*Tfp*^UUPHPo{YC8TVD3y?MWGZl9Ol zk3MG(QhJD)AZ`lpEUq6{b!PH=$T=2Y{CUE6Luv~i_fmGA{du{C^^oNpyS+3rFa2N9 zp4+;g#rhJVxZV?phJ$(_sn;f&YCD@|>A2V_E?d7V5RZnV+b(3${MlQZ^iY2|sdoj! zkx=LISx&n|vNIfQB8v5dqWaBt?G|Oyq}PUAGuwZCB;1jx@9PePqsxO%b3%()U$QCR z9yPCnG8(mJ<97b+wVx_61f$kMx@OuR*X|EjcF+p^Ka7R`R@=2t(FVKRfuCFLC)Ftw znR)zdSw~g*qAIhhJf+HV{4#~@_$ObwsyC`~lSl7Z5w$CMk1Fq1Wk!`FsywF3tSZl{ zveKTxmZht*QI+kgTygykH?CZDQ$yqGKu54M)aAzQiza$P!EjeN)TxK!@mO3bIs%<~ zPdpY3B@(tI9Nig+gdq#W1Kpuy$hE`p>ai}pJJcPE?}nwStEM5RG31HRw(d|g8Ayg> zQOu{lNYV-9`VB?8^e#+9M@pm|=;4IEBOJj5RSIXy^PKIAvvD-0C;Ech_0G`Fa1a8g z)T6Pa9@q)1K!@Ygk=Jtw_Sw;w;B52lZAC1ucZTDkVAAOe>tP!x6xxmp1i^SLkxa{LDs#Ps(6Sek|tzEIQze&GM z4|MCxY7@)UEcZh_sU`ZeOQ-{{6wawkqc?h(8I z3&+uod+Z|eQx}m};w5|`KPxUG-*gdq&qd^WFCstUk#9m4m*R8GWAFC&)J5cQcENsC zUPP{Y%OzJ6x{){pvj6Nb_>{9O0GstM{}ke-ICs;(W@Jr`un>f<5HMYFo76tz^1v_= z_${#Vf6A^~)_Nt!?_-^DQSy|M->BrrJbBi;d8+r0%dOG%ulQf5{P>mJspiLVt&&$N z{|l7;|K|Ev>t(x={}+@zf9&N_FmLUH9X>5B?Wb7BpV#xpH0NVfI2uj{W1XSpF|yCN zY}3gfr@}Ej27CIF!R?gTod^aZ5eU>1_2DxZs_us{_R`gf({nhQ?CMTJV?Rdwx+!5l zIezR@VOt^zvu%jM1x(#DpX0tQ7>Ol99H?z~I27rmovv3~#>Ujy*K-5yOhg@xlZCCd z4~pzL8sWpNI!xLs`*@4kfq2rcb4K<9oYBspa@E$xA6{)ae@bOn?1{747E1PnJ8h@I z?O`NF%M-i1lYtJD$+%N)cb}4>{v<7r$2tSafEt>AQ?UrGG2~>whNe96dSNN4bM2Hu zjeowr7>{gePe+vQIr#IHPW$`a1*4ic`?0E^C~TXT5@qK*?N`inr0k{zx7*&GudKGG zuwyp$v+`q3`&Orzl6$JeufYqo@8kuG3m-X(E^qrHrBsA~5D)@FKnMr{As_^VfDjM@ zLO=-o2O+?}C!YHC$A6GMNjHRm5D)@FKnMr{As_^VfDjM@LO=)zfnoyNug1UTz#slP z*#Cq3KjVH3cLw)<+y`(U#C-_&h?pne;MUQH5$s6q~;y`1m!YYX&Aot60J@@+eX8WKU!u!aVzCE z5$!uvz24k^AEggeY%mX1`^^ItnwizAhE_pmx5+l8KaFSImzwvYBZJtvy!mi^j_c-> zK`t4s#LRm0NEPR?{1SOo738rL&l9ta+~mN|21m5Eo;L7_Xu62g}S8m^hL2h~Mpo16IL>R9RqcnKb8LhDJLj#|(JCXi zctqcG0=h9gH{+S@{{i*MfoDz3uR5JlTX^h7EbF1kf#;@}xcU*z(I35l-kqbV%N(1( zF0gUiXOwQRK!>rwct2~Z_H{<3)Bdpn8`g7P$036`)>~70m-B3|W1g?$U_J`|_By`H zXTFF2pnpTT53PqL@SO6UZK$F7I}EVOYV+&PKt75RYqIXF{mk}NbrV=?r%<<_?^)K=gJqy^D9;ph{gWjLwte^W3_j2ds7-*A8JR;Yn6=W#^Uxh>EA-0fgG`iIL2l_ z?Rycj612k_s`H~<2hPo@rgVzdq$iQf?=eP&`u(d3vj#xeT3IM7j6T&(qlJo-Cow)!!(4^k4kcdK7c{9zXS-)k^E$B5EDe zuEf4`1@c@_m-4SYo+zc>fMvnX$RW}Z8u6uFn{o7qHtuSi0JGCuo6R$ect3f5u$^j- zpFlsyweq3AM?c@S=bN+N=5~~rk1Q~TIk)$a3+MMf=NHt>)5!0G#0(Q1+k4{OHR*4W zVV=i$W~$501&BT4t25J&uccf{Ggd!XM(LV;)T?7`Yjo|msS;W_rd@&D=MRk_@AfN= zM%J%G?g78CdI6>8ok5>UOHx}FVSIQjZ}p|N=wGMJUn+&KR$p&eqt*=cMSk9GYdnuV zoP8NsQ{OJ5mr@gLw(pS-itCFJ>lO5s^O<52Y%y~?)|($BHeg;kdoAk^-fHC9Ki}RT zB&uv5)b|jWwUuC9FQnFU%dmf}<+Z(VXf^hWW*^NDlp3qoAm<6~>YlAf$`-YM z%xGj?e}Vos^tIuA>n3K(Y`+Izod2D3skLdz*#^#?`@s2EVXym3ynnghoKG{}DI9OV zYV+MHYQ5^3;y%Bj><{n4`xg1S{pG&8&s5>`{qQbh_-;>ojf~+Ee`?DKt-5Xua{_Z3 zYyHZhn=l`HRNS}bAZ~SW+-DH?+(O9e^ZW_E+?e*4%Q%2YG9A^K|%%D~7{0(6Ko++ix z+;0De)+!#$J5O4+9ovu^ci7nLa}9EOdQNfP%$hkbK>pu^{#<3Tp8HjUcSXse)ZLG@ zQ}5TYA7G8|rOYj9`d#N)^3zkVVr^K3sZ!OecHeIiHE@wXlaoEq!LjBC?eiXM{ZGj!<>-Dp|P+vaXPgS9w+k5)M z>lx#hn_XbX?=0SH*w*bguf-^AGA$Upk)}N?XIZ$Q-JKX0Z++SPDe+J^S_u-mSlW=C49C%^s ze0eYSLX72Ve+i$>Q(NvMT2+d@w`IQ3i074rcLe7B4y@%?KKrtbQ@Z&)YvSB>jL%w- z93h%ze+F5M^}X)0=dOD5y!8;yRglw>W@A=*G1fMvW~ILZIeX|!gyMQnAQ}$pfuvrW zXsYdOnx*4nB`#aPD-e%{quVZI(frw4oAgkBIH`99!jVwt@>x#1M6xp+Z6b>GgrfS* zcI_5r(xlghTr=B$eI(qGsPF3zgrm!YPIE$wSYNU!-ySuugEAVmX5)7L?6sdN@$q)l zT1eMS`{UaE;mQtLK@U&GXTPiM+NWrPUGBiot@e}Z6pH&R`Ps5Esyw2~7ghN?Ri09% z+p>JF(@s}qqbgm!TW(bH&#SUWmG`T1P?bkj`JyVbsywU8@_8zrDmSWf#q~GbxN_A^ z4UMY<9l_2}mm9Y)n&=4y!(HJ}ryh#OV{xVE2z2T_@mMgFNZ68abY~zEhAa>dbcd25 z*ABz0$GY_HP`*Sh$qR<*=v_SqzAou&Wjb+_Hoq_-|_Rr>(%AN8H#onH*aV^fRU z>DsiP8eu21FUu-aYB+!yb6Db)YeagnoIRA#J4Nl;qMUzRz`rZEms^2)Q8R&aLzF%C#S?D3*T_RUdln%3owPF4w-tUXauMxbIKLI&a^u-cPRlt9ZB+hY?Sn zdq2XSJpFa^OnKry0zdQ6Gk3mStr9w<C38oJ(+`vmoE!-=oHHQ;N@eR^vL0 zo8`ysnq{Tzl1PIc%d?(5`Gvr8!z2Hdk~_`wasRWD+taWhANz%h7unK&igo;Xz5RN# zkFViqI2nv}hL*?3K3lL&Cm&3Xg5UNGuVeoo;jL?CZI~@l9=Qc3Rvr8o|MU z&FvEcTn6GvyUt144`@gH#-^P?#}}IOhX7m2r+uR3iQV1FKnKcX+$p!aPsvb!l9tC~ zoq=S4mUpO=pF{Xx`*RT*@)W>lqk@a)r#&&~W_x|+RnEBO>x=QomiB7~rJ>fdF~@1| ztf#!av!CVbI0D&wqCI_Btn&}cIx^2`@2*dl^5v};#&#^6pKkjR)iK^~SK@Z#(4F5JHVD#vp9R7>Z<+uoy=st^zYe*^@+jU3cF;Ey01 zNkIq*0U;m+gn$qb0zyCt2mv7=1cZPP`2T}I_G%w~uj#|@wu$~<-2C?g{JZzR#r=KU zKfrwyH~;;>)41`yKKdcGGk8%G5_w%^r5&}X%2nYcoAOwVf z5D)@FKnMr{As_^VfDjM@LO=)z0U;m+gn$qb0zyCt2mv7=1cZPP5CTF#2nYcoAOwVf z5D)@FKnMr{As_^VfDjM@LO=)z0U;m+gn$qb0zyCt2mv7=1pfU9{E}#XVYFkO`fpIr zYDR8!38iQFXdi5g>?a3)U`}e3o|sL!3C%w=ruphdH5xkjV5xcVK3_W1;!kI^5;I$p zns@LMl*??TVffffBdgQTw$bq8kCvHF+)BCSMEed^uQ&JKN9h9<8_Yx1e)B+uW@fdj zp(^O?Hu{?MQan%0Hgb~#KN}p;+Kx{S{HP$lZ)h*{?S5mJN@#AopN1XVAKSL@L6ZYN zG{=#j?T@rH=YHK%!#vbN=|8)AojJJ7XCAIW>Ukd!I>tPN&os9=j3CdT4UsxhW>BH++t6*rNkWj2HBNNB=h$(X;;% z&9NEu*l>Gyj;AhjY#uAHar4V4-NOYsH@<3LXH+`v|Ej=-+jGDED`c?WOGZaGbab~>&T7I@%(BmCJ>ALzlZn)8Ro*?#hIow#_geY z#JpC?c)kKlPr~O6%StY8}(A#J+X~@?21t@~=IfD5c(jWx>wKfyEIbf7-PfM}KJJ zuEq&4JH55pJhO=RtLF#Xspj|z^mAM*ANqUr^IhcX?w`3GCFUawjA730J>!L$>BrYnE~Oc(A1tGE%|7bYF}5|j z_S;kmtsK*?K<@L0Mv!;=l}01$*CF?S-&nnXQuEHBPo*WPEsHQdJeId&{peq(&0i{o zuU21gSfkbq^hJK&ZEHM_y_tO(ShL?QqnA<>ZMN@`4~pxH5$hH7mGhZm5^OPZJJy>Y zBsO4PIeR(l58i6z+CSgkA0(=5AJq2{n6;H)-&jbk=aylATFZOK!lBjJkD7fnKTv9{ zUW1$`w5x|+MhrJc&X@c93}kF`c!p#1bJQ)2E6dItc-LxX>OND2)Az%>jN!XI z?KLunOR)c)(5makFefmlvDU8~x(V~KN5y?>4&qi9$9)EI&n<+kKF^=v%Z+KzEwsOT zesMeQ<3B$|y?4!_r`j=}G7Xe_^(y0wcT|u)|FV2u@X^XA8;l%tc$m&MEGVPRHDz=? z&tdle%nVxP&ffs$@0n8C%$zVwco&r%O5OcfJN14Y`vKPYUdrB*rr&j*B|km&D%PfkwZ{yD`mD6e@jTws zAA`@uDcZcY-`9KFIlRwclPz<`CcKZ;wfw!n__ z_4-*}s4t)Hr>aoT?LGbB^^Eb$%`ULxcNXt8Z0q)$*J2blnHG%QNYfscvn*WD?oN!0 zH@}pp%?eMw|0CLO^4LG^sc-buGoJd(9~S5Q2I}tk|LQ|)TD~uM?2miu`d+>Ig|35>+gUed*d1eOg_4elr z{%pf%`@=P*CgIF7Iq<^N`So7xg&51#{t`a>r?%Wjw5k+)Z_9k65zi|L?+DEM9azh) ze70sAr*!jCpNDXc*ga`IghS(zeLtR+##!nJ_K_{TRt;lT`q$^Jhq|Hf!OgNUoJ(*P z9KyL~?k}Lb=Su4JA;xy4hxa4YK0Ez9boO(0cKSK{8Sl)3vteH%6xVwK(Qr@?B=y=v zQ*CF{EFG7V9@+X`fp|0=-F6|1=Fi^Rq=)*$Nxdr&j)Xdw&vF_jlAYmb6H%-u6xDCG zYquzqCcQS~n%VyABjJujeP4GV99{iHgD;^9huwyYzn98u-(RC!XBG}p1? zpL{J)^%bhz=+Qe?M4wmk?W#OZ@6*g zs+$@bR|h(RouMu_ZeKLf6AFgA!l6z*6pzQ^O3@MM)O+HwU?`EWCE@7KKqL%VARg!r zB}1+qhF6bu>D{63SbR4uRb4d=IgKGtgtm2uqRBuq9E)N;^+l3SAlGjw(xrD{B05qc z{Ca6+4Q=aE+U!0AjF+I^2+^%Usw;@K%vlfTp$R>V~K={u1CVr9oz_4Uo_m^6A5vjLAEOt zNILm)U$zAzdLk6EO_Q;h-W`bUwmXkl@sQ3-kEZ3L2O{xMpmR4b+C*}S>r}qnD^4!d z-IGMuu8a0XA|EHR$5gL%@lma6xiJ)p#6GFtrU$y+afa2@@}}lMaEBg_7U({yhcWEV z0Ol>%MIw42Li<_#g53Z$RA- z4zAqso9A=NWAAb>>>eq++E;qK1N9~ zA2;rxCoaDTF4f&i162A3N*HG^yU`QygDtfE-%+8dV_%U%1U9{VrC%!eMk z@)x-oF4sO~FYM`Y-S@U*owx5%??qSsRXkjZ!(LCGdq2Ydn8)7DbFU}v!|*c?J=>(l z_iB~UVI_Cxl|2lQ9Z+(2z2Ns!_VpblcljU12SmQ`dkovZq1I!+EhQ>Lo#VRm)Aj#- zWv|_8ukNs=ME^_4o&CtN)+_mhYj4*rE2ZQgD*0YdKIfF& z-N)SV`jwJ9&GXze#o67*-0}LhvbU!#){D#8e(~cpy|kZV9e-ZW$9U(M9gc>R!B}T# zd5r8c5ZiPTAJAiX4EFRTgWD;wI}r>-A`qx2>cbfwRriq2Ub;GQx(Y{=UEN7&?8j(d zw{ucxOC+J%2ESZDZNI-Ae{I1?ED>VQZMKwlx^Aemujd9Q2DgkxaDZVm`y>OGfq2rc zbCUK0>}hAvX-;iz{5i#z^9K_9VOuEK6YeBhp4i=;40NDO#+`Dz`;-jzCuw;+))`0! z9P0m;pNr6#mq=-m6P}+eC3XJDE1hx6*B9fFE$!D8O8307X^zwW=+po^ZJhnjdFG#m zZSzv>t5r^WU9D$!a}#d2y*r;~LS|3Z!p3;M?kcCfN@GgvNT;gy$CeAVziE+UiKArE zU)-!HDZ6SkjO3F?=88OQVIbfAOwVf5D)@FKnMr{As_^VfDjM@LO=)z z0U;m+gn$qb0zyCt2mv7=1cZPP5CTF#2nYcoAOwVf5D)@FKnMr{As_^VfDjM@LO=)z z0U;m+gn$qb0zyCt2mv7=1cZPP5CTF#2nYcoAOwVf5D)@FKnMr{As_^VfDjM@LO=)z z0U;m+gn$qb0zyCt2mv7=1cZPP5CTF#2nYcoAOwVf5D)@FKnMr{As_^VfDjM@LO=)z z0U;m+gn$qb0zyCt2mv7=1cZPP5CTF#2nYcoAOwVf5D)@FKnMr{As_^VfDjM@LO=)z z0U;m+gn$qb0zyCt2mv7=1cZPP5CTF#2nYcoAOwVf5D)@FKnMr{As_^VfDjM@LO=)z z0U;m+gn$qb0zyCt2mv7=1cZPP5CTF#2nYcoAOwVf5D)@FKnMr{As_^Vz@G+zm(k7N z;y#Z1f8hQd?pJWjB?N?k5D)@FKnMr{As_^VfDjM@LO=)z0U;m+gn$qb0zyCt2mv7= z1cZPP5CTF#2nYcoAOwVf5D)@FKnMr{As_^VfDjM@LO=)z0U;m+gn$qb0zyCt2mv7= z1cZPP5CTF#2>kmI$P&#jjP|OheiHSpX5>bfPf zj?Z!3oHEEIqm`IhZyxQdi}F~0i9D(b@>q)JiP=VOa^Po!BU;<>$$=jg@bC?#pl|mZ z!&E|Z+x;}`*#6kI<%}l>erS#(KRoUwV@8@;UboaR54BMG&#qo)4leVVhikxKg3`|& zdSKqOYiZbm&&*n5cyi$RDL?FsugQU@?D&J4mNqqidT>dJd9pM$&jj;9FlW|7_3O={ zdLMjm!0k8L=TLnqp3BVrd&=!t2Opeia=gbMon`0uL>uKeCxo!`{HLiLep!@*)#88Y zb&UP!o>FrR4q9Y+rL5cPh;N39$qt7&x{=QYc!_CwD(wn6`$ z^H%Fp8FEkii`5ZFPE=^K7qUp0DFz zJ_`Q!I=;(izK8yxe?z$st%oM?obsJ*sG<2g46w>-^Xtw)K8g}+vhJ+?%=T4v6Ig4f zP`97&T4zoU7a=P_JFKBPKgxCB+}z%jPGy_YlgQ=w7^6b{ep|oB)}My{E$E%hoS(02 zC3UIcgxU9*pRb&PF|uKhMuLMz9#E0FvA zp%LWWex=dK`gO=X;5Sw;pwzrG=u>G)YRe*w50B-ozSI`|>$LexrSR42>kVtvnt{H^ z&%14n=dp*gF9U1p+hz1pYNE~dJ@P?ueKBIag1&M-Q%r&_W^TuN^Mk|&%qwTFW&Och zja>WZ+xvq=mFG_a&8#m+$$LUgE%jcf zxN2J2qV|s&jjZc0(BFo>HoR}$#7vp(_uz~3zjH3NHZ3{Zz`1iDIR7f_b$^NXFZY}C zX~sK+%9tuBuH4C0uV?JdXDEI1B#ux9XAbb90`F-xAl}|PpIp**%oo!f9Mw@HO=z5;R?Eje=w91{o z0nFbsrL>vb?f=kP#bbHrNz1ll8&cyA8+(1OK~7K4DbAZ&Gv@`!|C`XCt1Q-YziRNV zC^?k6`>}TF{W|sotns~+x+P7&>pV+-dg@iIO$}?083spYrd^Kb@gDscd@fGW=C%F4 z-rLUMeFK|pnKLHGpyPJI)J*IbdH!#sT_tp5Z(FOLzM?pvDoPEHW38>ny1eUM>-#+R zF82)PHM58Q7p&(o{~G)pc+FaM?#g06_d?$MjT(LOVc^GuG%dKBvbd;1XQ{4@0NjGu1jG3&c)WeDSoS=@`>uN35}LafV|3?iYF(n{v#=5${ zP>;EMt=cnK{{`r`7V0mXu2(#A4Fz_Wv!8-E>}O}8e&%#Pid(L>z>f3v`dMD6FQ4wG zs!-4EJ^kVJjPc9OF0kWw7VkA|>-L-1ViY!+7L46U(;k+yEL_m;PK=8;zm%uV3QxWN zBie8B*gx&5Z}ikNp8Crl7U%p1>hAde>O*T!#fS5`W}eh=KF1k&@OGTfKSRTV%UbZcX9n)|_W7Sb1M%7Wa80R6 zII~O+yfAgXycc^R#&Wg4gwN)wE%y2!*Qgz#>`eRf$6~DQx6fM-;XHMO<<0H0(xYfcnOW(-fSgq%ClCz>^*~au zO*GYZHqFv;u@aZ9-xY{Q!_jRQvS|M7txbBUKb+LN0^vxgbNMW%T_V{Tjy4g+dO}hC zX1jKaGHKFlL#~mH1dY zYAvK|ru}j4{%~ant)Pde;VO0*QazvHKRC!XB zQ!VE@?G~tVg(|1?>cLR*O{(0k%9JWIsyw2~5mo+9m8Vom^He-l8me4z{S7y+Ty;}J z{{F1MY{AZOhiXYq#Wqs zguWvj!30$bXUg-O?TfQE=?m*&8z>aojtd0Acr1}n(e+3;x`P|x>WhZEdmCv=&^gtvY3Uuz~MVm-Yah=MSd&S9xx_gr7 z+I7*sNaW*0_L%CmE4Z6m5 z?1Vh3SiT+sg+!L*eCoE#R%-<7;lct;KF#)~fdf-cK6UxN*B3k-k`@YjV-6 z9#8#YQ9uh_w^(c8K-N5yB?UQKmeHFfc5G3}KhERF$klQy@?+$-YFF?-1NDpfU*xmr z`o-)WpS_8D%qRbds!#2&k>Ba#&-{Egf!G}GT6pS_8@*B6&JBLMy&qstzW$m# z2Yhi4z|U;-%*?m>I-##2)-qq(ooWa$D71R}DX&@p)X?oB6h)N;os=YJcS z?Y*nO^RSky=>~3wvwCsj#k6RE;@dvms9sy@m>P{mli_$rWLccl8G>y(__!L4<1yUR zpA2uO#O_2m)Wt70{Y3p}jH)^0vX{;doKB*#WM_908ub|K@AlrbZHXjQcf&6iP)~hj z!gE`=E1rnZPUD6;`g=a=#h|t}l^%PDb>Seu=ITTM*P*_os&lgH0qtmK*mHxHeBM`b zKG+j2OYH7WhPI(h_Ic%Y^OTGXBxzY+yd#thX}15dVji0FDB-hD(Z%!78@m;19TrsH zxE1OP@W__xb%N5I51MCs?eCu)V6Tn0Z+XxBv$$aINNhzO7r_z+N~5q zKnMr{As_^VfDjM@LO=)z0U;m+gn$qb0zyCt2mv7=1cZPP5CTF#2nYcoAOwVf5D)@F zKnMr{As_^VfDjM@LO=)z0U;m+gn$qb0zyCt2mv7=1cZPP5CTF#2nYcoAOwVf5D)@F zKnMr{As_^VfDjM@LO=)z0U;m+gn$qb0zyCt2mv7=1cZPP5CTF#2nYcoAOwVf5D)@F zKnMr{As_^VfDjM@LO=)z0U;m+gn$qb0zyCt2mv7=1cZPP5CTF#2nYcoAOwVf5D)@F zKnMr{As_^VfDjM@LO=)z0U;m+gn$qb0zyCt2mv7=1cZPP5CTF#2nYcoAOwVf5D)@F zKnMr{As_^VfDjM@LO=)z0U;m+gn$qb0zyCt2mv7=1cZPP5CTF#2nYcoAOwVf5D)@F zKnMr{As_^VfDjM@LO=)z0U_`QLf{no`LDQN!~JXAujBp=?%(2;O9%)7As_^VfDjM@ zLO=)z0U;m+gn$qb0zyCt2mv7=1cZPP5CTF#2nYcoAOwVf5D)@FKnMr{As_^VfDjM@ zLO=)z0U;m+gn$qb0zyCt2mv7=1cZPP5CTF#2nYcoAOwVf5cu5?I73uh9O;~|P64XT zSx$azA*ELZXdi4#>?a1lk)5z8JwAi-<5qC^z`9`i-~;8^gZBl}2NqKPxD{yF583`D zlpnKbI9HaMeehb!XSUJ^eC?&o>hv>hH1gOZ71_scq5LMIeTVASXZPPn=>t_8vWMz| z*#lKpHfPlguY?Y5j!-I){y3iPKx%f*lR@lV{(Lw-$2B=+kW0oY%jQb+I6^j+LMoX4k%Msi^O%8gDw zcV8g;Y8~dp^)%u((a2c6Hz!`fJjq$Xh7*`07_Z?I4^(7NVlIpZ&onK-xQta%el0oq zsXDGUodL#ISJiyJg>zw z+y4UfiNR;Gm|qPxrMB?cjk@lG6NArAGGV==bt66Hrl^h1F1_rn+J<*O#6)1?Jd$7|9TBIIMrVJ&lTBld+ztQAcOr8 z%22w^dsh1dc+{~Ua|I~%kA!}HTg(0W=Y00Oo=zw}!3MR@+!4sW4OZYk{NnrWgX4Hk z1`81<9I<^=jf8+SF2gV{S<*Je-8a{Vp^|tksR2i)}Ze5PtYllaXcl#AiGwatN_fXJTHHT8OPoq!e zWvMOmFg`q%w_yF)U!cu@S`J^Wf!?S^tr_S`{JhoHd=Bpz_T^yBf4PEQOpUiG-=pu9 z))yew&(K%SXOc;HT{gdCefGV?2Fxq(UBLQ-w>bIsJKG1sMAhv>_8tPWwleHHbE)<0 z66|lQdGDD!ybAl-TD)sR<<6=bk@L7UfB46UVRGbrxxY_C#x{qic{bli-Nm@F?DT<0 zujF>$)i&kjp3SqUw?p4NN}E|?3S+kCZ#TCcpSw9l_;`@_3% z9zedPzuZ^zOjTar5ASkD?((&_$QdaMrna24>Kcw?PGC-BtzR*G1Lk9oj{Evd#H}lh z`!wR7oeNoGfj_~QiK*rm+TT@M+K&79%cInL$4olfj`@^nqWsHOI-j|%iq!ne@p&Ua zE1qa_^335OI@2_#f;QJz&~-eA+5gkiXr-CIA(O@V{Q~v_wZ?JYpr3Y}p&%W7 z8EezR+GB>{RCU^L#CdSv)$q9>MVnU-1bS~hi}xXHauwc~9D|PAg`H~b7X|+KH>CDe zLwEc)?#ibwFU_ZhQX?m@*4ATP-tm_Ebsl@eJ%f48?4kbw>v_yS4?hQ9aaW$bqSVj5 zkgt8yUD-0HRL<>qUJOG2*l4uP*x}#3YTdsY`yOKN$9VGE;PFdCUxB`j z+tq2DxAHvJV^{~++lM*l@1c*UgY*d=vu=zn=bFR*THGJ*3y&S^c;BeQ+U9+V$2;&d zSIv`Mh_hxP&038yx_t(@V>o1+E*F5q~MRu35 zpQ1SIXJ@f~`cyxfTYhno9p~%!v#eNOIn_^1v7X!e`orrPev&%77%Igf3^shafK_SxwW z&U7?7JG~P2C3>e;>F?G0KboSSt@Ht{Uu!y z&F`Rs#@rdWy+3=^Q#HPZ7<1>+Ra5>LyZ^j$8!e}YCgZc;RjT$e+MvoE__sPG2p{aRQXj`}=(rM!M#}Yk}aI`ZT>98Yxeepi6*cR%rd-~$x zNFt#m(b&#VR}`{PU#L5hj2Jr%uO08SyCdE4zTL3Yb=@@LHHJJ9xw|_OONNrscntHY zzboklGJYdnopvWCq9-NN4(w>c-VyD>1l0;}%JZDb(P}0kn`*L@v%T7ch$}|~|+ufnqZq<3j>WkRC^k_;xcBrc_ z66)B^i#Cy*b zeuK?KWy1SZBkx0c2bqj#eD4ZHlbly9zKav$Dts=*{So^^;dn>n!}iBMW-qrt^dXyT zT^n_wRVA0zkt z7PzH+_LF(~x$u9m;SYFx&@gCQ=qH}dui$Ws@PXD%Qgy?{J-0Xd+a zHy_(4-vq0R@ppXoll{7YyypV))CJ^weR4BTmip$&AAdle8K0jv*z-Iw@($GbGWl0e z<62=}nYfNGE`QN;F+N9petKcgag98KI$tL4CVbg@zPN{cagBZT%=6oCnpGq-2@7hivm zo9)~6x-<6EkqXO?q2&ArxWM%fxLIy<1#%Pj2dJ^!%)@<3PxPXeJF4!wf3D>}({jVP zSIKekqR#n@>V6rXXSLi`iMy_?+7UggBM-ig;#U3D<9?fxVy!?wiwfgz{8Z}vQ>wk|uGjXnwEUQ=yKYL$7i+nr z^D+6)F@AKvY;C{5*!%i>5_-zK&C0>qleJ zB!8i{EKce>SGMWkulJ*IJcfJvli}@@*qsQ6y1F3HPt=dbsG7eNU@x5=_%=5hOLlhi zmwf6m*5B=YGuf6%LUlL%asjpeDiH44!d>x1#C$Qw0ory)BV8S|(|A)d{_c@F`g=a= z#iX`2l`eaXb>WLtZl}IcMW|3;Qq?(m^?-KdeOtIQ?0H3d{#z5Jf7gdSOFq99U;g&RI(ZfX zLO=-oJ`?x_l2flM{>#E0k-k`@>-RZb$xR3d0U;m+gn$qb0zyCt2mv7=1cZPP_;(O^ z18wl%a??EA%W;1k_kY9Pi~E1zegya5;{FHRatQ$;AOwVf5D)@FKnMr{As_^VfDjM@ z|1}95xiWy?fePSv0ExzNzm5AH-0$K(jr$DlJZ}EI$oFuc!##ofecV%id5UFn2>~G> z1cZPP5CTF#2nYcoAOwVf5D)^tcLeU)aNEa&fx=(z2k9H?WBnqPsp~pAo`zd4As_^VfDjM@LO=)z0U;m+gn$qb0zyCt2mv7=1cZPP5CTF# z2nYcoAOwVf5D)@FKnMr{As_^VfDjM@LO=)z0U;m+gn$qb0zyCt2mv7=1cZPP5CTF# z2nYcoAOwVf5D)@F;CDyhn?$w6(MEms=TOgCPJV15rC%r72ip?+iNSAVCoD>j&!GIc z6&yZp1scXI8b0_ydG_FafplhFFrBf=vbnOijApe5X~eVrwz5TBni%|M_5||7<6iuW2WifI=^`h4XdR_LG=EKY zXh|S@xE`^`DgErBFU@{tHI3lkNZ6NI?TkzeJ~!!yeepFhcvQt7vaEF03Z{n^mSsX&IFIcq&mIq?X6LGCBnRfN-00+U_XV=A)=~cW_2@$rjf~Y( zp2y~ug_O@(!G;q{ptAzQCmyKCp2Qf82G2Atuqi!OMftVlIx z@4O9pu95Pms+^HrJhF~T0FD;FHoNtd?t%NG}x5d z!eck;x(`kaK0C?8=tnJ2|Kxe}W{##V^=$s8$i}qKXx&he4r77wekQBiH#pT^`$vmx zSkHOA1R2aRc1`JB-m_XaJYP?M`55@Cb#q4``!@Q6{taIRyKy|H0%w})sdk40RynJ- zVIJgTD6tkA=BsD6uW1;^TB}7}J>Rh=J26<8=bS5F951L@jUyKOe;e@$GK|&U1?^2~ zaDA{HF|W}wo*N6=o6|pmJOepm4|9yoLE85MWMycFHPjG9xdxoqrkc_zYEDnQ?>_ij zj8U_%--3$HhX#=?6_&o06Kv6|QR z+~HN&FV+UAHdOAcx)C{#Tl0s1j2I?I&X@c9G-PaZc$#PPebilyE6YwFc=Sqc_g!sM zUhdgEi+Vfs&7-uL^_3`jPpDr^y-PJ$%c_{yeznugx`877t>|mhyY3Cll-Yg_zBvEe zXHjd*!ZS^rJNJR}ufbmT7kJ-szd4_^c&Bi@{kqLpYpC_gt4jO)nzlc@3-3APYx>K5 zHP2M#_5JWJXXGwldyAZrvS4bUU{n_eLJ@=~~?~1ZRsk=IJXM~?c_zc)SBYbJaY-`>t-y zLYlQ2V|4orYF()3o9hs zEqhqbvS?Ae+c7Tw{8GL)%YF5M4`{#1XaAJ1zR_3D`077?zclC9P&ecMqxan@`9ANn zf5}&`gn!fieEq%27k|iCUx7OI19eRdj(F$u?*!7>bpf2ut?Vfa=X0EahdzPx`6p>) zXvsQ!-k64ay*mH%XAeGmAFeOY63#3WgU?T%FYm=(h_PH1EaS6zYRi2@E6cI>uB&yL z@w|fYj=;R%fwkPqXJ5AQN|VnsS(foxr+>nIus;#$vwK3ZXxI)V?Zt_f#T_k|*tk4% zQu|xA$kSdaB09!!dU*T{Y#8vHQ;} zx6yKXXfi(gU8QOtqYbLufuCE|lkOCX7pnN#b@5q7m2c>>vf8W9(dBYovXrlly56qK z0iWKpBD!D8AJXL!UA~~pQ@T8(%gR~Y(sgZJHtTYuF57i^zb=actI;^9alp(N4R&QMnrvQS^BJCckTI}EQK z@3gxk-SNKNu+(+kG~zXeJQ2COI}%HVlF@h!^Qpfp=>;-=BVC<#CnlmNCDIP;Xu{qR z?ZO1r3UA8uobB&p<5=8I^oO_G9g&^UFa%y{$Kpvlv=dgLZJtk0-pC=?=eGU?XIp5m z6!AX0Bia`UC%wL~9=4%kp>kX#2=~Pk2_4<;ipF+uBV7HlXm?Lng!>G#osm$|%a{9d zcc{xwL?X&G8IRlDq1bNKdBp0A*u3;;NlZ$lsB+<2{ zvHq^E4-=^|wHF(Fw0N?mm*@F@S`^S?*DcmtI3T4iV@W~ImSyy2i5**1@{jZQF>*DH zi~Jb5t=bj*&p`cR{ulY|xqdNw$7lZ$RiD}~BhUEy+xP+gM}7XAP!EEGk$Zj%eBSWc z8xD>df&zz|VRx~9jr#02nz&l-#VN!+f)C-v<7na@yMX+S3&^Xj^ZQxulW&6A#q!_e zvp4)lefHtH7&PQ-F4OQ z;Q0wHH{-rf=}CRL%zjM%&07Afwm0+N*7EP8tcLv}RWFR=i>R|7vyM#uKiBf*x?i?x zM|4!n4Towie@x5GxZkGb$F(%klkz$3fLyH>Kt8YWXG~&$qPvh+fww&+lrv z*Sx^(ueIFEuOL@1DxCfn4N!dBryKd$=^YEBv1l?J?}#jmlX_>fO$YITGLFY^Pk%DJ zof5kf;ZRo>1p0~kai&Do94gsMX9rFL(O9yxI|+??jP-YWZ??8X5~{o5mkah2?@Yqh z^Vk;diYFqp(=?}!{+^F|zPUxL3olv(Zc}euxC-?pRh^Sl53r}5VXry0wefjc$@$<+ zv@Ef^I~m%BGTG;q+s#ulGLWQYeesS^GUTEB5B{8o_B_S$nW5<7d8a16elHhPW*!;I z0z9y#8g7*4J)XMEYwwr^q8oVoj;^36Zd;HNjn4AgZ`6sgo9l3!_V^xCUDNSXO+j5Z zMdci?{d-2BOHW!5@-}q7_SRg_5=Whq%iq34DHS0g1cZPP_;(ZF^MiiZQpb)+Uo6t~ z@Af4Pg@6zc0zyCt2mv7=1cZPP5CTF#2nd1SPXaH@3*bB90RGMa-}U0l*6Fw}!94@_ zrMUTP<(asva988zuZd^luEBjdZn=bj5D)@FKnMr{As_^VfDjM@LO=)zf!}`u_iVWB zKy(G}I^1(`&&7QuZn=bj5D)@FKnMr{ zAs_^VfDjM@LO=)z0U;m+gn$qb0zyCt2mv7=1cZPP5CTF#2nYcoAOwVf5D)@FKnMr{ zAs_^VfDjM@LO=)z0U;m+gn$qb0zyCt2mv7=1cZPP5CTF#2nd1S9f2HCZE>`hef3kQ z=PV~bwvf`Z0<;gdCH518-^fl_lpdc!`Ee^ad|+KLeei+u?7{m2=>rQXf7}W*?1yas z63UNRG@L6-%|3W7r1lR@lV{(Lw-$2B=+kW0oY%jW!f(5jL=mR%%| znxZ@w;dy+9lb;y;-q5Jk_R_@Qw~BZKhEveD2b~crqgm}i8u4tut!z2tiNSAXPar=$ z?j>hhnprMgV?Va1JbN7DldGbU9GJgyqm$3w z7s$R^hdFUQ`p`rpWA)ygcm?w$X9XKhV2)tChEF_Dkv)mIFd975w7{nHSQX{hl9NA< zKG@coCi^Dj@y4Hh3BL0-_3a`ikQc|Z1p@tyJ zHQ>CKTGFXhOL_vi{1#(WtlzKnH!A%*(7z77mznqTRjaIl*Q86TUTJ;xg2@UX-EZE zzmzDa-jM6U&dDRv5pn`)V{-!ip^dwm$HDBKt!uNV=kb2>+)z8!zjPA)JYiK1{~Y~% zOU*ZLzs>I`%RW5E8R6XCMlPJ+yPRLx%D#jA-b+l&qGNkc_T8BND{`{uFrJyZ%IqA( zo>nnE{n%>Crz~gH0~M66-$%VR#G@YNdVjat;2fxg7gTW!tfu!plR2W#rf74%|i zyiNHYeXq2>0I`0CzH&a3Ov3B3`5o)C?1UU_>h>kr=I)9pPKUVYFo;$n>`^DM-)rQKQRW~B%aclnYj}gP<$oX=ApN5QW4o~xJzK^<# zab?-*1CL(G?Y^sR%F8{QXHjp5zIl{3v%V4~?+NvbsduU7YFQQY+OKw+SvOFmzZHFL zde^;ynKIk2!58O$`z&g0S$L+2bLT#A{x#U^{sQk`?l{`f-e(O%`LRQtG2Wq_wkoUsrQbVbhI7wDbqyxm#=g_b6XXu z`IqDOd4N_t(d6Wr!$WkYX-)-guCJi$cn-7wr>D_MGk-&vzo*M-Gq>CSzPpmg^7d1% zvSS<4;|?3OK5s-$Pt7dNn^`mGImrJnp+8$)s^@;y<6Tj9D0SDP?bQ1P><3umd&#~z zO+W28LqR(FGS;SrwZ{y@?bFkS<2k%XuZGVBDcZbxAkcg3S-fvxldJH?v2?)Y!ql}}w>nokX-MowU@t;f2&<1P2=Jobir2J@QPL;nNT^O%1geh$3i zt~`51sh@ixU;C!JvSm)GoZIod7=-?_zefD3Qhk}$C!puC`!)8%{j0I>A@+WZC$9}2 zzclm}=-W6=Pvbn3=dm8cI>6pO%sGD#eLNkcPw<#^V{AFs9QN1Z{%~J->{!S9Mjh5R z?^8V9fuFf*p5T2~w`L*DT8%NfeFn8I)N}GIY<6Q^-CwN7T)sx{8La;k=(iT@FP*B_ zJn~INc9*fAqB!hlXR&_zR6m+qesPf<=j->gtXN+;)lW^ap4!Uz_E=`oIUY-{iA@%2(g$t7m-mAHQFk z^J}P^@&D2L?v#9=_u0SXt5?FmX@9={-sFovFl}y&gWM4 zl!fy-&cH*T!1?@>G%~bg9X|I=!@XXe|M@czpS=&)muCrQmWjdVC(oDnVlTv4t_qg% z**vx7KBAT7*n8L2I?Z@qL3l@C-tWL#ZsoHt+jynP=NX*Ga2~**&3HG;D{G_Tog#;*OR}Y+PP*Qu|xA$kSdaB09*D-f4T{Y#8vHQ;}x6yKXXfi(gU8QOtqYbLufuCE|lkOCX z?bZD3x+z`m)#Z>bpVZ|sU7D7sOnsKuevU4U-jqwVyjho|3uOF8x;pJnOhivgq#fANguNr$g$b$^-jwG#+uz5=vACV+4{x_SB0HmD2)xpc z#gleuC#*u-JfEJtkwdW0ZT$((w$NTF;(c~Uv@a4)dVOI%Y(vFD<+w-??u#c9I=bBz zjqTt@xcXz!?w+m)_Zeh6BcY_1FZbo{P?w#EM3iYV9=E$gvE8ckh}9RddFj!VeC$wH zUnJDAn-^^&ImvZ0U+xtr7wPUvqH9ZI{asxjCQ@T+FE;pSv2OYCHMic@Vz(}9)$5-3 zi$)B(9;eCm?DgjSKP?Jqh3ghuSU6bOGg(rQvt=2*Sz^Z)mHgv8evDi#pCUg-ZmV_$ z|1(g(nEypSd#+#1-tpO+xW|0*kEr_8{u;UC4jUK>qZEq{a6xQu_t;#wh3BX7TeJaqwi<^u9jpL`RdU5s1KXK(no z`}qIy2lzpdl76+J#{7-kEF{z49-qC5JK&4U8}7yOubx&Kw^zqC@(inSnYc%M_9pH> z`{Ek=p3As(VZ4m}{Z*y%&!8GWkB$5VR^u}Eo794w>c_kvJ?nzKqu)P9{y98cjKioe z&%GaDPrm+|JllP7|0n#+M$gQAo39gkT+0p5C-K3DuixNi`(t_?4Jawm-)gz7>aMGX zAJ1Rcax)+J3vs@_iJSd6Tmk<-S0&MNC|Pd$+pOh3)bf;`&$gET1Z6es7pZ!I|39P7 ze%iIY>DNguANBe9hL#&{HtKv0;buQ(p4_ISMELxnfXo zeECdhd&ei|_H6H61)lr0TunD{E7KP0#fZ1lq5+C;`*fpvZK-2uG!{*U;~kM@aZ+al zw&@@~z{c?y?&(j4w^L$wA{^@Kf3%LUXWuS_^?3wOm65!z`yP)C2yM?Lq{)}}J!#<4EEx*}Uk|6`_-(Su{Qyv|Bb}71e z{&{0&tMyk2c(+hrEYkIFX(l#8KnMr{As_^VfDjM@LO=)z0U;m+gus6x0x!%9;Je5G zz8fdH3f~1>jk_MVje7y^Yj7{by$JVW+#kZd1UG)Kks5HzB?N?k5D)@FKnMr{As_^V zfDjM@LO=-o{u8)o!)+fA@-NfrD@fnKw*mOPuj;PrX!%C{dq`Y$s2Mg6ephRnvQ@t# zs-Cp*k8{JV#Vp9PI^P{fDUMEGIvHK(6xqmjoRsmMNl3+0!=_o2G= z+5PuX`aso&?4i0~_CS@D%~>_WHD25!MAOrY@H{@l$xjS^Z)ntNdud|uTSf5$!}mkq z9&|>ijApe5X~eVrwz5TBni%|M_5||ta$Ag+0r=j4JD6pk!;R&5uE_4+Q>kJdd|-N(<30AsB`Uwi+bGXDAp}@i>3>DPyc~X5 zl7qW0_~NVJJhrDidpwYuovWge9GJgyqm$3w7s$R^NBQH|(}>$dBV+ZH=dpQZA?0&c zu;IiK=&Zo-i3cjOCou-2!81(@Y)X$+QGP8s`Qzw=ZJlYdZ$ch#{Mnb_J8wgtYoz?C zDrY2D&+`IyOPu_PYn}YdjZPl_bi_a9Fc)$dBjlFmF|ur$bpkouL)81l0JUG9 z&rUL7y%Y3wBR(BiV!XWeqn2lL#Am~LGlx@`dNz+2*_iejt;-bYjDOv}!KwDzKUieL z?K$uN4H@kBQmQ!}_ny_dI)*;7A9L~iK7lbB!+5E6c1IxlHu?`9!?VzC9M7r1nWlQG z-Qj?3&Z=#wf_w}m)=opUdS?5YhHhw)?7mnyw!K_XMf;j8)$7I#KI!>k8~smm|+P4XNPjmlEaF8**LPIoRgt z2sxBCHYd;@+PJHE9L(O?x;A@y9`8TT4YgDKODECK6ISK$&(Y7f)co}J!~Bl2?89@M z5zg&x^sQsy~MOE272$wzLxY~k&``#@yyg!X6GRGw2I2~W2-5jvYb^9 zR8YEpANAT8+j`sjNve!i9JekVZK&K?bt7^fx8@K37%@zaoG4AZSC*YV@aUD??z`Hi zyxg;S7WH=Mn@4Fg>nl<69#X%UdY5XhmQ^vY{c5L~bpu8EThZ61cikJ9DYN|=d~yD_ z&!X0rg=d;LckTn{UxU5zFYtckesex+@ebm6`*oYI)==w}SC#hpHEn-*7v77=*YubB zYM!ae>-*td&d6Q9_7*uKW!UddT6GP_F()vmvDU8`z5(;GN5_4ACgRqW#(f%b&(4Lc zvB00;%fwW33+?Z!Ep5kr{N+*Vy<;XFZO44dG*SNLE1l2WRz+(5<@o&{pcPLvIeF&r z5S?k7Q$d^SE9g3&!|ea*X|&SJ-w@{S>2li4?e@R#uH>=2{gkWh*oO4D!$z&o8YnPAoQR8HR4y5>dUk~0X>i1udyHQUyXeavG-#no@FsVUZTdtZNeJ!AaxGm7l^oyB_%+nRp!T8zOavkqf7 z+Omh`EQ=PkyB*`=&oAX`v)or7_<;7CeD+WI>KlD^oRLa${PFvx_19206@+&L=KT(=howVSm`i@i8v%#{DeuIRQzV}Exj8e10jniE>Y z`;#q&_Ne(CRM40^1Go2QuX?J+$Imf$E?qU{kFopDE4R^ddT25}`(34KAEOPb+<~84 z)syZNilx>3?7EIFH|cV_E(dhESC^*ckgkvF@~1w%u{)*ZXLMOT%Zs;Emm76?r!Kea zGNsFhbUCEUQC*(WWV@Z>I-#8k`ZHv;kDzPc6X#Z-nSc;x~`i>yvC3xB6oL3V#!c4 z8joQ<^>-z`K*n#RtJChpMD(OY+JPNS*gK+Kn4ntWO?jTP{e5g4i`$9*@OHZ+vNIZn zz$@)oJZXn^!YZ`Q^XbVOIRyLM)}P>P3+Y z#}0M%MM53BdC?}4lUyhBy{fMU0rc|SG=#Q zW5EKf>Ratlw;6DlPpld?keD)tv^{M?ba+=8og?t)OyO{q`HZQVo zLM;dmqb!Ep^IPEahL)=WKMV&)4M8F9&Cp#muFrlW>{VPX_u>@%AHj$H;yg{yw-DvVG(A#~0gA6N+$?wWdKpksd>^UhX1=LmgX}9>ZlmP>8u{O%#(vB^*{AeG-_UZ? z-)1d;#@Oq5Xlpr-LpA&?QuPAQ|ARXFH|x>#>!(`2$>-;PX}RI%pw8Dp+$vAa^EM?V z%4xZ`U%2jiE&r94JF4!wDOD1^jgtNJ=(r{yZqIVF&lvf4wA^c6;KpC-vAx-U=6DA5 zWi40Ju!ysIvEsD4Xn^9|KHbR2WbYUojYX5;ct>PeoYWbEZ94ck8;#>J+|!>7Z>PlW zL^#yd1%ZB|ew=|(HHTpK(%FI2OEi}3>`p?X9%KF8-kZ2Bk%a1Q_~im>^?vpIwS~Ll zi3sg9jj5x*=cArywvTn;Sb(r?>U;o4p}wT5b86}V_OvtXHK(>VKK&~>AM@42-H~KZ zv;+SbbYgdRGPDh4vd=5Go2O)CAW6&m;vJ!6NTd7@74y)Vmj*r?6IIf`ZB$ zuR?tR9@$d8Mo{MTyvtqYwI47GNH_8JF+m|S%A_Rnh5coF`n2(J_y<3Psf>K{B z()Dj>B_={Z2nYcoAOwVf5D)@FKnMr{As_^Vz<(tI+y5wl?>+-mgQP!-?*gvJy#n`2 z+&AEE!rhE}748~Hwn4Mzjw#Kcj%}RzawyM!N#Fx*f{tds0wB4{fc8jU-PW$W7>YMzB#Azw}zi`-3O=o@x}?c zOkNX%-^`vse<9dQPI;QgYUv^;duSb{KQw<$c4$c;d$=Bb8K?BKhrTrXnbkCMBRFPO zI~e!p^q5V$Lw-k9enXa(&RW6r(899psq)n9EZ7Z!9WxtlT%R3o48ZpW+`%mS9BwSf zb47Olo=O$#-~-dM9PhD5E>ZbC-bQ)O$>e$EWDY+p$zfvf@4Wcm2^Qi{`?tqGP*sU| zf#E}S)3cZd4TtM5Q7pLsDmGQ5-@_Pr{r^EpY^LC@3%>X&=FQli^6c?IYId%QMsk=# zuiWV5bN2_|>y|R$}`30l)o|DiW z$Mae|v;8knpBQ{bakBNEGKzUOG5G8x7o#7wygld1^XSc*OI_;O{7sRKX`j)$p&}jg zX8j4UV{SZNJb!br{cE2c+w#Ts#&+~FFWv)1w#9m?_VjzE>Uoc7aJ2qDpPqBz^@ngr zX!O$bjo!1`cX++?{NgpM_C@T$Z^IXO4uAK3_rY;Irvhi1>Zx{zgR#k3wGH2ad<-S_ zl7=6uXST0tz}j#4ChF=L`)*+mRb5p-I9{-5zi0n%BR=-%Etnfe7TA}iF^&(mBPQ=5 zEaSPjz@C}@3FH~b5qp?p%(7_T3y_t;275+B5ak+->ssfsbZYdn^jDF~Z!sr|_4}26 zrqVwK{p-*dvLkxcDr?|9$R+DF=CQr<`G@(v^v>2Ga~VE8kMi%LpYLFAexi&v*L|_# zkzWO=^+~LqdxF$^#wu@k9q*&#))g3&%aP}thE#C%ONnyo4Y@AtoV*%`RBhVWoIrnQ z$3SB z>$C4AHegA@%RQTCQE!L7 zd6YJ@z7i#`$NI(8yHsioQ0z>)yajneEr$i}SyI7PYo4Jk!Lv zb00YW8tkoqf%5|QoAX(VcQVJ@uiJdJhFY(@s@)ncEx9BciG;Ttd?dvx5_XCiK0Y22p~_v~EA8VmgSEM;P< zxrO$3)t0v7KK}A3_1-a)j<#byWtu4e@|DhKZmS|S|I~RRKr5bTa`MdKAv)7Er-C-u zSI~7lhuQzr(`coczah-u)8(|8+wFhfUCCp4`zcr1u?^{QhmBgF*c_YCGWvxoi%tmiTRJp3GZ#a(&!ic&xKLcaD*cV)|*QaQKdc`*q6 zXMc_ORi*ketxrJDWA|(9hx=D!-$U&E7*AdsJbr2Dakf4(?#xf){F3Lf9>Y4o-agDZ ze-C{;9i&h2m~~@pIoBNa*W&(gUwG_T$NNSd);8}`Jl=tyxoV#5LYy@VY1V3t(d{#+ zb)lY+1euJ?8Q?de30}pFqF0SbynMz2=c`Dzdwb{S?JvKRb){)2I5;-13Wy z>^NV)pJm1R%Bg;8iuK&y*B@Tb7{C0CB0GL(@m|BWrr*34W3b7r!`O|s>|r^}qDAd) z$GG_OOZnO?_tghJp#3JFJrhbp-!d0ywc>OKM$*OL|`}o{VnIbb6*a7EAK_0M4i{HIpeKE{X%CvKHnX{IyYxY zobPb9b4Jt|k3X-MIlU>&wa?;A%%9!w#aRJoyuCK!E=bjPgJ_~}>Ub5@eXVJIundHas;VeCkjy{3UF@IM>`FGIA8TviT z-zn_>4!`qihO>A_@>x0)XsHeG7wGLouOz~q+{77Ub{rHBN}TViuXif z_D!mGvo>k57e|bl^557M-Ii$V?+!&{%feoBLW_8RvZc@-HNS%j8gpmh_WtZuPu2MP zYRsKWS55h2?Edr0ZM2*onvBnWSE<^^XoD(u;OAELq&tP8`Eq`C-A%e|*X4jN@7HBU zm!{>Dx_(TTr+j*2cSg%Aukhm7x@^|vox0qv%K=^P)#Z>bpVZ|sU7pe9^6Nf&{fd=0 zG&QdZZ3}lqI!)aESfVEqj&?>P9d@LzFW#pW+d>_7PhUJ7NhFjc8rvD_ib59Z3w1}5 z5o3qpwd0+3cceSsw;PtauA4@@#*il>cXvl($xt#Hk6}LbcO|_*#&4vn)9%DX^rS@E zfgMfQJEC2fpjzQgd7iWVeQX?y+ll`0cDp08Ga81#EA3c3X@_>gDzwe>>B$>81pD09 zpWti@?Uf?lXLm&VBH^Uh7uLfzR4i1Eiv;1mcp{;r+g;Jv4sL|2KNjuo>56clLAEmz zN_zQnU+xZd*@;L*nI_|LyE_!ytvZibeG!|N9!<%|4t4cKLLIw#(I%3UTqpD8UU71f z?w%yNwlvn?)%9T_HKz7rgO3*LmiAWL-qsZl@38Iv&iUF~Cq3!9ZLj8QQg0`AcXJ1g zeRw;jLKri_vv)WF;~oAMcP!n`<_RRh{!v%)itD?Rk;I4X7TZ~}II+arfR-t1qOJt; zX|WT@NRPb&D(q}q7l;2l{9F&mA4b^>hq{DM=ocsS9;RJLAq0eg5D)@FKnVO_PN34} z4!Z8ex@i$Ug{rI6&cYX~Z_H+eUvA;xaXxt&`Qx$YGJ11Le0(p7Yie90?@{_fTqDo; zY#%285zV9!L+4^7cL;r`Q*b0Hya~h*4KOvkLxIImYZ>Z(o2x8!@k4vcD-*5C~Kl` zX?wFjt7Qq< zT@&|&mYe;=R(hfzX?af73-@$nB{C$+w$dmQO<2X(8G2{M2 zr6;;n%X?IUu4_17uH|O`v9AI4u4_@wqQZN>Qrj=r{>}Sky_T;sa@F2-Q(Eq5xtY)Nm7eGpBiG|@ zIPm{JiTkz5$LBUJw|(>ERm5TYBPw2D9RHiP&uF>H|7+TQRLfNue#X~6_%eCX04?fJ zUxJ|$-}dQ7{yy9LzBw9;Cd2WL$g()8F9Tt~UsK0XhI{&x;qBDv6^Y%6aHy+`_1+Wp z;|pt4%{SKnKRfRN9!GWM`>yViB^d*fErVnmq7t@aM`0urY$S_^%P%k&Jn=(0li+WT z)M~Ztj-+m>yJgEL8Jc*`mxsxW8k}naIS0u$Mx~R7&)}0-kO^|a4CHmM69Sh^?m216 z7@P|vfG~C&i@N`{t4iHne(-%~^39i%S;be{wbxpEt+jvqQM-hIJ+-ysSB>FFqOCIl zje3mqblSf*Yl{F6K(?^MZz1v9S(Pofq#FD#FA-=#nTXltHuotJ>P=8hEZQ1K1gOUT+2Z^X zSdG=c?#Dbd$CM!TTAOmk`z}k#IfqXvu%`2`Dpc80Z3`&fbNgGf?DiFEUS`|4^Ualf z3=eFnj!i7o`3K_bc6+bAw_XctmbvYx)~nmzcEe89x z|CI~HA|XDx%OwPafDjM@LO=)z0U;m+gn$qb0zyCt{5cUQUf?7ChyV50I{Yr+PTc+PiP}`?4SMe)=AM8$h%`rEXb<(}i`{Wo-14J}l*?%#YmrYPLM2equ1 z)qh7zv_^NeSwF5sN+BQwgn$qb0zyCt2mv7=1cZPP5CTF#2nYcoAOwVf5D)@FKnMr{ zAs_^VfDjM@LO=)z0U;m+gn$qb0zyCt2mv7=1cZPP5CTF#2nYcoAOwVf5D)@FKnMr{ zAs_^VfDjM@LO=)z0U;m+gn$qb0zyCt2mv7=1cZPP5CTF#2nYcoAOwVf5D)@FKnMr{ zAs_^VfDjM@LO=)z0U;m+gn$qb0zyCt2mv7=1cZPP5CTF#2nYcoAOwVf5D)@FKnMr{ zAs_^VfDjM@LO=)z0U;m+gn$qb0zyCt2mv7=1cZPP5CTF#2nYcoAOwVf5D)@FKnMr{ zAs_^VfDjM@LO=)z0U;m+gn$qb0zyCt2mv7=1cZPP5CTF#2nYcoAOwVf5D)@FKnMr{ zAs_^VfDjM@LO=)z0U;m+gn$qb0zyCt2mv7=1cZPP`12>w=p*_Mxc>_Gr*VG<_dU3` z;Fe1W2mv7=1cZPP5CTF#2nYcoAOwVf5D)@FKnMr{As_^VfDjM@LO=)z0U;m+gn$qb z0zyCt2mv7=1cZPP5CTF#2nYcoAOwVf5D)@FKnMr{As_^VfDjM@LO=)z0U;m+gn$tE zw;?dwN3(MyRe0*xpq|!@%-DQN1$?vzwt4oGecv-DHA+p)pv;8kAKtgxpW6RWp}BvT zFSTzzWyUpM^&n(}izqXu(QvvTIcxvNDKoHzM&RoZ4HTxHX`+$GA1yMUxPvkqi1r*P zUtUav9JH z%(OQTy(lk_noHzSmXpTrX&#a)4PhuPf zRv06bea}w$VPAYr_8nI7hcqo^YW~#F`~vf2VRDuU=0jl4tcPpYn8UR`_+E?KZ?ezf z+Cn@RnS;BFRjmCFO*c8-?ComUA{b%Z` zbV`ktP-YbwnQ`<%*Ur@Gw;+$VKK~$mXLQKZwUjwoVvM9Kd7i>lu^#%s!eO zyYJ-Kx&D(r+y79G&TVh_ZQYsek`N>KNZVf4tt0tMT(!t9@{@&wQ7eYTB^*fwgx6&q?2zx=Nbe zZeXsZwb|8AKt6^Nds_97dS?5w>Iv*s1E{O#%`45xzU)5X=Fjo`&ODhwEcX8{;uB<; z>xZhSIE8t+w;3_7b7Z`>s%U!ZMaTyrN9`;1mt{U%Z4xOO%6?W>SyMRn4@;-z>Ybq6d9c18v) z4pDPS%C$L;{?NJ|^%G!rYRf9~^gP~=pB-wZ%9l={pU1W0;eSCt-$AbK`yta_U_Me| zjBsx6A{Wl@JoodQSdiQ_597mQ zc?Z^-{s3+KTp@fl`ntm!H4Z?Z=jWZK`g3>(vM&RB(l?6e7s-hx<$Lt~{CX8){ThAc ze5RNLSDTskHRk*AwU}4-yNdPu?=UjWpKk6A5|uU&>AMNcnhLNl&Y{M$i?IK$;JtCq z@N(?8t8f+w6dKEKM$Qx3HN!6;hMObj%l&;CGPXH5&9?ao>K4Y8WvBN&HkaG|lVej@ zXxkh`y&3xYQQFA*VwAiuRW79N#ST|ZE1K8*5u=`Uy*c_j(bu~7tecoAv;8)FasKOP zQe(sXGj*Ih_kr^-!@J@uIKObeIiFQ{Cv&_(r_Hy@sB!MK`F(!fu|K#2=N07Z_LuwW zK2wR^_k%l(k$XJtH8MsDuwS0g%B#mQCore6)~_DE3G=bbiTmc2h+Cc?_i4mEI|s7b zEPsM8H>R3fXn)V_{C3>O-yWv!%~#UlX3VF7I?B8<*SPoY5>oRo&F3K>Eq$`i$S{Y8 z=}cWk5pAq2q8oS)v;U{3(K2`b1~7k57t%&hv~fkRulvrkcwfUNU1X2RG3dBm&?v`#k>&q;v@3;f{P)(fr>@G+ zr;L&#$FbJdU|nv0$NDahy~}+7^P1U1|5MiUn1305_PuH?J9~A$pZg(S^|rOFp(0<- z?RZ|`y;%N*HxR!hUti$pOF>_RzD?-mDVzs0Jl11a z2iV((Ip-gvkEi`~7mryd#+GwUV}H%<5BG(~j&;0mlw)o4KE>nh`?aO!$qvL>IiF^( zz!|DVX|u#r@BNVW8$9+;dFtyt^#M=)g%9#`ejRmp{D1y|bzZ*DdF)^E z)QjQYZGWNu-r|YRXSQ5sOHs#upsvZj5&Qe*N4}J~+K2BO%{-~$`v%_-LwDi(=94rs zv}iTH8>iu3qrRW`8Y3;5e6xoH>CvO?^=t7jYa zcwS0)M_}H!V=XuGw*%YQrJYaJH8g8A&N1V#4Z_E`+NZSLD`!)E6V5UGASaUc(rS7|$-X6pFjeylvXn(kc zQgiS&r}^%7!m4#=yYMk`VguelWxMbD?9wAO@Vn!)+mG~qHgRMHp5wD9 z#b?HG>^Eb0xAEK$pv`@o=Cd+>i5Ub<(Iv zT?;uu_5wpe`_|r`cqpcK z1tQ^~9!Tg5;|&X28!ppv!I&?l-w}vK!jY{PvRL)0I~w#*Z#bd11;QPn)|$)gcJV}O zIMP5A?FvQoTU6~f$D~1D7;?>&|JsglOT4zHGZ2o{1nuU87SWzWL$*C?UI#@qX3fBD z|I@FYO7Ux?F>4N8d)_~;-5;-Pp(XV2RDAZkT-9!+wW@5#&n@c7=@g2$@Jk*d%PLl1 zm6lcEluMk_aLP|RrQ5R0sej2S2RwS$?vNuNb;_5W@-3$<#p2=kx>GK3%5_fJ?37(j z`6Z`3)v0@>emmZy$j~?iVg#xWR zdC|rbQ(UL=5^1r}i&-F{$8y1PZmx{YVq0aWV z4)%WVapkt(EVrb`ekxCo{I(B^ix+9`IIeT!I`X$zjm!0KV2E?=U3v3G2~?Cb;hu2a`i>~|dZn6vNo zI`XI^ch`mM|5@nSURQRO<;q`i<<2~G<##xKNu*&_lt#(7Bv3KWBy?UPz{evTS*U{aM{anY7 zyZ^ZU7rAmLt~>5GI`Sk+o>yxeKesq?_kEpoCx)%|#7FKw;(>Ig>?ZJh~d)MKQl(>_5r@i{cIRUJnoTWz&EKie*wf*sL# zh;Te6KEmRdOWR#v9I}ZI$kf`?^$9zolJR#9H8rWMl|Y@oIYgubUjW=peF`9UAeK;d z?u~kYE5bKdJd~iVp#&etn^3WJj)1lY?GzA$e-WVC;d|(&CbVjTO69MloU?l1UkIoN z_~1?;a-t_re2OQkiSO)81X@rgVs^RBeM*FS6I2t6wgwUbs%de)8F>W%SFd?!$g@nn zwx*o$x@IZZ#basrvb_ZpZ(ET}){_Rjj!S zx1W2RZCNwb(thx^_o$^R1cZPP5CTF#2nYcoAOwWKpCJMMJ;SS^GzX6a-2nYcoAOwVf5D)@FKnMr{As_^VfDri8Cve}|yKndVvLEt( z`W}88KvVx+h2eDJ$hBD;gX&>p;BU33DO>fQkfqc9 zi5W&_vhT-3qgvBTlYKwP;o%#882VMdk_2g;D>Rx+EebBWtb^0yH z2;UhU@^md_PL>!W=}Mlbuv=tgj(^<9yi#jq@V~P7RT^_MjWI%QDIOzDrE{clj8?0d$< z{HoR|xrxVa)Ux(Y_8pyK!g|Bejd*lmiSc^IeP`+_X?DATu}f>St6zY8 z3?=r6>R+j6wlAxmz}o*A>gsv(N^`O=yYAe4IG*3(K7m;5|6Rl<$S`*fRbg&mocA^( z=5>yY=W3PiOT7sB0OW{0%rQ3lY0vi|D?mG}>1sd9l^B;*2Bngtl=?Mt`2+f!s~=Q) zpVI#V`Zqm1UegMyd5^G2d!2b~D}L?~&hyk3KXVyAJ&!W)p`WL)_dZ!b8_U01^yu&W z)OZAI>pnkqpV11d-y~`r*RIAKxC(hzR44r_UWyk|cfhh>XJo+Q5G9LJuFY}uht}<= zp8&H{TUME;=kfmb>`*gRzH|cpJgyZF{|oy04svz(+e~|b`ACH^!nwVRTsXh?IKQA~ zo@UGR$)r&w=t{vjVbdca)?aUqP9qW-NcGh*Fh%s9VR_R_fY| z$pTtBu3d%PXAh4e@8+wGde*N*?g78Cyn>RmPNPqS1<6hGFg`q%cVNBf575TX6~b4e zuRE+!;{fz|e%@)SKZkb;`!cY`zfnZLNKP~<-=pv6*Q*fg*XS$fGsPsh+RU`CG2f4` z#k{ije%9~5!^kv$y16$zTNqcCo!sJrog!$Oz#C`Kh#4XQ{`!wR7oda2ImOsIl8&l0Kw7+L| zemm~tZx2)V<}2xNGv?Dk9c5mbYutNx390#)=JSP*mOfc$WSGOlbf&JNh&EOh(G5I@ z+5gkiXqh{I1DL<33uz;_8~ngp#$&nuq^0cGhMaMSjar{KBd4dX%+H%yGv^BA|262( zmgeiZUzK=w6&y(3^H?)=KY;x}t?}l96usPfhWvE+6|7ASYmXTQ2TD>dN1Xrmd;~tL zlC*I}udn;gvv~i(CS7EY$ua1-U9heZ zrzTfle7>KuTs^n<^oQ3o#xFA?$By4wyw|X;+izZr5!ei@#@LNE>}EO3!a42MV_dxX zB|UAHcPu0_exR<&z7hL-;YYrdx!Q;C1dOR;BydyC0+p(4# z`7F*hcIoExk5k`I-ILZ{-p7n0YW#~rYUHz-kCMMcn^_U)k~kaq=*Tbm?DbjHNu#BM zPxAg=R^7&WA1xcK#TtEa?+A^X_IY~U)>k?D+r0YGQq|t)=x_4s$<~iyZ({#7(5u`=OH!L5`#aQ^ zqYr#eI|^3ZhlQy3#6vN?D-a0>^*};j7;jkE+HjeU3yt|w`W=B-Bplg#A&XU?x}!l4 z^@bCATOiyKYOT4Y+0qV>{55idZ%3Hl+8}bR($n3^?s)u^5|_Vq9cy{d8bS}niowDC4k2vL$8$NO4(q%W*)h`dU1Y1LGZrq+oyekw8w}nHk zdMFl)#vDaUpjGdRMT4PuTuH)_?SYOkWPw+RtVOi)K*PkElRJux

4P65*}9-qqHCVu`;jeL&KvU0T=4!(y?ABfnnN&+o4*Kjz_33;RpO-Jpc*WQ&kUqqg~hIL)BJ@O5(x)guIV?Wiei^#h!B2Rkc?tFO5GavryL-OhO_-TSYkBcjBMV&7< zPnyQH?0j(J7ORDnmGff16o&zipKjQ5TvtATI$v&F1BcHG#qIFKb?xcO3)@#X=K$97 zbuVroBD?bEm6XWsSK4Fm%H8vXV?Jentc)dD4qrs=QXDpT^6dW*`+kqTo6lRGxL=2# zS?C#p&UgAhE!O7%-J`3l{FFbX1T8FmgUC%xg&Sy+a9GSdclz!s%}|->&SoO$X%ZGj{LYI zch|M<$lpL&int3@Jv&|#sB=D}PW~=8{659bQzd9wx@t%CBS+rs$W;N^|K-Tt@zPbl ziN5Q|-T8Sp>Uow@P zzxI{o|DPSXT3(nxZokyQ1}BIGy%cTvY^^$0sLzOSB%I(AKuwg?8INsR`Ew#1#bdCm zClTC6@tyHtprZo<=ZSjI7*+QpguS%2;`A7fB-%O?(5S~qPp5qnYKkYI+6up1Ks|M& zfy1U?M>HOCk0uonwnI?>@(7VqZ_wVXE+24#1g8` z>8l5{qwPW4A6oJknv(OmNj>mUh^QvMvojHBL79lz;z5$a7)O)T0PNCX`0|B7NB z+VbeB?QzNv&u4o~?Q_zU0&_n9szQ}5)oU1~d#*ZEVz)0=`*gOAy}xGbxoxvjd=K$N z#C6Eo7kK_G$L+Rv_l4<@sVRvv_u6)i-TtF?F{ScUiMQtqwf{Sw@VIb<&bz$r^OXM^ zSN<9uv)2Ocqy)txp^pEJdn2g}0U;m+gn$qb0zyCt2mv7=1cZPP5CRt>FofTC@IU;o zo8$OhKmvCU?(MjD;O@n}6ZgHi`*42__vdl{4elgvxrBfa5CTF#2nYcoAOwVf5D)@F zKnMtdKYarCt-bqpzc2eC@2Bsnzw20_QnxI_>B7l*SsR1uVPjAc{(6L^_Md8G_1V8; zv#gWq@5^AMyPk6G9OqI90U;m+gn$qb0zyCt2mv7=1cZPP5CTF#2nYcoAOwVf5D)@F zKnMr{As_^VfDjM@LO=)z0U;m+gn$qb0zyCt2mv7=1cZPP5CTF#2nYcoAOwVf5D)@F zKnMr{As_^VfDjM@LO=)z0U;m+gn$qb0zyCt2mv7=1cZPP5CTF#2nYcoAOwVf5D)@F zKnMr{As_^VfDjM@LO=)z0U;m+gn$qb0zyCt2mv7=1cZPP5CTF#2nYcoAOwVf5D)@F zKnMr{As_^VfDjM@LO=)z0U;m+gn$qb0zyCt2mv7=1cZPP5CTF#2nYcoAOwVf5D)@F zKnMr{As_^VfDjM@LO=)z0U;m+gn$qb0zyCt2mv7=1cZPP5CTF#2nYcoAOwVf5D)@F zKnMr{As_^VfDjM@LO=)z0U;m+gn$qb0zyCt2mv7=1cbn!J%RgtMEB$V0`6V7{}%Uu z!Y!8&5CTF#2nYcoAOwVf5D)@FKnMr{As_^VfDjM@LO=)z0U;m+gn$qb0zyCt2mv7= z1cZPP5CTF#2nYcoAOwVf5D)@FKnMr{As_^VfDjM@LO=)z0U;m+gn$qb0zyCt2mvAR zZ$}_aG&?uiE1vpE)YF=g8Jkb3nLgSB+dTWpzVDfn8l@&?P-a5&5AR#;PwjuG(A>Yv zm)bXXUuXn3!MHIwd#p*o|7&-pRhBQ%qd_sAlVrTtM&6(d6Z}&3AHa z-1Y;GZYW2GvA}pgV><1tjZ(Y)V>vdg=e%Bm4CYvGx1@I1&uSg>d_4~4W8kmW@n)a- zF8YK14QD>E_D~&8T(8ltw7CrhqKQ$h~T)xjw-DkAI>NklR$F-}m z?_7mEE2@+J6)(jLsXJgL3O`LJUi4(l`ow@ zKaXq0!~cSQzN6-wz29cq3(Q9~y~p_lHS-kmdp|zSM92D1#BNJ{oec9F z#&e*&*sMV8X+@W(9$!J3q-HFCsEAUPd#GE-*jDP=i^&37I<8%X+-DDuBJbv_je6Fv zMD78*JvNuy{gY!;SZLcEMZFpN`cc}*`eKy4CsZz^?!^vQO)Hw${1Ky` zb-g+IJJHv=_pFhRM_8J)@1^(ow6IyxoIOYWAG}ijn z!#80*b~$n1yb^KC^W#2^xM$};R-5Hd@a4u-a|`Y7nVsK``}o_#)V=viI^2x;G*CyG zSLPb`-d#d!{-ybS?xUqo))^V*@GzaJt0?d%YFK;BFxY%~%H?@hh8 z9k&bWmt()k^8Y>Bl|ncEdu!QKSLNqZM#+)mSZiyrE;qkpeV51H*`Cndd%hPoIQi}pMic$ zuKx1#^$w3rU5?!q>?bD<``MnWpMJg{hg)W0jveRg^;46pFFxN-S+1Vjd-}uc8RM6k zkz>d2EZ%F_*6la1#RzN$R%7f&8+NmtW#OE5>oG3g{F0tFOFZ@74{5)_WB-(=zRpu0 z@YG-UAV25VQFq7x=O0+-<@=n+{v}Vn82;V%7wYdVp7=wa`cl-fAE;}xZ^S;I|Hzjz zSNm{2*UXa|&gVD-58Z|H`6p>)XwhnX_e{gRMxFop8;H-|2P+Fr!kJ~V@42b-CH~)S zOEH$q{RMnBPj1>pw5$+&@9Nn`J)V~m-VvDh?O4l=eD-Al)PMhd%Shd z+KY44v6VzAjgr5d=Da_G@P8kzdE3Bv>%GAv7~`d|JH%~Q(G{tqSo4&;BK0KB+I{fB z?GfvjT<3jS@70eWmsBNu;C0NhixEN zs2t}Ag0W~k?nKu+!jX1vgsUeK?(FIaai2l9Efh%D`Ep;j20HY3D5Oji(Wu@Ti0o9I zN32*#=cPyI<)a5WVxd6mPF}R}#1z-5e7RSgT&S}vfvzo%^mKH5j7W{CzR<--3!Rqh z1MR#s#F3TWf*$OM#@ge0EWCAFqNWC`yd}If5Q_zNZdus+*Ki$Kl+c46A*9z@rFIJ5 zUuqMfcmj=i@#opMhFiJ8wrEGI9!|t{Or4&P-R?7?`1ShMXtcFP*Z(rQLvM|$4NBh; zhV716ID!7Rwdsj%Aw3jnRjDYRPAIV2i8+Xv&VHqo7x|%)&(T;`u8D?&Yx+u-c;y)(~@9{CK^FXexM$DZq#vNt^TZro!Y`Sq%PJ_lD$ zC2Ww*zZSJi#oeG#XYK1??*|`OZu`x0OM2|5^7P1W`>?opk>=*P&W-EH-(oc`*S~=w z&b4>t%@>g;FCrhfh+HO8?boRVmX+TND<697 z%AeE7EU+>~(`7vi5^(raRw;j2z>XxO3AJ0!Za(6!LQF>DU5`+C1s%}|- z>&TCyWVzeldPk0LTAE3i%sLC0o52tCOWbUqbmqCvLR`PWT?+XERm*Z7N1g4P9ecOG z_4~BX@bGQRnM%-0a7lKX)rB(S##UI(*hRa{RtR*&kDN%St-(BG|E?v?m|U zksBU4K17tg-8{?rzc_L=ZNbOouU_w76xY0|yrO0Kl0vsUr-02Vx0T=WNvj z9MJZl?GG;a14YUCON6Ko}sRy-`gg7VG@O zvQC!S?cH_6Qog+Hb7R{U_D{F{sM9e{V`)x%{JvaW)A4gEwj<+WXs#XqayJb}=c!`N zU%36@dA4PZ)6Olt?OnN)Lg0@{Acl$N>}7bMSR~Z($8;~ggn$qb0zyCt2mv7=1cZPP z5CTF#2nc~c0Ro@C#)sd_`S80~qA%fh0bj=b72LaVKY$y5A5DLU`|ol0<4)mz5cfm4 zzlK{bAs_^VfDjM@LO=)z0U;m+gn$qb0z%+VpTK=<@4nsd%YMlF>3iz$blkr;Gn_8G z?EIYzR}HF%je)-fnx<^+Ki$dt$o@T`Wp(i{7I3+x`}ab&RDKEpAs_^VfDjM@LO=)z z0U;m+gn$qb0zyCt2mv7=1cZPP5CTF#2nYcoAOwVf5D)@FKnMr{As_^VfDjM@LO=)z z0U;m+gn$qb0zyCt2mv7=1cZPP5CTF#2nYcoAOwVf5D)@FKnMr{As_^VfDjM@LO=)z z0U;m+gn$qb0zyCt2mv7=1cZPP5CTF#2nYcoAOwVf5D)@FKnMr{As_^VfDjM@LO=)z z0U;m+gn$qb0zyCt2mv7=1cZPP5CTF#2nYcoAOwVf5D)@FKnMr{As_^VfDjM@LO=)z z0U;m+gn$qb0zyCt2mv7=1cZPP5CTF#2nYcoAOwVf5D)@FKnMr{As_^VfDjM@LO=)z z0U;m+gn$qb0zyCt2mv7=1cZPP5CTF#2nYcoAOwVf5D)@FKnMr{As_^VfDjM@LO=)z z0U;m+guuTS0(*Q!|0nLfxF5#-b=;5OmP-f-0U;m+gn$qb0zyCt2mv7=1cZPP5CTF# z2nYcoAOwVf5D)@FKnMr{As_^VfDjM@LO=)z0U;m+gn$qb0zyCt2mv7=1cZPP5CTF# z2nYcoAOwVf5D)@FKnMr{As_^VfDrh%Bk(h#*}2h@U7<>>do3S(rl@7XCo?2E6-zQZd1kfxoX8cAP zvFd1KtdcT3Hm}a7Oj`3-A72EW<{Li#P?328V=(GJQ&*)^YOI7ZtH{WVqYt`vrcS>F zdA#-c2jM%TL!Pds%*hgCBwfk#0(Ogx%<+#KnOAC!4F1;Q@`Iap+LFxYo^ly52yrvaY^P0Cv zd!2b~D}L?~&hyk3KXVyAJ&!W)p`WL)cRg7^8_U01^yu&W)OZAI%&d7kpAxcxqwKQ^@cA_%ss(-+yA(&8e@G zVV=Wy4wM(06^K3UiQ?4bD=3rHjO7m%QL1teb?X@0N?m&~SwKt2wX2Z(?BP-5-F&rC z&-#_fJ>WN%S5R`+Y4oYEAh~HC#)rr94qtMU{s3+KTp@fl`ntm!H4Z?Z=jWZK`g7Qu z*_VMe`;8*{MRKA^`5t{gzg~q{zeZm zqSEFeeK&zwQvvpkIn;P|5%#AQym!nQUXJ}}m5*iz3XSDABj*Y2n&B4^!_ATN<^Dbm z8QUD3X50J(bqnLlveWw>o6GI~$+0Ocv~7-}-VA;HC~ahYF-qQZDi>1sVu!1y70qk@ zh*8hF-W>g%=xg13)=kWm*?t?oIREuCsj*@HnL5s$`@s2^VK4m(-lyDe&Sw?gSsZWB zY4fczYMgs*exF}=><{k1dldP){pG&8&s1Xf{ooE`nuM=mws{?EmR$w9K8q0nFdig|v~| z4SrxP?d%YFK;BFi6u=F2{3tPk#hHtCF;FMX#^>&a-%*!6sc~ zkI6CUxLt6x82d$*|L@VR6uR->Tg#rhDnFkxN{$@IT3ds4x%nOIyFB(T_W{gnW)J;O zS>`7v4bpl6-xEqmM(+WA_I3!@(8U z_Yiv!7-$1eqa5&AYUP@KY9CBtJqhIN3weVB9pG5UDgPj~T{bz*Ee*EIIm-2QN1 zcPxwL%;oExJ%jb1 zfqqM_{_^wn4v$P-j@=dPCnpa3*`BMPe!d@vTV`R79p~%yQhCR{_(PuhQq-{@sB5xs#6JK3 z$d@u#`*8l(%##|<|2Sh0-G%f2CuwA8(Q16pOvAlKeP8gm4WI1~Ru-CsGs|S(b5rNn z`>_{dESLKW`0StDw2Nq2A@<(YvyFN@FD1MqFz?&3mK*tO%{F%F=3}29w@>n!j;bL& zMEdm9OXsY;J&>t-a(e2QsB>HQ?A#L%#q_Q~BplQO34LL_VPR{-WjZd~Eh+tuKr9lD zY`u`hs!!d~poeYLh*P>Pk2&>|u6(BL$JM)KsUz2& zvfe2-IAxbpCY`e1DTkbL)G3cSV3on^;ca>FNXT)OP0y87jTmSAhB&5hd=iFbv9 z;kIz7RS(5t(U_xX3AE~6v1l+9k1I(yvOUldhAa>ZbcPZk*ABz0N89wyP-ir@6P8Zh zX&SN{Lmm%p?F>Z{fkZeO!F=lJNZ5f~zoCvcy$uu5mJ&G*^l)5n4|iaKItqKr^PKI8 zv2i4-$9sa?^w!Y!a1a8!)FaV^9@q}6K#T3ume+C!_Sw=C=WMg>l_DC`Tf?zXFk$zF z^{@@(3YFs=K`<7L$DQbUM>x{Xjd1lu!kt|mA?`ECwuJ%-J74b0)AdvlynOUPM=TU*-N}nKo|xh~l`r>-lM8irCD66Sk)Do@j}fUc)fc+> zXra^cPCd{G@%0$wa3rC3ZG|IV-0A_N9FI>eb-P0s_^FZ4ky=)+TEl^>_)3;!&Wdm*|>-BQJ*`GZro!Rk-v2j zd8u|`KT9qm-*6Fmmq)$M?G?PeAju#_pd%A?lF&_CfM`%x^j0x zy7?Fd`Fz~C&7QctabK#xX^)?7C$1|W;4rw{xXEc;%g!G+?z5h_uDx-^h3&ga^5yrU z>O+rR`SYyC<@&Ez3;X=I?t9j@&e|__-ea!(i+H#chkj3<{U2g~%wzB7xy}>!VfdMa zo^5c(_ZlameU9Aac?2I;eBoTL>0kbD7);q{Km0&*R|WApZCZ&B)KNSJyLuAAZ4}=b4+c6qAaI_j2WM(j-9t5dX=}x42rucj z&IB~-G1Aj%zuBAO38=QhFBhyhiJRom|Ak$7!kuw59Dq+Z8o6@t1{?^CyID$${0=)=;7=+)7jv-`SZ6w4hAH>~fp? zlnC`Es3sO|4I}~%_6~b2goK6!o-bWX9%GYQ2y4z&6i* zvhRE5q(-TU8I+mO{KNZJ`&0WLDm3@+@}>68r_8wKs~&`Ga1mw3G#XA9BxmjaIAsR5 z&QXjo$ zr8%_7XCABsg9%C*3lp=5VbKzSrXRo9uJAwh+%n=HTvP6>I-P(@l=|_@kGp{GMo{4CjOp zD$lu7Is7ay2Wz$e7q4OL$95N*;~1ZG35}$|{MDO{OnR5ke61XF;zk;=>S$!F(w-Bq zVxFWmfAw+95scUH@rR1c6POF5{xfw|I;F-+D6@);%sBd>YiH{8Tad?FpMMa(Gdkqy zTFRU(F-FprJWpY_$jBW3xRH6K*2v(0WAUpr=4KjWgxpd*Mw(7Dk0Xcsh`PV(qehOA z*0gGl^QvW)`=RF?o1p*ZIjeE3%*a%Y>bp-sH;(63cxLm(bhNBzt=)e-=^^D1VW*^Ot-FI^ARC}N8+su)3TfW@*NylcuW24&p?e<^y z=yKbSYPL z2|Oo#XX+|xcDsSOk=ABcKMna9O6)<^-&fCUUsgSVz2z|K>Ur}@bFweHuHAe&p5GbQ z3B+Rm?;<|;f&VuvRiniz%%i={hm%YvPe0gFSVm!@2sh9;6_5$;f3S)$Gdl$KIe(!O9LCrjc{N9gGGcnMIPHeg?^>s4La~RKo@?x_BvT3iC zrygHHnWSbcf2fF3m3ydL$Jkct+Kb5oS~{*>h1_Qkk0S5ptBrcruSD(vzp=c6lCw^u zPlW}^P4h55JeGH0t?3WY#?KYPSEH{xtWo0t^m%^XX{tYmcNY6Hu!nr3h<=frXi~mM z-_NgCA=aK{ z-(z#R-9I@tg@v}wQPi8EuOFq2tS?5%`&Z>c>R#+{)wH5{%^xx9S=XDRzY~3}d(XOw znKE0f`*O~I{Y+|Xn17~@bLT#A{$<$Pzk>4y_nY%sg?A{&8+6)ytBe}wUYpaxrO%k%+7Deef;fV>fU@M9d5>a8mObpD|3x|?=B%V|I&PZ z@zK&J>x>L@c$m)ARTR<2$|Aaf=P>(!dKxWr=WhV>_jDm`SET6W)-&X% z!>?d%YFK;BFlbbyT#n~(9{LD;RwZfUie6v$ooDfWg-yE19+P9xal0T@V86)n|2^83 zLO1?ymI$&uq&YiqDBH@{v_z-3_tr`wU(W|I^WOz zkgs~%TGmjJFXwhVFZ!VW!W)QRlCLjt^l|8U?B2kBIJg4)9%2t-Jb7*K_@$sPLf) z{y+b~IxpYnJoYbn>c#Nyw!ct+Z}G$*^3<22j{QJglYJxh_sEZYDRZ?C-y@oNQp5KM z&iq4n;d|thG%~bkHNNwv;a;P@cleu+zcmh47Mg@J%VghkQ|D=%d5#$vll?#c#xHmM zkEh?+PowV);!L;_C4EkNl&;6O!S{^^4WbO4yZZ(I=Z-$=BmYlV7OdP|b?!gDa9uL+ z!bH|-)?R*1cR^=zXa&r8_{ z)>%9Djz-%T_M04^RoBq0)i^he!!`&X<7!{jcCVaG^-VZ)9i3`tXEE9Lk7gC(xMx!w zf{&dlqs*?B!qioG-gh0owUHlk`DMWXrF`4wAMtIw?g*9QEJ6jZa~_pO;oH17ET6RY z8n}7?vG?M#>gIttDV)tVZa-)39fq$fu^+7;D@=V6{(m}ndFsAGy1|%6jdu@H>Uq?E zr(HIDLc4sp5b^l^Gfsv0_L)BHLylZ8SYsSn{kN-d9Gv$0{RP@Qj_(V9^vFLWpWhD9 z+xKJsO+%i=@b^3Td4{fdy%}ro0n}et^ThV+hYkDlZEHt=7U8Z|eZn``>j9@vWAMef zbB;cop{6^11G#?E=cZPoPdrAitJ39p#Qef#eAdn~J-nY%cxU_)XZzuY@m}Y+ucA%z z-W$;m>K-Vd#-HMyhFGcl{sSqNDn@ffYBB8C{(qu0e6)C+V{UB$*ZNqrAI z@6OwP8mLIkfsW%Q?~SWC>%rkykhyVI!Vl(mKl@sx&q=LB-qbuN^~C%5KKPrmd7nY9 z2JU^>&vxN^%UArvxq8w+k3L89Xfe&i{GZnaJ@>pP9*XH*fk-&02NL?ic*DZhhRbwZ zti+}CI|8vtII{IZ7OOsWM}r>f4JY)rK)55+T63A*E}m!&M;eHtU7?76i>lq`m^A1M zL#~!66ntQolNfBMx^DSquVX3e2%&-=%< z`{R`@w1gg>iqC$RtJ_3`rQ34AsULC5 zV;;S0mv-cDIb|_61CFOVrQwttoU+*|d!4f1DF>YLh*PGWa>)&!xN+&So9gP92U>!y zp*A;ePbA(I3WnRlp;kQ{5?J6MA4ftO6~zPg`EgA=qb2Pn@&OwpWU1 zOm7XxLcxUH7uLfzkSkPmA`pJ2%4B6A5>Ab%eOjAlnuSBr}qnD^4!d*_A-o7DswI zIzC3E##CSE;-iI5%eH8&U5~a;t>+zq{H>rfkcfqQG2HREvblZbop(3rjWvzVKEeA- zt!<>%ei!D(!d6Zq))P^i&SF#(;m%O}V^#3r=v$-ze>?XcB-K?XaQt-lFc6V91|m@| z0vkk^K@H(y0v#YGu1N_Amb=LxGfhteEi*mCbPoa9E!&M%x*Of14O_{YRN3SuJ+%)? zS!EJ8nP@EIu2$h{QfB?b2FzpESCmBP80`K1&OK+k`!X|Fy1SLC{oYh_@BN+MdEC!E z_x9}zm0U)5Sr9+~0R#|0009L4mjvW}$3>Tf@mckHb8qmjf1#(3pNlwmNd83qZn3AValES~KQ2B#9=6+XD@O81#~F#QzgWNW0;738 z+~E6l@t3q%7TNkkA@B8hF#P7REI|n2AzMZPQaYx60;rGZxcKnU_L3`iwo)><<4B7lf zyfy#A@11|L`HgsMe%#EzR1z+5j+(ownDf^bo_xGP0UH%`t{GzONzHN5#t6cnq z-LKb+F8(Jz-;L8B*EK%w&R5{_-{<1V$aze-c$n6h4+2W@3Xx4bpZ<_bG{3sPb?bA!F%8_>?Yyw1$+Gy|zzw-n|w z`wG&BhGcK_HKuH&bZ@)%u^|-sQIndax@7`nvV#R)l|Um)E-XXjix`sp+=T z?u7kes|8YBPrbc;jW(s%X*eOiZ=ZKZ3*0@Si{&9sk|vi842elRPnKjJS)S0(&D zaXYe-SKqe2owgbd^bhGj2ih2pZB>_O{k1OU>-*n7zia(X zk^263((hWo*-ppj7hk{V-ofj*KPVqRe*b-PiNyTaeO+CDYI*3_8Lmc)BD?aJm1Ea` z;T>V7{M~5F60P4-$j{KrYksd$&bJ1Q0*~0R#|0009ILKmY** z5O{qGYh%-$-^t4VxWKLbzwhyxtPwx}0R#|0009ILKmY**5I_I{ z1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009IL zKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~ z0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY** z5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0 z009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**{(lHO9y8_%`Fu@2Uzg95^7)2*SP(z} z0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY** z5I_I{1Q0*~0R#|0009ILKmY**5J2GnPT-s|bDMj6EwXQ7W^Of+tesqJN~>b#kmR=H zpFi-G%K3yTotbKCXA<$D`!~f)58TyWdEky%>Hfv0b}A8Depq6MuQ0Wf2{Tk}E6#r4 z9i~>k%?wLh2hGUUrN=wX@I&`bu6*HoQ`=(9p$F$}t{lF@l~G>#>* zeP(L1cK*P(j*KKapE-Zvu_lk$&_OBhiYJFno0-)WH^X7>*KDqi`22ydR!+d& zPAaM8vL(sNgPTn0k1oHqa^#9w<>7^5aK@CLc<{5cAKzex7t47lZ%7WGKk(#eJKB~m z=MNmS{U1prN|i*sbYyW`<;C{m?24El5p%U3>eyTv>WE3(*U2Yd(Kd%V+GTrk%CV9Fg-ibo#EzmFHwGjKt5aUKE(p z$r+}$(Iji9sMa*kwdB|S%pz|1-=WB8acdp;90+q92rBC%ZidA?qs7ngD!W2cn={cH7Y zC+8$a(AnPL4; zHsxr2J?_UOCi!wbUsLJ~x9xpGJmzUTet9yoeE5zD`Cw>5vx@-uSne_QtL_AS>|&L3!8 zcYZwDU)-IaGt!r~|FZONBqnp`;G$rCNzU`#UDD^U^pG7=gNVF~2dqmwN zexe?`rhM~rI?h*ai>u4f*#)NdYdOwW>?679=> zX3VxziA&`^_$C?W{N=^?hG+8aW>2bKm%L<6ERLGuq>|4$EyrVSxP8qTF?;2pRe0~$@&~R@*1G<*YhT)!nO#SMJB^riw#j|wGPCW~E98E*LGLw}4Xu^? z*GBnnO|>W2zE{S4CUN=D_oNR$Mjfvn@1qjaoQEfcIsYpAbvdsZJA41<->mh%;d0vB z!<;8%ze~#3jF_!jK1J4gk6O6Y>{;eq6N$+Sy55>xqhnxQ!*!HPRq4^>Cja&ANyUuKbtQ7 z&TH-WQR(;9%Ouv(@Hf(y@6*mLsekL-)_Qu3KRRai+%nx9>yr6YUTtbWesl7pH_kA2 z{#Et)B4)1q(&}VQ9quvbR?nYowl17(-lcO`+kbSDS>@+%O6Kp`cC%IM9sX^7m7dF+ zUaZ?Z%`xu0OOCxh-z#H!WP0nk)mokB%lQ9R%3qz?TCT^nP`8}`Nay#H1C{*#>QXA3Kj%SW34xg55ZL?gLxBRO9Wj*)ay)5%u?WO$R zv|P{mUr9UnKVM(<>ZPsid|cuie_3C(ZhmWA>*>5WAmxAgQ|W(3Yk8Y1&r7+UyPwMa z@bCt??@8Z><$UV3q35q8<&)*u&LpRmG_U5TetJ%cIoHZ#b(w9IY&25HQN@uIr*yO?3e56 zQ_bZvm*3{@8Cw2lQhr-=`5P}RcOJFXO?lI_ou+=YoxRQFmt5G6bE_?F%G2>i+gZ_E zKIOu8<}{aU{mAj?^(^PFHnk~Fzq9mSqq+XL^;*nHPI;4@yODKwYFuO4rg}HYd5Mm% z7^$-|vcK;l^|wUwABpVW5ZRY!u$D2(^GVD8Ph{Vp|L^>^e!+OZ8OeVpvOh)I_w~m* z-m4@1%QIt3|5wVs+z;$>{=jhfd*SP`Qe{(2elH{{FDB&of;^KS`GEXh_@Eg+a>XY3 z9X3fmo9*ur{Y|F71s-15UNQ2_a{jtf8v z&q8^AsZK9F@tQmz%5%Q9*?86~&fY0`<)`nx`9reSdzUa@}ma(^(% z>7|>beoOhtG+Ul><=091U_LVt^rv#!bdV|pOY`fNcCULwAWJxWW6N((4dk-9onvKe z{LuC5g3P{bA=r`1_GG$OydkWZFLY;f>x}8^&*Xxu?cREqvMyMf@oBdGj-KrHe8*sK zDw|u8cKu3?zQMw}Mt#|fHZa+otWT9s_}@XhHB)}hpR8YI-g3czKJR~Sz1^%d_l)+h z?XI)+gNhMYlfZ)-&bhfR1;KTd#cAyWex=s;gG7S)1CP?#}G+ z{SM~x{h4%jM>f+PWCjNM23*1RRCmxn(3j5S^EQ&r?M?M$C6*dU^=1kgpC_j`=-UzW zW_tSu_DiPQcU3cCWr^oAJ9{&^LaLDM%gKBi>?wpD`F1lsJAxfD5yPl4u7Mz%4|ZpJ zWP-YaaLVhP9URc)TwjnMOz#T1GkddX354q)*H;KqdnGHiJ!~_KcW4*d=Jvt7jA`IcOVh3Zw`6@_E;L(38#W)=IJr=CZy0JsCY_k=>C=6~giA zvFuFs1o=$HrWX48g5Feazdd;AYakQorDrY}Pmt;v$fUaW>qVO{jJl4FSC2{um+9>< z$e}IE4fgcB-57gLgQcF&EOj;e~^ZY1uc=zTyga+{&|kB)p#CeyF{qnEeQ zy=7AQ?ZGwINOd!sA~PeOAH6+<2;=;jFfmtBuQ#V8O z#LMHvN5+X)$B7H-nBxf|@hy@yk$*ChKYCo_#QVpI7b9^$ANnKn;lmdh&%sDLoszHT z#mBp4Ul%`4GfDRv^TGG~sYt*2#W9gXInvG^$=7~;ye#{=_qHCg;|bw$3XecXS~huMw%Eps%|X#bP4H<81Z$T$yPB>!L} z-;bw1((irJ&TKg{>0Fk2{c>H&^00g~?mdso4@g~(%17f}ZvM-AF1uVJO=$dtyWaP? z_^_-s9@zbQ-F73}&$_supFaMWi~ISbe=(xVlk(AaimqQj&hNN*Qr0^DPuR6FPs>{K z{q^hff8^rcbB)V?!NvV`9k{rjXKlw{Uw)h~NqnZnm)N|1e%Z@*+-Z+nC7oTYh8gcu;Az$-L_L;uzAL+1P|Mu}do68o`echQAea1d}X->D% z9|nE0neHDfq<5M8{(L%R9~0b`8I;OW?tdI;D?7U7sViQUW%_J&U=o%wu96}r7Y-pkVj)5bTCDW%j1SE~E|p^>9m2k5&J$ zUI?03}&g)*_i*f=-uX$qza+#jj{g@db0tg_000IagfB*srAb Date: Sat, 1 Dec 2018 22:26:55 +0800 Subject: [PATCH 19/22] aarch64/mmu: move crate aarch64 to remote --- crate/aarch64/Cargo.toml | 16 - crate/aarch64/src/addr.rs | 441 --------------- crate/aarch64/src/asm.rs | 158 ------ crate/aarch64/src/barrier.rs | 91 --- crate/aarch64/src/lib.rs | 29 - crate/aarch64/src/paging/frame_alloc.rs | 15 - crate/aarch64/src/paging/memory_attribute.rs | 64 --- crate/aarch64/src/paging/mod.rs | 549 ------------------- crate/aarch64/src/paging/page_table.rs | 267 --------- crate/aarch64/src/paging/recursive.rs | 425 -------------- crate/aarch64/src/regs/cntfrq_el0.rs | 31 -- crate/aarch64/src/regs/cnthctl_el2.rs | 75 --- crate/aarch64/src/regs/cntp_ctl_el0.rs | 62 --- crate/aarch64/src/regs/cntp_tval_el0.rs | 30 - crate/aarch64/src/regs/cntpct_el0.rs | 29 - crate/aarch64/src/regs/cntvoff_el2.rs | 32 -- crate/aarch64/src/regs/currentel.rs | 52 -- crate/aarch64/src/regs/daif.rs | 90 --- crate/aarch64/src/regs/elr_el2.rs | 30 - crate/aarch64/src/regs/far_el1.rs | 31 -- crate/aarch64/src/regs/hcr_el2.rs | 123 ----- crate/aarch64/src/regs/id_aa64mmfr0_el1.rs | 82 --- crate/aarch64/src/regs/macros.rs | 85 --- crate/aarch64/src/regs/mair_el1.rs | 82 --- crate/aarch64/src/regs/mod.rs | 55 -- crate/aarch64/src/regs/mpidr_el1.rs | 30 - crate/aarch64/src/regs/sctlr_el1.rs | 103 ---- crate/aarch64/src/regs/sp.rs | 28 - crate/aarch64/src/regs/sp_el0.rs | 31 -- crate/aarch64/src/regs/sp_el1.rs | 36 -- crate/aarch64/src/regs/spsel.rs | 48 -- crate/aarch64/src/regs/spsr_el2.rs | 106 ---- crate/aarch64/src/regs/tcr_el1.rs | 313 ----------- crate/aarch64/src/regs/ttbr0_el1.rs | 61 --- crate/aarch64/src/regs/ttbr1_el1.rs | 61 --- kernel/Cargo.lock | 6 +- kernel/Cargo.toml | 2 +- 37 files changed, 5 insertions(+), 3764 deletions(-) delete mode 100644 crate/aarch64/Cargo.toml delete mode 100644 crate/aarch64/src/addr.rs delete mode 100644 crate/aarch64/src/asm.rs delete mode 100644 crate/aarch64/src/barrier.rs delete mode 100644 crate/aarch64/src/lib.rs delete mode 100644 crate/aarch64/src/paging/frame_alloc.rs delete mode 100644 crate/aarch64/src/paging/memory_attribute.rs delete mode 100644 crate/aarch64/src/paging/mod.rs delete mode 100644 crate/aarch64/src/paging/page_table.rs delete mode 100644 crate/aarch64/src/paging/recursive.rs delete mode 100644 crate/aarch64/src/regs/cntfrq_el0.rs delete mode 100644 crate/aarch64/src/regs/cnthctl_el2.rs delete mode 100644 crate/aarch64/src/regs/cntp_ctl_el0.rs delete mode 100644 crate/aarch64/src/regs/cntp_tval_el0.rs delete mode 100644 crate/aarch64/src/regs/cntpct_el0.rs delete mode 100644 crate/aarch64/src/regs/cntvoff_el2.rs delete mode 100644 crate/aarch64/src/regs/currentel.rs delete mode 100644 crate/aarch64/src/regs/daif.rs delete mode 100644 crate/aarch64/src/regs/elr_el2.rs delete mode 100644 crate/aarch64/src/regs/far_el1.rs delete mode 100644 crate/aarch64/src/regs/hcr_el2.rs delete mode 100644 crate/aarch64/src/regs/id_aa64mmfr0_el1.rs delete mode 100644 crate/aarch64/src/regs/macros.rs delete mode 100644 crate/aarch64/src/regs/mair_el1.rs delete mode 100644 crate/aarch64/src/regs/mod.rs delete mode 100644 crate/aarch64/src/regs/mpidr_el1.rs delete mode 100644 crate/aarch64/src/regs/sctlr_el1.rs delete mode 100644 crate/aarch64/src/regs/sp.rs delete mode 100644 crate/aarch64/src/regs/sp_el0.rs delete mode 100644 crate/aarch64/src/regs/sp_el1.rs delete mode 100644 crate/aarch64/src/regs/spsel.rs delete mode 100644 crate/aarch64/src/regs/spsr_el2.rs delete mode 100644 crate/aarch64/src/regs/tcr_el1.rs delete mode 100644 crate/aarch64/src/regs/ttbr0_el1.rs delete mode 100644 crate/aarch64/src/regs/ttbr1_el1.rs diff --git a/crate/aarch64/Cargo.toml b/crate/aarch64/Cargo.toml deleted file mode 100644 index 7514dd0..0000000 --- a/crate/aarch64/Cargo.toml +++ /dev/null @@ -1,16 +0,0 @@ -[package] -name = "aarch64" -version = "0.1.0" -authors = ["koumingyang <1761674434@qq.com>"] - -[dependencies] -register = "0.2.0" -bit_field = "0.9.0" -bitflags = "1.0.1" -usize_conversions = "0.2.0" -os_bootinfo = "0.2.0" -bare-metal = "0.2.0" - -[dependencies.ux] -default-features = false -version = "0.1.0" \ No newline at end of file diff --git a/crate/aarch64/src/addr.rs b/crate/aarch64/src/addr.rs deleted file mode 100644 index cf3f092..0000000 --- a/crate/aarch64/src/addr.rs +++ /dev/null @@ -1,441 +0,0 @@ -use core::convert::{Into, TryInto}; -use core::fmt; -use core::ops::{Add, AddAssign, Sub, SubAssign}; - -use bit_field::BitField; -use usize_conversions::FromUsize; -use ux::*; - -#[derive(Debug)] -#[repr(u8)] -pub enum VirtAddrRange { - /// 0x0000000000000000 to 0x0000FFFFFFFFFFFF - BottomRange = 0, - /// 0xFFFF000000000000 to 0xFFFFFFFFFFFFFFFF. - TopRange = 1, -} - -#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] -#[repr(transparent)] -pub struct VirtAddr(u64); - -#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] -#[repr(transparent)] -pub struct PhysAddr(u64); - -#[derive(Debug)] -pub struct VirtAddrNotValid(u64); - -impl VirtAddr { - /// Creates a new canonical virtual address. - /// - /// This function performs sign extension of bit 47 to make the address canonical. Panics - /// if the bits in the range 48 to 64 contain data (i.e. are not null and no sign extension). - pub fn new(addr: u64) -> VirtAddr { - Self::try_new(addr).expect( - "invalid virtual address", - ) - } - - /// Tries to create a new canonical virtual address. - /// in aarch64, valid virtual address starts with 0x0000 or 0xffff. - pub fn try_new(addr: u64) -> Result { - match addr.get_bits(48..64) { - 0 | 0xffff => Ok(VirtAddr(addr)), // address is canonical - other => Err(VirtAddrNotValid(other)), - } - } - - pub fn new_unchecked(addr: u64) -> VirtAddr { - VirtAddr(addr) - } - - /// Creates a virtual address that points to `0`. - pub const fn zero() -> VirtAddr { - VirtAddr(0) - } - - /// Converts the address to an `u64`. - pub fn as_u64(self) -> u64 { - self.0 - } - - /// Creates a virtual address from the given pointer - pub fn from_ptr(ptr: *const T) -> Self { - use usize_conversions::FromUsize; - - Self::new(u64::from_usize(ptr as usize)) - } - - /// Converts the address to a raw pointer. - #[cfg(target_pointer_width = "64")] - pub fn as_ptr(self) -> *const T { - use usize_conversions::usize_from; - - usize_from(self.as_u64()) as *const T - } - - /// Converts the address to a mutable raw pointer. - #[cfg(target_pointer_width = "64")] - pub fn as_mut_ptr(self) -> *mut T { - self.as_ptr::() as *mut T - } - - /// Aligns the virtual address upwards to the given alignment. - /// - /// See the `align_up` function for more information. - pub fn align_up(self, align: U) -> Self - where - U: Into, - { - VirtAddr(align_up(self.0, align.into())) - } - - /// Aligns the virtual address downwards to the given alignment. - /// - /// See the `align_down` function for more information. - pub fn align_down(self, align: U) -> Self - where - U: Into, - { - VirtAddr(align_down(self.0, align.into())) - } - - /// Checks whether the virtual address has the demanded alignment. - pub fn is_aligned(self, align: U) -> bool - where - U: Into, - { - self.align_down(align) == self - } - - /// Returns the 12-bit page offset of this virtual address. - pub fn page_offset(&self) -> u12 { - u12::new((self.0 & 0xfff).try_into().unwrap()) - } - - /// Returns the VA range - pub fn va_range(&self) -> Result { - match self.va_range_bits() { - 0x0000 => Ok(VirtAddrRange::BottomRange), - 0xffff => Ok(VirtAddrRange::TopRange), - _ => Err(VirtAddrNotValid(self.0)), - } - } - - /// Returns the top 16 bits - pub fn va_range_bits(&self) -> u16 { - ((self.0 >> 48) & 0xffff) as u16 - } - - /// Returns the 9-bit level 1 page table index. - pub fn p1_index(&self) -> u9 { - u9::new(((self.0 >> 12) & 0o777).try_into().unwrap()) - } - - /// Returns the 9-bit level 2 page table index. - pub fn p2_index(&self) -> u9 { - u9::new(((self.0 >> 12 >> 9) & 0o777).try_into().unwrap()) - } - - /// Returns the 9-bit level 3 page table index. - pub fn p3_index(&self) -> u9 { - u9::new(((self.0 >> 12 >> 9 >> 9) & 0o777).try_into().unwrap()) - } - - /// Returns the 9-bit level 4 page table index. - pub fn p4_index(&self) -> u9 { - u9::new(((self.0 >> 12 >> 9 >> 9 >> 9) & 0o777).try_into().unwrap()) - } -} - -impl fmt::Debug for VirtAddr { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "VirtAddr({:#x})", self.0) - } -} - -impl Add for VirtAddr { - type Output = Self; - fn add(self, rhs: u64) -> Self::Output { - VirtAddr::new(self.0 + rhs) - } -} - -impl AddAssign for VirtAddr { - fn add_assign(&mut self, rhs: u64) { - *self = *self + rhs; - } -} - -impl Add for VirtAddr -where - u64: FromUsize, -{ - type Output = Self; - fn add(self, rhs: usize) -> Self::Output { - self + u64::from_usize(rhs) - } -} - -impl AddAssign for VirtAddr -where - u64: FromUsize, -{ - fn add_assign(&mut self, rhs: usize) { - self.add_assign(u64::from_usize(rhs)) - } -} - -impl Sub for VirtAddr { - type Output = Self; - fn sub(self, rhs: u64) -> Self::Output { - VirtAddr::new(self.0.checked_sub(rhs).unwrap()) - } -} - -impl SubAssign for VirtAddr { - fn sub_assign(&mut self, rhs: u64) { - *self = *self - rhs; - } -} - -impl Sub for VirtAddr -where - u64: FromUsize, -{ - type Output = Self; - fn sub(self, rhs: usize) -> Self::Output { - self - u64::from_usize(rhs) - } -} - -impl SubAssign for VirtAddr -where - u64: FromUsize, -{ - fn sub_assign(&mut self, rhs: usize) { - self.sub_assign(u64::from_usize(rhs)) - } -} - -impl Sub for VirtAddr { - type Output = u64; - fn sub(self, rhs: VirtAddr) -> Self::Output { - self.as_u64().checked_sub(rhs.as_u64()).unwrap() - } -} - -/// A passed `u64` was not a valid physical address. -/// -/// This means that bits 52 to 64 are not were not all null. -#[derive(Debug)] -pub struct PhysAddrNotValid(u64); - -impl PhysAddr { - /// Creates a new physical address. - /// - /// Panics if a bit in the range 52 to 64 is set. - pub fn new(addr: u64) -> PhysAddr { - assert_eq!( - addr.get_bits(52..64), - 0, - "physical addresses must not have any bits in the range 52 to 64 set" - ); - PhysAddr(addr) - } - - /// Tries to create a new physical address. - /// - /// Fails if any bits in the range 52 to 64 are set. - pub fn try_new(addr: u64) -> Result { - match addr.get_bits(52..64) { - 0 => Ok(PhysAddr(addr)), // address is valid - other => Err(PhysAddrNotValid(other)), - } - } - - /// Converts the address to an `u64`. - pub fn as_u64(self) -> u64 { - self.0 - } - - /// Convenience method for checking if a physical address is null. - pub fn is_null(&self) -> bool { - self.0 == 0 - } - - /// Aligns the physical address upwards to the given alignment. - /// - /// See the `align_up` function for more information. - pub fn align_up(self, align: U) -> Self - where - U: Into, - { - PhysAddr(align_up(self.0, align.into())) - } - - /// Aligns the physical address downwards to the given alignment. - /// - /// See the `align_down` function for more information. - pub fn align_down(self, align: U) -> Self - where - U: Into, - { - PhysAddr(align_down(self.0, align.into())) - } - - /// Checks whether the physical address has the demanded alignment. - pub fn is_aligned(self, align: U) -> bool - where - U: Into, - { - self.align_down(align) == self - } -} - -impl fmt::Debug for PhysAddr { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "PhysAddr({:#x})", self.0) - } -} - -impl fmt::Binary for PhysAddr { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - self.0.fmt(f) - } -} - -impl fmt::LowerHex for PhysAddr { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - self.0.fmt(f) - } -} - -impl fmt::Octal for PhysAddr { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - self.0.fmt(f) - } -} - -impl fmt::UpperHex for PhysAddr { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - self.0.fmt(f) - } -} - -impl Add for PhysAddr { - type Output = Self; - fn add(self, rhs: u64) -> Self::Output { - PhysAddr::new(self.0 + rhs) - } -} - -impl AddAssign for PhysAddr { - fn add_assign(&mut self, rhs: u64) { - *self = *self + rhs; - } -} - -impl Add for PhysAddr -where - u64: FromUsize, -{ - type Output = Self; - fn add(self, rhs: usize) -> Self::Output { - self + u64::from_usize(rhs) - } -} - -impl AddAssign for PhysAddr -where - u64: FromUsize, -{ - fn add_assign(&mut self, rhs: usize) { - self.add_assign(u64::from_usize(rhs)) - } -} - -impl Sub for PhysAddr { - type Output = Self; - fn sub(self, rhs: u64) -> Self::Output { - PhysAddr::new(self.0.checked_sub(rhs).unwrap()) - } -} - -impl SubAssign for PhysAddr { - fn sub_assign(&mut self, rhs: u64) { - *self = *self - rhs; - } -} - -impl Sub for PhysAddr -where - u64: FromUsize, -{ - type Output = Self; - fn sub(self, rhs: usize) -> Self::Output { - self - u64::from_usize(rhs) - } -} - -impl SubAssign for PhysAddr -where - u64: FromUsize, -{ - fn sub_assign(&mut self, rhs: usize) { - self.sub_assign(u64::from_usize(rhs)) - } -} - -impl Sub for PhysAddr { - type Output = u64; - fn sub(self, rhs: PhysAddr) -> Self::Output { - self.as_u64().checked_sub(rhs.as_u64()).unwrap() - } -} - -/// Align address downwards. -/// -/// Returns the greatest x with alignment `align` so that x <= addr. The alignment must be -/// a power of 2. -pub fn align_down(addr: u64, align: u64) -> u64 { - assert!(align.is_power_of_two(), "`align` must be a power of two"); - addr & !(align - 1) -} - -/// Align address upwards. -/// -/// Returns the smallest x with alignment `align` so that x >= addr. The alignment must be -/// a power of 2. -pub fn align_up(addr: u64, align: u64) -> u64 { - assert!(align.is_power_of_two(), "`align` must be a power of two"); - let align_mask = align - 1; - if addr & align_mask == 0 { - addr // already aligned - } else { - (addr | align_mask) + 1 - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - pub fn test_align_up() { - // align 1 - assert_eq!(align_up(0, 1), 0); - assert_eq!(align_up(1234, 1), 1234); - assert_eq!(align_up(0xffffffffffffffff, 1), 0xffffffffffffffff); - // align 2 - assert_eq!(align_up(0, 2), 0); - assert_eq!(align_up(1233, 2), 1234); - assert_eq!(align_up(0xfffffffffffffffe, 2), 0xfffffffffffffffe); - // address 0 - assert_eq!(align_up(0, 128), 0); - assert_eq!(align_up(0, 1), 0); - assert_eq!(align_up(0, 2), 0); - assert_eq!(align_up(0, 0x8000000000000000), 0); - } -} diff --git a/crate/aarch64/src/asm.rs b/crate/aarch64/src/asm.rs deleted file mode 100644 index bada049..0000000 --- a/crate/aarch64/src/asm.rs +++ /dev/null @@ -1,158 +0,0 @@ -//! Miscellaneous assembly instructions and functions - -use paging::PhysFrame; -use addr::{PhysAddr, VirtAddr}; -use regs::*; - -/// Returns the current stack pointer. -#[inline(always)] -pub fn sp() -> *const u8 { - let ptr: usize; - unsafe { - asm!("mov $0, sp" : "=r"(ptr)); - } - - ptr as *const u8 -} - -/// Returns the current point counter. -#[inline(always)] -pub unsafe fn get_pc() -> usize { - let pc: usize; - asm!("adr $0, ." : "=r"(pc)); - pc -} - -/// The classic no-op -#[inline] -pub fn nop() { - match () { - #[cfg(target_arch = "aarch64")] - () => unsafe { asm!("nop" :::: "volatile") }, - - #[cfg(not(target_arch = "aarch64"))] - () => unimplemented!(), - } -} - -/// Wait For Interrupt -#[inline] -pub fn wfi() { - match () { - #[cfg(target_arch = "aarch64")] - () => unsafe { asm!("wfi" :::: "volatile") }, - - #[cfg(not(target_arch = "aarch64"))] - () => unimplemented!(), - } -} - -/// Wait For Event -#[inline] -pub fn wfe() { - match () { - #[cfg(target_arch = "aarch64")] - () => unsafe { asm!("wfe" :::: "volatile") }, - - #[cfg(not(target_arch = "aarch64"))] - () => unimplemented!(), - } -} - -/// Exception return -/// -/// Will jump to wherever the corresponding link register points to, and -/// therefore never return. -#[inline] -pub fn eret() -> ! { - use core; - - match () { - #[cfg(target_arch = "aarch64")] - () => unsafe { - asm!("eret" :::: "volatile"); - core::intrinsics::unreachable() - }, - - #[cfg(not(target_arch = "aarch64"))] - () => unimplemented!(), - } -} - -/// Invalidate all TLB entries. -#[inline(always)] -pub fn tlb_invalidate_all() { - unsafe { - asm!( - "dsb ishst - tlbi vmalle1is - dsb ish - isb" - ); - } -} - -/// Invalidate TLB entries that would be used to translate the specified address. -#[inline(always)] -pub fn tlb_invalidate(vaddr: VirtAddr) { - unsafe { - asm!( - "dsb ishst - tlbi vaae1is, $0 - dsb ish - isb" :: "r"(vaddr.as_u64() >> 12) - ); - } -} - -/// Invalidate all instruction caches in Inner Shareable domain to Point of Unification. -#[inline(always)] -pub fn flush_icache_all() { - unsafe { - asm!( - "ic ialluis - dsb ish - isb" - ); - } -} - -/// Address Translate. -#[inline(always)] -pub fn address_translate(vaddr: usize) -> usize { - let paddr: usize; - unsafe { - asm!("at S1E1R, $1; mrs $0, par_el1" : "=r"(paddr) : "r"(vaddr)); - } - paddr -} - -/// Read TTBRx_EL1 as PhysFrame -pub fn ttbr_el1_read(which: u8) -> PhysFrame { - let baddr = match which { - 0 => TTBR0_EL1.get_baddr(), - 1 => TTBR1_EL1.get_baddr(), - _ => 0, - }; - PhysFrame::containing_address(PhysAddr::new(baddr)) -} - -/// Write TTBRx_EL1 from PhysFrame -pub fn ttbr_el1_write(which: u8, frame: PhysFrame) { - let baddr = frame.start_address().as_u64(); - match which { - 0 => TTBR0_EL1.set_baddr(baddr), - 1 => TTBR1_EL1.set_baddr(baddr), - _ => {} - }; -} - -/// write TTBRx_EL1 from PhysFrame and ASID -pub fn ttbr_el1_write_asid(which: u8, asid: u16, frame: PhysFrame) { - let baddr = frame.start_address().as_u64(); - match which { - 0 => TTBR0_EL1.write(TTBR0_EL1::ASID.val(asid as u64) + TTBR0_EL1::BADDR.val(baddr >> 1)), - 1 => TTBR1_EL1.write(TTBR1_EL1::ASID.val(asid as u64) + TTBR1_EL1::BADDR.val(baddr >> 1)), - _ => {} - }; -} diff --git a/crate/aarch64/src/barrier.rs b/crate/aarch64/src/barrier.rs deleted file mode 100644 index 3211480..0000000 --- a/crate/aarch64/src/barrier.rs +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Copyright (c) 2018 by the author(s) - * - * ============================================================================= - * - * Licensed under either of - * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) - * - MIT License (http://opensource.org/licenses/MIT) - * at your option. - * - * ============================================================================= - * - * Author(s): - * - Andre Richter - */ - -// Borrow implementations from the pending upstream ACLE implementation until it -// is merged. Afterwards, we'll probably just reexport them, hoping that the API -// doesn't change. -// -// https://github.com/rust-lang-nursery/stdsimd/pull/557 - -mod sealed { - pub trait Dmb { - unsafe fn __dmb(&self); - } - - pub trait Dsb { - unsafe fn __dsb(&self); - } - - pub trait Isb { - unsafe fn __isb(&self); - } -} - -macro_rules! dmb_dsb { - ($A:ident) => { - impl sealed::Dmb for $A { - #[inline(always)] - unsafe fn __dmb(&self) { - asm!(concat!("DMB ", stringify!($A)) : : : "memory" : "volatile") - } - } - impl sealed::Dsb for $A { - #[inline(always)] - unsafe fn __dsb(&self) { - asm!(concat!("DSB ", stringify!($A)) : : : "memory" : "volatile") - } - } - }; -} - -pub struct SY; -pub struct ISH; -pub struct ISHST; - -dmb_dsb!(SY); -dmb_dsb!(ISH); -dmb_dsb!(ISHST); - -impl sealed::Isb for SY { - #[inline(always)] - unsafe fn __isb(&self) { - asm!("ISB SY" : : : "memory" : "volatile") - } -} - -#[inline(always)] -pub unsafe fn dmb(arg: A) -where - A: sealed::Dmb, -{ - arg.__dmb() -} - -#[inline(always)] -pub unsafe fn dsb(arg: A) -where - A: sealed::Dsb, -{ - arg.__dsb() -} - -#[inline(always)] -pub unsafe fn isb(arg: A) -where - A: sealed::Isb, -{ - arg.__isb() -} diff --git a/crate/aarch64/src/lib.rs b/crate/aarch64/src/lib.rs deleted file mode 100644 index bb8bf72..0000000 --- a/crate/aarch64/src/lib.rs +++ /dev/null @@ -1,29 +0,0 @@ -#![no_std] -//#![deny(warnings)] -#![feature(asm)] -#![feature(const_fn)] -#![feature(core_intrinsics)] -#![feature(try_from)] - -extern crate bare_metal; -#[macro_use] -extern crate register; -#[macro_use] -extern crate bitflags; -extern crate bit_field; -extern crate os_bootinfo; -extern crate usize_conversions; - -/// Provides the non-standard-width integer types `u2`–`u63`. -/// -/// We use these integer types in various APIs, for example `u9` for page tables indices. -pub extern crate ux; - -pub use addr::{align_down, align_up, PhysAddr, VirtAddr}; - -pub mod asm; -pub mod addr; -pub mod paging; -pub mod barrier; -pub mod regs; - diff --git a/crate/aarch64/src/paging/frame_alloc.rs b/crate/aarch64/src/paging/frame_alloc.rs deleted file mode 100644 index f3ea4c8..0000000 --- a/crate/aarch64/src/paging/frame_alloc.rs +++ /dev/null @@ -1,15 +0,0 @@ -//! Traits for abstracting away frame allocation and deallocation. - -use paging::{PageSize, PhysFrame}; - -/// A trait for types that can allocate a frame of memory. -pub trait FrameAllocator { - /// Allocate a frame of the appropriate size and return it if possible. - fn alloc(&mut self) -> Option>; -} - -/// A trait for types that can deallocate a frame of memory. -pub trait FrameDeallocator { - /// Deallocate the given frame of memory. - fn dealloc(&mut self, frame: PhysFrame); -} diff --git a/crate/aarch64/src/paging/memory_attribute.rs b/crate/aarch64/src/paging/memory_attribute.rs deleted file mode 100644 index 4a24071..0000000 --- a/crate/aarch64/src/paging/memory_attribute.rs +++ /dev/null @@ -1,64 +0,0 @@ -//!Memory region attributes (D4.5, page 2174) - -use super::{PageTableAttribute, MEMORY_ATTRIBUTE}; -use regs::*; - -pub trait MairType { - const INDEX: u64; - - #[inline] - fn config_value() -> u64; - - #[inline] - fn attr_value() -> PageTableAttribute; -} - -pub enum MairDevice {} -pub enum MairNormal {} -pub enum MairNormalNonCacheable {} - -impl MairType for MairDevice { - const INDEX: u64 = 0; - - #[inline] - fn config_value() -> u64 { - (MAIR_ATTR::Attr_HIGH::Device + MAIR_ATTR::Attr_LOW_DEVICE::Device_nGnRE).value - } - - #[inline] - fn attr_value() -> PageTableAttribute { - MEMORY_ATTRIBUTE::SH::OuterShareable + MEMORY_ATTRIBUTE::AttrIndx.val(Self::INDEX) - } -} - -impl MairType for MairNormal { - const INDEX: u64 = 1; - - #[inline] - fn config_value() -> u64 { - (MAIR_ATTR::Attr_HIGH::Memory_OuterWriteBack_NonTransient_ReadAlloc_WriteAlloc - + MAIR_ATTR::Attr_LOW_MEMORY::InnerWriteBack_NonTransient_ReadAlloc_WriteAlloc) - .value - } - - #[inline] - fn attr_value() -> PageTableAttribute { - MEMORY_ATTRIBUTE::SH::InnerShareable + MEMORY_ATTRIBUTE::AttrIndx.val(Self::INDEX) - } -} - -impl MairType for MairNormalNonCacheable { - const INDEX: u64 = 2; - - #[inline] - fn config_value() -> u64 { - (MAIR_ATTR::Attr_HIGH::Memory_OuterNonCacheable - + MAIR_ATTR::Attr_LOW_MEMORY::InnerNonCacheable) - .value - } - - #[inline] - fn attr_value() -> PageTableAttribute { - MEMORY_ATTRIBUTE::SH::OuterShareable + MEMORY_ATTRIBUTE::AttrIndx.val(Self::INDEX) - } -} diff --git a/crate/aarch64/src/paging/mod.rs b/crate/aarch64/src/paging/mod.rs deleted file mode 100644 index 552d6f6..0000000 --- a/crate/aarch64/src/paging/mod.rs +++ /dev/null @@ -1,549 +0,0 @@ -//! Abstractions for page tables and other paging related structures. -//! -//! Page tables translate virtual memory “pages” to physical memory “frames”. - -pub use self::frame_alloc::*; -pub use self::page_table::*; -#[cfg(target_arch = "aarch64")] -pub use self::recursive::*; - -use core::fmt; -use core::marker::PhantomData; -use core::ops::{Add, AddAssign, Sub, SubAssign}; -use os_bootinfo; -use ux::*; -use addr::{PhysAddr, VirtAddr}; - -mod frame_alloc; -mod page_table; -mod recursive; - -pub mod memory_attribute; - -/// Trait for abstracting over the three possible block/page sizes on aarch64, 4KiB, 2MiB, 1GiB. -pub trait PageSize: Copy + Eq + PartialOrd + Ord { - /// The page size in bytes. - const SIZE: u64; - - /// A string representation of the page size for debug output. - const SIZE_AS_DEBUG_STR: &'static str; -} - -/// This trait is implemented for 4KiB and 2MiB pages, but not for 1GiB pages. -pub trait NotGiantPageSize: PageSize {} - -/// A standard 4KiB page. -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] -pub enum Size4KiB {} - -/// A “huge” 2MiB page. -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] -pub enum Size2MiB {} - -/// A “giant” 1GiB page. -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] -pub enum Size1GiB {} - -impl PageSize for Size4KiB { - const SIZE: u64 = 4096; - const SIZE_AS_DEBUG_STR: &'static str = "4KiB"; -} - -impl NotGiantPageSize for Size4KiB {} - -impl PageSize for Size2MiB { - const SIZE: u64 = Size4KiB::SIZE * 512; - const SIZE_AS_DEBUG_STR: &'static str = "2MiB"; -} - -impl NotGiantPageSize for Size2MiB {} - -impl PageSize for Size1GiB { - const SIZE: u64 = Size2MiB::SIZE * 512; - const SIZE_AS_DEBUG_STR: &'static str = "1GiB"; -} - -/// A virtual memory page. -#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] -#[repr(C)] -pub struct Page { - start_address: VirtAddr, - size: PhantomData, -} - -impl Page { - /// Returns the page that starts at the given virtual address. - /// - /// Returns an error if the address is not correctly aligned (i.e. is not a valid page start). - pub fn from_start_address(address: VirtAddr) -> Result { - if !address.is_aligned(S::SIZE) { - return Err(()); - } - Ok(Page::containing_address(address)) - } - - /// Returns the page that contains the given virtual address. - pub fn containing_address(address: VirtAddr) -> Self { - Page { - start_address: address.align_down(S::SIZE), - size: PhantomData, - } - } - - /// Returns the start address of the page. - pub fn start_address(&self) -> VirtAddr { - self.start_address - } - - /// Returns the size the page (4KB, 2MB or 1GB). - pub const fn size(&self) -> u64 { - S::SIZE - } - - /// Returns the level 4 page table index of this page. - pub fn va_range_bits(&self) -> u16 { - self.start_address().va_range_bits() - } - - /// Returns the level 4 page table index of this page. - pub fn p4_index(&self) -> u9 { - self.start_address().p4_index() - } - - /// Returns the level 3 page table index of this page. - pub fn p3_index(&self) -> u9 { - self.start_address().p3_index() - } - - /// Returns a range of pages, exclusive `end`. - pub fn range(start: Self, end: Self) -> PageRange { - PageRange { start, end } - } - - /// Returns a range of pages, inclusive `end`. - pub fn range_inclusive(start: Self, end: Self) -> PageRangeInclusive { - PageRangeInclusive { start, end } - } - - pub fn of_addr(address: usize) -> Self { - Self::containing_address(VirtAddr::new(address as u64)) - } - - pub fn range_of(begin: usize, end: usize) -> PageRange { - Self::range(Page::of_addr(begin), Page::of_addr(end - 1) + 1) - } -} - -impl Page { - /// Returns the level 2 page table index of this page. - pub fn p2_index(&self) -> u9 { - self.start_address().p2_index() - } -} - -impl Page { - /// Returns the 1GiB memory page with the specified page table indices. - pub fn from_page_table_indices_1gib(p4_index: u9, p3_index: u9) -> Self { - use bit_field::BitField; - - let mut addr = 0; - addr.set_bits(39..48, u64::from(p4_index)); - addr.set_bits(30..39, u64::from(p3_index)); - Page::containing_address(VirtAddr::new(addr)) - } -} - -impl Page { - /// Returns the 2MiB memory page with the specified page table indices. - pub fn from_page_table_indices_2mib(p4_index: u9, p3_index: u9, p2_index: u9) -> Self { - use bit_field::BitField; - - let mut addr = 0; - addr.set_bits(39..48, u64::from(p4_index)); - addr.set_bits(30..39, u64::from(p3_index)); - addr.set_bits(21..30, u64::from(p2_index)); - Page::containing_address(VirtAddr::new(addr)) - } -} - -impl Page { - /// Returns the 4KiB memory page with the specified page table indices. - pub fn from_page_table_indices(p4_index: u9, p3_index: u9, p2_index: u9, p1_index: u9) -> Self { - use bit_field::BitField; - - let mut addr = 0; - addr.set_bits(39..48, u64::from(p4_index)); - addr.set_bits(30..39, u64::from(p3_index)); - addr.set_bits(21..30, u64::from(p2_index)); - addr.set_bits(12..21, u64::from(p1_index)); - Page::containing_address(VirtAddr::new(addr)) - } - - /// Returns the level 1 page table index of this page. - pub fn p1_index(&self) -> u9 { - self.start_address().p1_index() - } -} - -impl fmt::Debug for Page { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - f.write_fmt(format_args!( - "Page[{}]({:#x})", - S::SIZE_AS_DEBUG_STR, - self.start_address().as_u64() - )) - } -} - -impl Add for Page { - type Output = Self; - fn add(self, rhs: u64) -> Self::Output { - Page::containing_address(self.start_address() + rhs * u64::from(S::SIZE)) - } -} - -impl AddAssign for Page { - fn add_assign(&mut self, rhs: u64) { - *self = self.clone() + rhs; - } -} - -impl Sub for Page { - type Output = Self; - fn sub(self, rhs: u64) -> Self::Output { - Page::containing_address(self.start_address() - rhs * u64::from(S::SIZE)) - } -} - -impl SubAssign for Page { - fn sub_assign(&mut self, rhs: u64) { - *self = self.clone() - rhs; - } -} - -impl Sub for Page { - type Output = u64; - fn sub(self, rhs: Self) -> Self::Output { - (self.start_address - rhs.start_address) / S::SIZE - } -} - -/// A range of pages with exclusive upper bound. -#[derive(Clone, Copy, PartialEq, Eq)] -#[repr(C)] -pub struct PageRange { - /// The start of the range, inclusive. - pub start: Page, - /// The end of the range, exclusive. - pub end: Page, -} - -impl PageRange { - /// Returns wether this range contains no pages. - pub fn is_empty(&self) -> bool { - !(self.start < self.end) - } -} - -impl Iterator for PageRange { - type Item = Page; - - fn next(&mut self) -> Option { - if self.start < self.end { - let page = self.start.clone(); - self.start += 1; - Some(page) - } else { - None - } - } -} - -impl PageRange { - /// Converts the range of 2MiB pages to a range of 4KiB pages. - pub fn as_4kib_page_range(self) -> PageRange { - PageRange { - start: Page::containing_address(self.start.start_address()), - end: Page::containing_address(self.end.start_address()), - } - } -} - -impl fmt::Debug for PageRange { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - f.debug_struct("PageRange") - .field("start", &self.start) - .field("end", &self.end) - .finish() - } -} - -/// A range of pages with inclusive upper bound. -#[derive(Clone, Copy, PartialEq, Eq)] -#[repr(C)] -pub struct PageRangeInclusive { - /// The start of the range, inclusive. - pub start: Page, - /// The end of the range, inclusive. - pub end: Page, -} - -impl PageRangeInclusive { - /// Returns wether this range contains no pages. - pub fn is_empty(&self) -> bool { - !(self.start <= self.end) - } -} - -impl Iterator for PageRangeInclusive { - type Item = Page; - - fn next(&mut self) -> Option { - if self.start <= self.end { - let page = self.start.clone(); - self.start += 1; - Some(page) - } else { - None - } - } -} - -impl fmt::Debug for PageRangeInclusive { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - f.debug_struct("PageRangeInclusive") - .field("start", &self.start) - .field("end", &self.end) - .finish() - } -} - -/// A physical memory frame. -#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] -#[repr(C)] -pub struct PhysFrame { - start_address: PhysAddr, - size: PhantomData, -} - -impl PhysFrame { - /// Returns the frame that starts at the given virtual address. - /// - /// Returns an error if the address is not correctly aligned (i.e. is not a valid frame start). - pub fn from_start_address(address: PhysAddr) -> Result { - if !address.is_aligned(S::SIZE) { - return Err(()); - } - Ok(PhysFrame::containing_address(address)) - } - - /// Returns the frame that contains the given physical address. - pub fn containing_address(address: PhysAddr) -> Self { - PhysFrame { - start_address: address.align_down(S::SIZE), - size: PhantomData, - } - } - - /// Returns the start address of the frame. - pub fn start_address(&self) -> PhysAddr { - self.start_address - } - - /// Returns the size the frame (4KB, 2MB or 1GB). - pub fn size(&self) -> u64 { - S::SIZE - } - - /// Returns a range of frames, exclusive `end`. - pub fn range(start: PhysFrame, end: PhysFrame) -> PhysFrameRange { - PhysFrameRange { start, end } - } - - /// Returns a range of frames, inclusive `end`. - pub fn range_inclusive(start: PhysFrame, end: PhysFrame) -> PhysFrameRangeInclusive { - PhysFrameRangeInclusive { start, end } - } - - pub fn of_addr(address: usize) -> Self { - Self::containing_address(PhysAddr::new(address as u64)) - } - - pub fn range_of(begin: usize, end: usize) -> PhysFrameRange { - Self::range(Self::of_addr(begin), Self::of_addr(end - 1) + 1) - } -} - -impl fmt::Debug for PhysFrame { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - f.write_fmt(format_args!( - "PhysFrame[{}]({:#x})", - S::SIZE_AS_DEBUG_STR, - self.start_address().as_u64() - )) - } -} - -impl Add for PhysFrame { - type Output = Self; - fn add(self, rhs: u64) -> Self::Output { - PhysFrame::containing_address(self.start_address() + rhs * u64::from(S::SIZE)) - } -} - -impl AddAssign for PhysFrame { - fn add_assign(&mut self, rhs: u64) { - *self = self.clone() + rhs; - } -} - -impl Sub for PhysFrame { - type Output = Self; - fn sub(self, rhs: u64) -> Self::Output { - PhysFrame::containing_address(self.start_address() - rhs * u64::from(S::SIZE)) - } -} - -impl SubAssign for PhysFrame { - fn sub_assign(&mut self, rhs: u64) { - *self = self.clone() - rhs; - } -} - -impl Sub> for PhysFrame { - type Output = u64; - fn sub(self, rhs: PhysFrame) -> Self::Output { - (self.start_address - rhs.start_address) / S::SIZE - } -} - -/// An range of physical memory frames, exclusive the upper bound. -#[derive(Clone, Copy, PartialEq, Eq)] -#[repr(C)] -pub struct PhysFrameRange { - /// The start of the range, inclusive. - pub start: PhysFrame, - /// The end of the range, exclusive. - pub end: PhysFrame, -} - -impl PhysFrameRange { - /// Returns whether the range contains no frames. - pub fn is_empty(&self) -> bool { - !(self.start < self.end) - } -} - -impl Iterator for PhysFrameRange { - type Item = PhysFrame; - - fn next(&mut self) -> Option { - if self.start < self.end { - let frame = self.start.clone(); - self.start += 1; - Some(frame) - } else { - None - } - } -} - -impl From for PhysFrameRange { - fn from(range: os_bootinfo::FrameRange) -> Self { - PhysFrameRange { - start: PhysFrame::from_start_address(PhysAddr::new(range.start_addr())).unwrap(), - end: PhysFrame::from_start_address(PhysAddr::new(range.end_addr())).unwrap(), - } - } -} - -impl Into for PhysFrameRange { - fn into(self) -> os_bootinfo::FrameRange { - os_bootinfo::FrameRange::new( - self.start.start_address().as_u64(), - self.end.start_address().as_u64(), - ) - } -} - -impl fmt::Debug for PhysFrameRange { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - f.debug_struct("PhysFrameRange") - .field("start", &self.start) - .field("end", &self.end) - .finish() - } -} - -/// An range of physical memory frames, inclusive the upper bound. -#[derive(Clone, Copy, PartialEq, Eq)] -#[repr(C)] -pub struct PhysFrameRangeInclusive { - /// The start of the range, inclusive. - pub start: PhysFrame, - /// The start of the range, exclusive. - pub end: PhysFrame, -} - -impl PhysFrameRangeInclusive { - /// Returns whether the range contains no frames. - pub fn is_empty(&self) -> bool { - !(self.start <= self.end) - } -} - -impl Iterator for PhysFrameRangeInclusive { - type Item = PhysFrame; - - fn next(&mut self) -> Option { - if self.start <= self.end { - let frame = self.start.clone(); - self.start += 1; - Some(frame) - } else { - None - } - } -} - -impl fmt::Debug for PhysFrameRangeInclusive { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - f.debug_struct("PhysFrameRangeInclusive") - .field("start", &self.start) - .field("end", &self.end) - .finish() - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - pub fn test_page_ranges() { - let page_size = Size4KiB::SIZE; - let number = 1000; - - let start_addr = VirtAddr::new(0xdeadbeaf); - let start: Page = Page::containing_address(start_addr); - let end = start.clone() + number; - - let mut range = Page::range(start.clone(), end.clone()); - for i in 0..number { - assert_eq!( - range.next(), - Some(Page::containing_address(start_addr + page_size * i)) - ); - } - assert_eq!(range.next(), None); - - let mut range_inclusive = Page::range_inclusive(start, end); - for i in 0..=number { - assert_eq!( - range_inclusive.next(), - Some(Page::containing_address(start_addr + page_size * i)) - ); - } - assert_eq!(range_inclusive.next(), None); - } -} diff --git a/crate/aarch64/src/paging/page_table.rs b/crate/aarch64/src/paging/page_table.rs deleted file mode 100644 index 6d33fa6..0000000 --- a/crate/aarch64/src/paging/page_table.rs +++ /dev/null @@ -1,267 +0,0 @@ -use core::fmt; -use core::ops::{Index, IndexMut}; - -use super::{PhysFrame, PageSize}; -use addr::PhysAddr; - -use usize_conversions::usize_from; -use ux::*; - -use register::FieldValue; -use register::cpu::RegisterReadWrite; - -/// Memory attribute fields mask -const MEMORY_ATTR_MASK: u64 = (MEMORY_ATTRIBUTE::SH.mask << MEMORY_ATTRIBUTE::SH.shift) - | (MEMORY_ATTRIBUTE::AttrIndx.mask << MEMORY_ATTRIBUTE::AttrIndx.shift); -/// Output address mask -const ADDR_MASK: u64 = 0x0000_ffff_ffff_f000; -/// Other flags mask -const FLAGS_MASK: u64 = !(MEMORY_ATTR_MASK | ADDR_MASK); - -/// Memory attribute fields -pub type PageTableAttribute = FieldValue; - -/// The error returned by the `PageTableEntry::frame` method. -#[derive(Debug, Clone, Copy, PartialEq)] -pub enum FrameError { - /// The entry does not have the `PRESENT` flag set, so it isn't currently mapped to a frame. - FrameNotPresent, - /// The entry does have the `HUGE_PAGE` flag set. The `frame` method has a standard 4KiB frame - /// as return type, so a huge frame can't be returned. - HugeFrame, -} - -/// A 64-bit page table entry. -#[derive(Clone)] -#[repr(transparent)] -pub struct PageTableEntry { - pub entry: u64, -} - -impl RegisterReadWrite for PageTableEntry { - #[inline] - fn get(&self) -> u64 { - self.entry - } - - #[inline] - fn set(&self, value: u64) { - unsafe { *(&self.entry as *const u64 as *mut u64) = value } - } -} - -impl PageTableEntry { - /// Returns whether this entry is zero. - #[inline] - pub fn is_unused(&self) -> bool { - self.entry == 0 - } - - /// Sets this entry to zero. - #[inline] - pub fn set_unused(&mut self) { - self.entry = 0; - } - - /// Returns the flags of this entry. - #[inline] - pub fn flags(&self) -> PageTableFlags { - PageTableFlags::from_bits_truncate(self.entry) - } - - /// Returns the physical address mapped by this entry, might be zero. - #[inline] - pub fn addr(&self) -> PhysAddr { - PhysAddr::new(self.entry & ADDR_MASK) - } - - /// Returns the memory attribute fields of this entry. - #[inline] - pub fn attr(&self) -> PageTableAttribute { - PageTableAttribute::new(MEMORY_ATTR_MASK, 0, self.entry & MEMORY_ATTR_MASK) - } - - /// Returns the physical frame mapped by this entry. - /// - /// Returns the following errors: - /// - /// - `FrameError::FrameNotPresent` if the entry doesn't have the `PRESENT` flag set. - /// - `FrameError::HugeFrame` if the entry has the `HUGE_PAGE` flag set (for huge pages the - /// `addr` function must be used) - pub fn frame(&self) -> Result { - if !self.flags().contains(PageTableFlags::VALID) { - Err(FrameError::FrameNotPresent) - } else if !self.flags().contains(PageTableFlags::TABLE_OR_PAGE) { - // is a huge page (block) - Err(FrameError::HugeFrame) - } else { - Ok(PhysFrame::containing_address(self.addr())) - } - } - - /// Map the entry to the specified physical frame with the specified flags and memory attribute. - pub fn set_frame(&mut self, frame: PhysFrame, flags: PageTableFlags, attr: PageTableAttribute) { - // is not a block - assert!(flags.contains(PageTableFlags::TABLE_OR_PAGE)); - self.set(frame.start_address().as_u64() | flags.bits() | attr.value); - } - - /// The descriptor gives the base address of a block of memory, and the attributes for that - /// memory region. - pub fn set_block(&mut self, addr: PhysAddr, flags: PageTableFlags, attr: PageTableAttribute) { - // is a block - assert!(!flags.contains(PageTableFlags::TABLE_OR_PAGE)); - self.set(addr.align_down(S::SIZE).as_u64() | flags.bits() | attr.value); - } - - /// Map the entry to the specified physical address with the specified flags. - pub fn modify_addr(&mut self, addr: PhysAddr) { - self.entry = (self.entry & !ADDR_MASK) | addr.as_u64(); - } - - /// Sets the flags of this entry. - pub fn modify_flags(&mut self, flags: PageTableFlags) { - self.entry = (self.entry & !FLAGS_MASK) | flags.bits(); - } - - /// Sets the flags of this entry. - pub fn modify_attr(&mut self, attr: PageTableAttribute) { - self.entry = (self.entry & !MEMORY_ATTR_MASK) | attr.value; - } -} - -impl fmt::Debug for PageTableEntry { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - let mut f = f.debug_struct("PageTableEntry"); - f.field("value", &self.entry); - f.field("addr", &self.addr()); - f.field("flags", &self.flags()); - f.field("attr", &self.attr().value); - f.finish() - } -} - -register_bitfields! {u64, - // Memory attribute fields in the VMSAv8-64 translation table format descriptors (Page 2148~2152) - MEMORY_ATTRIBUTE [ - /// Shareability field - SH OFFSET(8) NUMBITS(2) [ - NonShareable = 0b00, - OuterShareable = 0b10, - InnerShareable = 0b11 - ], - - /// Memory attributes index into the MAIR_EL1 register - AttrIndx OFFSET(2) NUMBITS(3) [] - ] -} - -bitflags! { - /// Possible flags for a page table entry. - pub struct PageTableFlags: u64 { - /// identifies whether the descriptor is valid - const VALID = 1 << 0; - /// the descriptor type - /// 0, Block - /// 1, Table/Page - const TABLE_OR_PAGE = 1 << 1; - /// Access permission: accessable at EL0 - const AP_EL0 = 1 << 6; - /// Access permission: read-only - const AP_RO = 1 << 7; - /// Access flag - const AF = 1 << 10; - /// not global bit - const nG = 1 << 11; - /// Dirty Bit Modifier - const DBM = 1 << 51; - - /// A hint bit indicating that the translation table entry is one of a contiguous set or - /// entries - const Contiguous = 1 << 52; - /// Privileged Execute-never - const PXN = 1 << 53; - /// Execute-never/Unprivileged execute-never - const XN = 1 << 54; - - /// Software Dirty Bit Modifier - const WRITE = 1 << 51; - /// Software dirty bit - const DIRTY = 1 << 55; - /// Software swapped bit - const SWAPPED = 1 << 56; - /// Software writable shared bit for COW - const WRITABLE_SHARED = 1 << 57; - /// Software readonly shared bit for COW - const READONLY_SHARED = 1 << 58; - - /// Privileged Execute-never for table descriptors - const PXNTable = 1 << 59; - /// Execute-never/Unprivileged execute-never for table descriptors - const XNTable = 1 << 60; - } -} - -impl Default for PageTableFlags { - #[inline] - fn default() -> Self { - Self::VALID | Self::TABLE_OR_PAGE | Self::AF | Self::WRITE | Self::PXN | Self::XN - } -} - -/// The number of entries in a page table. -const ENTRY_COUNT: usize = 512; - -/// Represents a page table. -/// -/// Always page-sized. -/// -/// This struct implements the `Index` and `IndexMut` traits, so the entries can be accessed -/// through index operations. For example, `page_table[15]` returns the 15th page table entry. -#[repr(transparent)] -pub struct PageTable { - entries: [PageTableEntry; ENTRY_COUNT], -} - -impl PageTable { - /// Clears all entries. - pub fn zero(&mut self) { - for entry in self.entries.iter_mut() { - entry.set_unused(); - } - } -} - -impl Index for PageTable { - type Output = PageTableEntry; - - fn index(&self, index: usize) -> &Self::Output { - &self.entries[index] - } -} - -impl IndexMut for PageTable { - fn index_mut(&mut self, index: usize) -> &mut Self::Output { - &mut self.entries[index] - } -} - -impl Index for PageTable { - type Output = PageTableEntry; - - fn index(&self, index: u9) -> &Self::Output { - &self.entries[usize_from(u16::from(index))] - } -} - -impl IndexMut for PageTable { - fn index_mut(&mut self, index: u9) -> &mut Self::Output { - &mut self.entries[usize_from(u16::from(index))] - } -} - -impl fmt::Debug for PageTable { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - self.entries[..].fmt(f) - } -} diff --git a/crate/aarch64/src/paging/recursive.rs b/crate/aarch64/src/paging/recursive.rs deleted file mode 100644 index a1633ff..0000000 --- a/crate/aarch64/src/paging/recursive.rs +++ /dev/null @@ -1,425 +0,0 @@ -#![cfg(target_arch = "aarch64")] - -use paging::{ - frame_alloc::FrameAllocator, - page_table::{FrameError, PageTable, PageTableEntry, PageTableFlags}, - NotGiantPageSize, Page, PageSize, PhysFrame, Size4KiB, -}; -use paging::{page_table::PageTableFlags as Flags, PageTableAttribute, memory_attribute::*}; -use asm::{ttbr_el1_read, tlb_invalidate}; -use barrier; -use ux::u9; -use addr::{PhysAddr, VirtAddr}; - -/// This type represents a page whose mapping has changed in the page table. -/// -/// The old mapping might be still cached in the translation lookaside buffer (TLB), so it needs -/// to be flushed from the TLB before it's accessed. This type is returned from function that -/// change the mapping of a page to ensure that the TLB flush is not forgotten. -#[derive(Debug)] -#[must_use = "Page Table changes must be flushed or ignored."] -pub struct MapperFlush(Page); - -impl MapperFlush { - /// Create a new flush promise - fn new(page: Page) -> Self { - MapperFlush(page) - } - - /// Flush the page from the TLB to ensure that the newest mapping is used. - pub fn flush(self) { - tlb_invalidate(self.0.start_address()); - } - - /// Don't flush the TLB and silence the “must be used” warning. - pub fn ignore(self) {} -} - -/// A trait for common page table operations. -pub trait Mapper { - /// Creates a new mapping in the page table. - /// - /// This function might need additional physical frames to create new page tables. These - /// frames are allocated from the `allocator` argument. At most three frames are required. - fn map_to( - &mut self, - page: Page, - frame: PhysFrame, - flags: PageTableFlags, - attr: PageTableAttribute, - allocator: &mut A, - ) -> Result, MapToError> - where - A: FrameAllocator; - - /// Removes a mapping from the page table and returns the frame that used to be mapped. - /// - /// Note that no page tables or pages are deallocated. - fn unmap(&mut self, page: Page) -> Result<(PhysFrame, MapperFlush), UnmapError>; - - /// Updates the flags of an existing mapping. - fn update_flags( - &mut self, - page: Page, - flags: PageTableFlags, - ) -> Result, FlagUpdateError>; - - /// Return the frame that the specified page is mapped to. - fn translate_page(&self, page: Page) -> Option>; - - /// Maps the given frame to the virtual page with the same address. - fn identity_map( - &mut self, - frame: PhysFrame, - flags: PageTableFlags, - attr: PageTableAttribute, - allocator: &mut A, - ) -> Result, MapToError> - where - A: FrameAllocator, - S: PageSize, - Self: Mapper, - { - let page = Page::containing_address(VirtAddr::new(frame.start_address().as_u64())); - self.map_to(page, frame, flags, attr, allocator) - } -} - -/// A recursive page table is a last level page table with an entry mapped to the table itself. -/// -/// This recursive mapping allows accessing all page tables in the hierarchy: -/// -/// - To access the level 4 page table, we “loop“ (i.e. follow the recursively mapped entry) four -/// times. -/// - To access a level 3 page table, we “loop” three times and then use the level 4 index. -/// - To access a level 2 page table, we “loop” two times, then use the level 4 index, then the -/// level 3 index. -/// - To access a level 1 page table, we “loop” once, then use the level 4 index, then the -/// level 3 index, then the level 2 index. -/// -/// This struct implements the `Mapper` trait. -#[derive(Debug)] -pub struct RecursivePageTable<'a> { - p4: &'a mut PageTable, - recursive_index: u9, -} - -/// An error indicating that the given page table is not recursively mapped. -/// -/// Returned from `RecursivePageTable::new`. -#[derive(Debug)] -pub struct NotRecursivelyMapped; - -/// This error is returned from `map_to` and similar methods. -#[derive(Debug)] -pub enum MapToError { - /// An additional frame was needed for the mapping process, but the frame allocator - /// returned `None`. - FrameAllocationFailed, - /// An upper level page table entry has the `HUGE_PAGE` flag set, which means that the - /// given page is part of an already mapped huge page. - ParentEntryHugePage, - /// The given page is already mapped to a physical frame. - PageAlreadyMapped, -} - -/// An error indicating that an `unmap` call failed. -#[derive(Debug)] -pub enum UnmapError { - /// An upper level page table entry has the `HUGE_PAGE` flag set, which means that the - /// given page is part of a huge page and can't be freed individually. - ParentEntryHugePage, - /// The given page is not mapped to a physical frame. - PageNotMapped, - /// The page table entry for the given page points to an invalid physical address. - InvalidFrameAddress(PhysAddr), -} - -/// An error indicating that an `update_flags` call failed. -#[derive(Debug)] -pub enum FlagUpdateError { - /// The given page is not mapped to a physical frame. - PageNotMapped, -} - -impl<'a> RecursivePageTable<'a> { - /// Creates a new RecursivePageTable from the passed level 4 PageTable. - /// - /// The page table must be recursively mapped, that means: - /// - /// - The page table must have one recursive entry, i.e. an entry that points to the table - /// itself. - /// - The reference must use that “loop”, i.e. be of the form `0o_xxx_xxx_xxx_xxx_0000` - /// where `xxx` is the recursive entry. - /// - The page table must be active, i.e. the CR3 register must contain its physical address. - /// - /// Otherwise `Err(NotRecursivelyMapped)` is returned. - pub fn new(table: &'a mut PageTable) -> Result { - let page = Page::containing_address(VirtAddr::new(table as *const _ as u64)); - let recursive_index = page.p4_index(); - - if page.p3_index() != recursive_index - || page.p2_index() != recursive_index - || page.p1_index() != recursive_index - { - return Err(NotRecursivelyMapped); - } - if Ok(ttbr_el1_read(page.start_address().va_range().unwrap() as u8)) != - table[recursive_index].frame() - { - return Err(NotRecursivelyMapped); - } - - Ok(RecursivePageTable { - p4: table, - recursive_index, - }) - } - - /// Creates a new RecursivePageTable without performing any checks. - /// - /// The `recursive_index` parameter must be the index of the recursively mapped entry. - pub unsafe fn new_unchecked(table: &'a mut PageTable, recursive_index: u9) -> Self { - RecursivePageTable { - p4: table, - recursive_index, - } - } - - /// Internal helper function to create the page table of the next level if needed. - /// - /// If the passed entry is unused, a new frame is allocated from the given allocator, zeroed, - /// and the entry is updated to that address. If the passed entry is already mapped, the next - /// table is returned directly. - /// - /// The `next_page_table` page must be the page of the next page table in the hierarchy. - /// - /// Returns `MapToError::FrameAllocationFailed` if the entry is unused and the allocator - /// returned `None`. Returns `MapToError::ParentEntryHugePage` if the `HUGE_PAGE` flag is set - /// in the passed entry. - unsafe fn create_next_table<'b, A>( - entry: &'b mut PageTableEntry, - next_table_page: Page, - allocator: &mut A, - ) -> Result<&'b mut PageTable, MapToError> - where - A: FrameAllocator, - { - /// This inner function is used to limit the scope of `unsafe`. - /// - /// This is a safe function, so we need to use `unsafe` blocks when we do something unsafe. - fn inner<'b, A>( - entry: &'b mut PageTableEntry, - next_table_page: Page, - allocator: &mut A, - ) -> Result<&'b mut PageTable, MapToError> - where - A: FrameAllocator, - { - let created; - - if entry.is_unused() { - if let Some(frame) = allocator.alloc() { - entry.set_frame(frame, Flags::default(), MairNormal::attr_value()); - created = true; - } else { - return Err(MapToError::FrameAllocationFailed); - } - } else { - created = false; - } - // is a huge page (block) - if !entry.flags().contains(Flags::TABLE_OR_PAGE) { - return Err(MapToError::ParentEntryHugePage); - } - - let page_table_ptr = next_table_page.start_address().as_mut_ptr(); - let page_table: &mut PageTable = unsafe { &mut *(page_table_ptr) }; - if created { - unsafe { barrier::dsb(barrier::ISHST); } - page_table.zero(); - } - Ok(page_table) - } - - inner(entry, next_table_page, allocator) - } - - pub fn p3_ptr(&self, page: Page) -> *mut PageTable { - self.p3_page(page).start_address().as_mut_ptr() - } - - pub fn p2_ptr(&self, page: Page) -> *mut PageTable { - self.p2_page(page).start_address().as_mut_ptr() - } - - pub fn p1_ptr(&self, page: Page) -> *mut PageTable { - self.p1_page(page).start_address().as_mut_ptr() - } - - fn p3_page(&self, page: Page) -> Page { - Page::from_page_table_indices( - self.recursive_index, - self.recursive_index, - self.recursive_index, - page.p4_index(), - ) - } - - fn p2_page(&self, page: Page) -> Page { - Page::from_page_table_indices( - self.recursive_index, - self.recursive_index, - page.p4_index(), - page.p3_index(), - ) - } - - fn p1_page(&self, page: Page) -> Page { - Page::from_page_table_indices( - self.recursive_index, - page.p4_index(), - page.p3_index(), - page.p2_index(), - ) - } -} - -impl<'a> Mapper for RecursivePageTable<'a> { - fn map_to( - &mut self, - page: Page, - frame: PhysFrame, - flags: PageTableFlags, - attr: PageTableAttribute, - allocator: &mut A, - ) -> Result, MapToError> - where - A: FrameAllocator, - { - let self_mut = unsafe { &mut *(self as *const _ as *mut Self) }; - let p4 = &mut self_mut.p4; - - let p3_page = self.p3_page(page); - let p3 = unsafe { Self::create_next_table(&mut p4[page.p4_index()], p3_page, allocator)? }; - - let p2_page = self.p2_page(page); - let p2 = unsafe { Self::create_next_table(&mut p3[page.p3_index()], p2_page, allocator)? }; - - let p1_page = self.p1_page(page); - let p1 = unsafe { Self::create_next_table(&mut p2[page.p2_index()], p1_page, allocator)? }; - - if !p1[page.p1_index()].is_unused() { - return Err(MapToError::PageAlreadyMapped); - } - p1[page.p1_index()].set_frame(frame, flags, attr); - - Ok(MapperFlush::new(page)) - } - - fn unmap( - &mut self, - page: Page, - ) -> Result<(PhysFrame, MapperFlush), UnmapError> { - let self_mut = unsafe { &mut *(self as *const _ as *mut Self) }; - let p4 = &mut self_mut.p4; - - let p4_entry = &p4[page.p4_index()]; - p4_entry.frame().map_err(|err| match err { - FrameError::FrameNotPresent => UnmapError::PageNotMapped, - FrameError::HugeFrame => UnmapError::ParentEntryHugePage, - })?; - - let p3 = unsafe { &mut *(self.p3_ptr(page)) }; - let p3_entry = &p3[page.p3_index()]; - p3_entry.frame().map_err(|err| match err { - FrameError::FrameNotPresent => UnmapError::PageNotMapped, - FrameError::HugeFrame => UnmapError::ParentEntryHugePage, - })?; - - let p2 = unsafe { &mut *(self.p2_ptr(page)) }; - let p2_entry = &p2[page.p2_index()]; - p2_entry.frame().map_err(|err| match err { - FrameError::FrameNotPresent => UnmapError::PageNotMapped, - FrameError::HugeFrame => UnmapError::ParentEntryHugePage, - })?; - - let p1 = unsafe { &mut *(self.p1_ptr(page)) }; - let p1_entry = &mut p1[page.p1_index()]; - - let frame = p1_entry.frame().map_err(|err| match err { - FrameError::FrameNotPresent => UnmapError::PageNotMapped, - FrameError::HugeFrame => UnmapError::ParentEntryHugePage, - })?; - - p1_entry.set_unused(); - Ok((frame, MapperFlush::new(page))) - } - - fn update_flags( - &mut self, - page: Page, - flags: PageTableFlags, - ) -> Result, FlagUpdateError> { - let self_mut = unsafe { &mut *(self as *const _ as *mut Self) }; - let p4 = &mut self_mut.p4; - - if p4[page.p4_index()].is_unused() { - return Err(FlagUpdateError::PageNotMapped); - } - - let p3 = unsafe { &mut *(self.p3_ptr(page)) }; - - if p3[page.p3_index()].is_unused() { - return Err(FlagUpdateError::PageNotMapped); - } - - let p2 = unsafe { &mut *(self.p2_ptr(page)) }; - - if p2[page.p2_index()].is_unused() { - return Err(FlagUpdateError::PageNotMapped); - } - - let p1 = unsafe { &mut *(self.p1_ptr(page)) }; - - if p1[page.p1_index()].is_unused() { - return Err(FlagUpdateError::PageNotMapped); - } - - p1[page.p1_index()].modify_flags(flags); - - Ok(MapperFlush::new(page)) - } - - fn translate_page(&self, page: Page) -> Option> { - let self_mut = unsafe { &mut *(self as *const _ as *mut Self) }; - let p4 = &mut self_mut.p4; - - if p4[page.p4_index()].is_unused() { - return None; - } - - let p3 = unsafe { &*(self.p3_ptr(page)) }; - let p3_entry = &p3[page.p3_index()]; - - if p3_entry.is_unused() { - return None; - } - - let p2 = unsafe { &*(self.p2_ptr(page)) }; - let p2_entry = &p2[page.p2_index()]; - - if p2_entry.is_unused() { - return None; - } - - let p1 = unsafe { &*(self.p1_ptr(page)) }; - let p1_entry = &p1[page.p1_index()]; - - if p1_entry.is_unused() { - return None; - } - - PhysFrame::from_start_address(p1_entry.addr()).ok() - } -} diff --git a/crate/aarch64/src/regs/cntfrq_el0.rs b/crate/aarch64/src/regs/cntfrq_el0.rs deleted file mode 100644 index df56ac9..0000000 --- a/crate/aarch64/src/regs/cntfrq_el0.rs +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2018 by the author(s) - * - * ============================================================================= - * - * Licensed under either of - * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) - * - MIT License (http://opensource.org/licenses/MIT) - * at your option. - * - * ============================================================================= - * - * Author(s): - * - Andre Richter - */ - -//! Counter-timer Frequency register - EL0 -//! -//! This register is provided so that software can discover the frequency of the -//! system counter. It must be programmed with this value as part of system -//! initialization. The value of the register is not interpreted by hardware. - -use register::cpu::RegisterReadOnly; - -pub struct Reg; - -impl RegisterReadOnly for Reg { - sys_coproc_read_raw!(u32, "CNTFRQ_EL0"); -} - -pub static CNTFRQ_EL0: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/cnthctl_el2.rs b/crate/aarch64/src/regs/cnthctl_el2.rs deleted file mode 100644 index f5e3c2c..0000000 --- a/crate/aarch64/src/regs/cnthctl_el2.rs +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright (c) 2018 by the author(s) - * - * ============================================================================= - * - * Licensed under either of - * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) - * - MIT License (http://opensource.org/licenses/MIT) - * at your option. - * - * ============================================================================= - * - * Author(s): - * - Andre Richter - */ - -//! Counter-timer Hypervisor Control register - EL2 -//! -//! Controls the generation of an event stream from the physical counter, and -//! access from Non-secure EL1 to the physical counter and the Non-secure EL1 -//! physical timer. - -use register::cpu::RegisterReadWrite; - -// When HCR_EL2.E2H == 0: -// TODO: Figure out how we can differentiate depending on HCR_EL2.E2H state -// -// For now, implement the HCR_EL2.E2H == 0 version -register_bitfields! {u32, - CNTHCTL_EL2 [ - /// Traps Non-secure EL0 and EL1 accesses to the physical timer - /// registers to EL2. - /// - /// 0 From AArch64 state: Non-secure EL0 and EL1 accesses to the - /// CNTP_CTL_EL0, CNTP_CVAL_EL0, and CNTP_TVAL_EL0 are trapped to EL2, - /// unless it is trapped by CNTKCTL_EL1.EL0PTEN. - /// - /// From AArch32 state: Non-secure EL0 and EL1 accesses to the - /// CNTP_CTL, CNTP_CVAL, and CNTP_TVAL are trapped to EL2, unless it - /// is trapped by CNTKCTL_EL1.EL0PTEN or CNTKCTL.PL0PTEN. - /// - /// 1 This control does not cause any instructions to be trapped. - /// - /// If EL3 is implemented and EL2 is not implemented, behavior is as if - /// this bit is 1 other than for the purpose of a direct read. - EL1PCEN OFFSET(1) NUMBITS(1) [], - - /// Traps Non-secure EL0 and EL1 accesses to the physical counter - /// register to EL2. - /// - /// 0 From AArch64 state: Non-secure EL0 and EL1 accesses to the - /// CNTPCT_EL0 are trapped to EL2, unless it is trapped by - /// CNTKCTL_EL1.EL0PCTEN. - /// - /// From AArch32 state: Non-secure EL0 and EL1 accesses to the CNTPCT - /// are trapped to EL2, unless it is trapped by CNTKCTL_EL1.EL0PCTEN - /// or CNTKCTL.PL0PCTEN. - /// - /// 1 This control does not cause any instructions to be trapped. - /// - /// If EL3 is implemented and EL2 is not implemented, behavior is as if - /// this bit is 1 other than for the purpose of a direct read. - EL1PCTEN OFFSET(0) NUMBITS(1) [] - ] -} - -pub struct Reg; - -impl RegisterReadWrite for Reg { - sys_coproc_read_raw!(u32, "CNTHCTL_EL2"); - sys_coproc_write_raw!(u32, "CNTHCTL_EL2"); -} - -#[allow(non_upper_case_globals)] -pub static CNTHCTL_EL2: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/cntp_ctl_el0.rs b/crate/aarch64/src/regs/cntp_ctl_el0.rs deleted file mode 100644 index 76991eb..0000000 --- a/crate/aarch64/src/regs/cntp_ctl_el0.rs +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright (c) 2018 by the author(s) - * - * ============================================================================= - * - * Licensed under either of - * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) - * - MIT License (http://opensource.org/licenses/MIT) - * at your option. - * - * ============================================================================= - * - * Author(s): - * - Andre Richter - */ - -//! Counter-timer Physical Timer Control register - EL0 -//! -//! Control register for the EL1 physical timer. - -use register::cpu::RegisterReadWrite; - -register_bitfields! {u32, - CNTP_CTL_EL0 [ - /// The status of the timer. This bit indicates whether the timer - /// condition is met: - /// - /// 0 Timer condition is not met. - /// 1 Timer condition is met. - /// - /// When the value of the ENABLE bit is 1, ISTATUS indicates whether the - /// timer condition is met. ISTATUS takes no account of the value of the - /// IMASK bit. If the value of ISTATUS is 1 and the value of IMASK is 0 - /// then the timer interrupt is asserted. - /// - /// When the value of the ENABLE bit is 0, the ISTATUS field is UNKNOWN. - /// - /// This bit is read-only. - ISTATUS OFFSET(2) NUMBITS(1) [], - - /// Timer interrupt mask bit. Permitted values are: - /// - /// 0 Timer interrupt is not masked by the IMASK bit. - /// 1 Timer interrupt is masked by the IMASK bit. - IMASK OFFSET(1) NUMBITS(1) [], - - /// Enables the timer. Permitted values are: - /// - /// 0 Timer disabled. - /// 1 Timer enabled. - ENABLE OFFSET(0) NUMBITS(1) [] - ] -} - -pub struct Reg; - -impl RegisterReadWrite for Reg { - sys_coproc_read_raw!(u32, "CNTP_CTL_EL0"); - sys_coproc_write_raw!(u32, "CNTP_CTL_EL0"); -} - -pub static CNTP_CTL_EL0: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/cntp_tval_el0.rs b/crate/aarch64/src/regs/cntp_tval_el0.rs deleted file mode 100644 index bdf5f6a..0000000 --- a/crate/aarch64/src/regs/cntp_tval_el0.rs +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (c) 2018 by the author(s) - * - * ============================================================================= - * - * Licensed under either of - * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) - * - MIT License (http://opensource.org/licenses/MIT) - * at your option. - * - * ============================================================================= - * - * Author(s): - * - Andre Richter - */ - -//! Counter-timer Physical Timer TimerValue register - EL0 -//! -//! Holds the timer value for the EL1 physical timer. - -use register::cpu::RegisterReadWrite; - -pub struct Reg; - -impl RegisterReadWrite for Reg { - sys_coproc_read_raw!(u32, "CNTP_TVAL_EL0"); - sys_coproc_write_raw!(u32, "CNTP_TVAL_EL0"); -} - -pub static CNTP_TVAL_EL0: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/cntpct_el0.rs b/crate/aarch64/src/regs/cntpct_el0.rs deleted file mode 100644 index b381d99..0000000 --- a/crate/aarch64/src/regs/cntpct_el0.rs +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (c) 2018 by the author(s) - * - * ============================================================================= - * - * Licensed under either of - * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) - * - MIT License (http://opensource.org/licenses/MIT) - * at your option. - * - * ============================================================================= - * - * Author(s): - * - Andre Richter - */ - -//! Counter-timer Physical Count register - EL0 -//! -//! Holds the 64-bit physical count value. - -use register::cpu::RegisterReadOnly; - -pub struct Reg; - -impl RegisterReadOnly for Reg { - sys_coproc_read_raw!(u64, "CNTPCT_EL0"); -} - -pub static CNTPCT_EL0: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/cntvoff_el2.rs b/crate/aarch64/src/regs/cntvoff_el2.rs deleted file mode 100644 index aff7074..0000000 --- a/crate/aarch64/src/regs/cntvoff_el2.rs +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (c) 2018 by the author(s) - * - * ============================================================================= - * - * Licensed under either of - * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) - * - MIT License (http://opensource.org/licenses/MIT) - * at your option. - * - * ============================================================================= - * - * Author(s): - * - Andre Richter - */ - -//! Counter-timer Virtual Offset register - EL2 -//! -//! Holds the 64-bit virtual offset. This is the offset between the physical -//! count value visible in CNTPCT_EL0 and the virtual count value visible in -//! CNTVCT_EL0. - -use register::cpu::RegisterReadWrite; - -pub struct Reg; - -impl RegisterReadWrite for Reg { - sys_coproc_read_raw!(u64, "CNTVOFF_EL2"); - sys_coproc_write_raw!(u64, "CNTVOFF_EL2"); -} - -pub static CNTVOFF_EL2: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/currentel.rs b/crate/aarch64/src/regs/currentel.rs deleted file mode 100644 index 91b8e0a..0000000 --- a/crate/aarch64/src/regs/currentel.rs +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (c) 2018 by the author(s) - * - * ============================================================================= - * - * Licensed under either of - * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) - * - MIT License (http://opensource.org/licenses/MIT) - * at your option. - * - * ============================================================================= - * - * Author(s): - * - Andre Richter - */ - -//! Current Exception Level -//! -//! Holds the current Exception level. - -use register::cpu::RegisterReadOnly; - -register_bitfields! {u32, - CurrentEL [ - /// Current Exception level. Possible values of this field are: - /// - /// 00 EL0 - /// 01 EL1 - /// 10 EL2 - /// 11 EL3 - /// - /// When the HCR_EL2.NV bit is 1, Non-secure EL1 read accesses to the - /// CurrentEL register return the value of 0x2 in this field. - /// - /// This field resets to a value that is architecturally UNKNOWN. - EL OFFSET(2) NUMBITS(2) [ - EL0 = 0, - EL1 = 1, - EL2 = 2, - EL3 = 3 - ] - ] -} - -pub struct Reg; - -impl RegisterReadOnly for Reg { - sys_coproc_read_raw!(u32, "CurrentEL"); -} - -#[allow(non_upper_case_globals)] -pub static CurrentEL: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/daif.rs b/crate/aarch64/src/regs/daif.rs deleted file mode 100644 index bf810a2..0000000 --- a/crate/aarch64/src/regs/daif.rs +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright (c) 2018 by the author(s) - * - * ============================================================================= - * - * Licensed under either of - * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) - * - MIT License (http://opensource.org/licenses/MIT) - * at your option. - * - * ============================================================================= - * - * Author(s): - * - Andre Richter - */ - -//! Interrupt Mask Bits -//! -//! Allows access to the interrupt mask bits. - -use register::cpu::RegisterReadWrite; - -register_bitfields! {u32, - DAIF [ - /// Process state D mask. The possible values of this bit are: - /// - /// 0 Watchpoint, Breakpoint, and Software Step exceptions targeted at - /// the current Exception level are not masked. - /// - /// 1 Watchpoint, Breakpoint, and Software Step exceptions targeted at - /// the current Exception level are masked. - /// - /// When the target Exception level of the debug exception is higher - /// than the current Exception level, the exception is not masked by - /// this bit. - /// - /// When this register has an architecturally-defined reset value, this - /// field resets to 1. - D OFFSET(9) NUMBITS(1) [ - Unmasked = 0, - Masked = 1 - ], - - /// SError interrupt mask bit. The possible values of this bit are: - /// - /// 0 Exception not masked. - /// 1 Exception masked. - /// - /// When this register has an architecturally-defined reset value, this - /// field resets to 1 . - A OFFSET(8) NUMBITS(1) [ - Unmasked = 0, - Masked = 1 - ], - - /// IRQ mask bit. The possible values of this bit are: - /// - /// 0 Exception not masked. - /// 1 Exception masked. - /// - /// When this register has an architecturally-defined reset value, this - /// field resets to 1 . - I OFFSET(7) NUMBITS(1) [ - Unmasked = 0, - Masked = 1 - ], - - /// FIQ mask bit. The possible values of this bit are: - /// - /// 0 Exception not masked. - /// 1 Exception masked. - /// - /// When this register has an architecturally-defined reset value, this - /// field resets to 1 . - F OFFSET(6) NUMBITS(1) [ - Unmasked = 0, - Masked = 1 - ] - ] -} - - -pub struct Reg; - -impl RegisterReadWrite for Reg { - sys_coproc_read_raw!(u32, "DAIF"); - sys_coproc_write_raw!(u32, "DAIF"); -} - -pub static DAIF: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/elr_el2.rs b/crate/aarch64/src/regs/elr_el2.rs deleted file mode 100644 index 0786fbb..0000000 --- a/crate/aarch64/src/regs/elr_el2.rs +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (c) 2018 by the author(s) - * - * ============================================================================= - * - * Licensed under either of - * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) - * - MIT License (http://opensource.org/licenses/MIT) - * at your option. - * - * ============================================================================= - * - * Author(s): - * - Andre Richter - */ - -//! Exception Link Register - EL2 -//! -//! When taking an exception to EL2, holds the address to return to. - -use register::cpu::RegisterReadWrite; - -pub struct Reg; - -impl RegisterReadWrite for Reg { - sys_coproc_read_raw!(u64, "ELR_EL2"); - sys_coproc_write_raw!(u64, "ELR_EL2"); -} - -pub static ELR_EL2: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/far_el1.rs b/crate/aarch64/src/regs/far_el1.rs deleted file mode 100644 index fc809fa..0000000 --- a/crate/aarch64/src/regs/far_el1.rs +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2018 by the author(s) - * - * ============================================================================= - * - * Licensed under either of - * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) - * - MIT License (http://opensource.org/licenses/MIT) - * at your option. - * - * ============================================================================= - * - * Author(s): - * - Andre Richter - */ - -//! Fault Address Register - EL1 -//! -//! Holds the faulting Virtual Address for all synchronous Instruction or Data -//! Abort, PC alignment fault and Watchpoint exceptions that are taken to EL1. - -use register::cpu::RegisterReadWrite; - -pub struct Reg; - -impl RegisterReadWrite for Reg { - sys_coproc_read_raw!(u64, "FAR_EL1"); - sys_coproc_write_raw!(u64, "FAR_EL1"); -} - -pub static FAR_EL1: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/hcr_el2.rs b/crate/aarch64/src/regs/hcr_el2.rs deleted file mode 100644 index 683bbef..0000000 --- a/crate/aarch64/src/regs/hcr_el2.rs +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Copyright (c) 2018 by the author(s) - * - * ============================================================================= - * - * Licensed under either of - * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) - * - MIT License (http://opensource.org/licenses/MIT) - * at your option. - * - * ============================================================================= - * - * Author(s): - * - Andre Richter - */ - -//! Hypervisor Configuration Register - EL2 -//! -//! Provides configuration controls for virtualization, including defining -//! whether various Non-secure operations are trapped to EL2. - -use register::cpu::RegisterReadWrite; - -register_bitfields! {u64, - HCR_EL2 [ - /// Execution state control for lower Exception levels: - /// - /// 0 Lower levels are all AArch32. - /// 1 The Execution state for EL1 is AArch64. The Execution state for - /// EL0 is determined by the current value of PSTATE.nRW when - /// executing at EL0. - /// - /// If all lower Exception levels cannot use AArch32 then this bit is - /// RAO/WI. - /// - /// In an implementation that includes EL3, when SCR_EL3.NS==0, the PE - /// behaves as if this bit has the same value as the SCR_EL3.RW bit for - /// all purposes other than a direct read or write access of HCR_EL2. - /// - /// The RW bit is permitted to be cached in a TLB. - /// - /// When ARMv8.1-VHE is implemented, and the value of HCR_EL2.{E2H, TGE} - /// is {1, 1}, this field behaves as 1 for all purposes other than a - /// direct read of the value of this bit. - RW OFFSET(31) NUMBITS(1) [ - AllLowerELsAreAarch32 = 0, - EL1IsAarch64 = 1 - ], - - /// Default Cacheability. - /// - /// 0 This control has no effect on the Non-secure EL1&0 translation - /// regime. - /// - /// 1 In Non-secure state: - /// - When EL1 is using AArch64, the PE behaves as if the value of - /// the SCTLR_EL1.M field is 0 for all purposes other than - /// returning the value of a direct read of SCTLR_EL1. - /// - /// - When EL1 is using AArch32, the PE behaves as if the value of - /// the SCTLR.M field is 0 for all purposes other than returning - /// the value of a direct read of SCTLR. - /// - /// - The PE behaves as if the value of the HCR_EL2.VM field is 1 - /// for all purposes other than returning the value of a direct - /// read of HCR_EL2. - /// - /// - The memory type produced by stage 1 of the EL1&0 translation - /// regime is Normal Non-Shareable, Inner Write-Back Read-Allocate - /// Write-Allocate, Outer Write-Back Read-Allocate Write-Allocate. - /// - /// This field has no effect on the EL2, EL2&0, and EL3 translation - /// regimes. - /// - /// This field is permitted to be cached in a TLB. - /// - /// In an implementation that includes EL3, when the value of SCR_EL3.NS - /// is 0 the PE behaves as if this field is 0 for all purposes other - /// than a direct read or write access of HCR_EL2. - /// - /// When ARMv8.1-VHE is implemented, and the value of HCR_EL2.{E2H, TGE} - /// is {1, 1}, this field behaves as 0 for all purposes other than a - /// direct read of the value of this field. - DC OFFSET(12) NUMBITS(1) [], - - /// Set/Way Invalidation Override. Causes Non-secure EL1 execution of - /// the data cache invalidate by set/way instructions to perform a data - /// cache clean and invalidate by set/way: - /// - /// 0 This control has no effect on the operation of data cache - /// invalidate by set/way instructions. - /// - /// 1 Data cache invalidate by set/way instructions perform a data cache - /// clean and invalidate by set/way. - /// - /// When the value of this bit is 1: - /// - /// AArch32: DCISW performs the same invalidation as a DCCISW - /// instruction. - /// - /// AArch64: DC ISW performs the same invalidation as a DC CISW - /// instruction. - /// - /// This bit can be implemented as RES 1. - /// - /// In an implementation that includes EL3, when the value of SCR_EL3.NS - /// is 0 the PE behaves as if this field is 0 for all purposes other - /// than a direct read or write access of HCR_EL2. - /// - /// When HCR_EL2.TGE is 1, the PE ignores the value of this field for - /// all purposes other than a direct read of this field. - SWIO OFFSET(1) NUMBITS(1) [] - ] -} - -pub struct Reg; - -impl RegisterReadWrite for Reg { - sys_coproc_read_raw!(u64, "HCR_EL2"); - sys_coproc_write_raw!(u64, "HCR_EL2"); -} - -pub static HCR_EL2: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/id_aa64mmfr0_el1.rs b/crate/aarch64/src/regs/id_aa64mmfr0_el1.rs deleted file mode 100644 index f75813c..0000000 --- a/crate/aarch64/src/regs/id_aa64mmfr0_el1.rs +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright (c) 2018 by the author(s) - * - * ============================================================================= - * - * Licensed under either of - * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) - * - MIT License (http://opensource.org/licenses/MIT) - * at your option. - * - * ============================================================================= - * - * Author(s): - * - Andre Richter - */ - -//! AArch64 Memory Model Feature Register 0 - EL1 -//! -//! Provides information about the implemented memory model and memory -//! management support in AArch64 state. - -use register::cpu::RegisterReadOnly; - -register_bitfields! {u64, - ID_AA64MMFR0_EL1 [ - /// Support for 4KiB memory translation granule size. Defined values - /// are: - /// - /// 0000 4KiB granule supported. - /// 1111 4KiB granule not supported. - /// - /// All other values are reserved. - TGran4 OFFSET(28) NUMBITS(4) [ - Supported = 0b0000, - NotSupported = 0b1111 - ], - - /// Support for 64KiB memory translation granule size. Defined values - /// are: - /// - /// 0000 64KiB granule supported. - /// 1111 64KiB granule not supported. - /// - /// All other values are reserved. - TGran64 OFFSET(24) NUMBITS(4) [ - Supported = 0b0000, - NotSupported = 0b1111 - ], - - /// Physical Address range supported. Defined values are: - /// - /// 0000 32 bits, 4GiB. - /// 0001 36 bits, 64GiB. - /// 0010 40 bits, 1TiB. - /// 0011 42 bits, 4TiB. - /// 0100 44 bits, 16TiB. - /// 0101 48 bits, 256TiB. - /// 0110 52 bits, 4PiB. - /// - /// All other values are reserved. - /// - /// The value 0110 is permitted only if the implementation includes - /// ARMv8.2-LPA, otherwise it is reserved. - PARange OFFSET(0) NUMBITS(4) [ - Bits_32 = 0b0000, - Bits_36 = 0b0001, - Bits_40 = 0b0010, - Bits_42 = 0b0011, - Bits_44 = 0b0100, - Bits_48 = 0b0101, - Bits_52 = 0b0110 - ] - ] -} - -pub struct Reg; - -impl RegisterReadOnly for Reg { - sys_coproc_read_raw!(u64, "ID_AA64MMFR0_EL1"); -} - -pub static ID_AA64MMFR0_EL1: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/macros.rs b/crate/aarch64/src/regs/macros.rs deleted file mode 100644 index bd4439c..0000000 --- a/crate/aarch64/src/regs/macros.rs +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright (c) 2018 by the author(s) - * - * ============================================================================= - * - * Licensed under either of - * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) - * - MIT License (http://opensource.org/licenses/MIT) - * at your option. - * - * ============================================================================= - * - * Author(s): - * - Andre Richter - */ - -macro_rules! __read_raw { - ($width:ty, $asm_instr:tt, $asm_reg_name:tt) => { - /// Reads the raw bits of the CPU register. - #[inline] - fn get(&self) -> $width { - match () { - #[cfg(target_arch = "aarch64")] - () => { - let reg; - unsafe { - asm!(concat!($asm_instr, " $0, ", $asm_reg_name) : "=r"(reg) ::: "volatile"); - } - reg - } - - #[cfg(not(target_arch = "aarch64"))] - () => unimplemented!(), - } - } - }; -} - -macro_rules! __write_raw { - ($width:ty, $asm_instr:tt, $asm_reg_name:tt) => { - /// Writes raw bits to the CPU register. - #[cfg_attr(not(target_arch = "aarch64"), allow(unused_variables))] - #[inline] - fn set(&self, value: $width) { - match () { - #[cfg(target_arch = "aarch64")] - () => { - unsafe { - asm!(concat!($asm_instr, " ", $asm_reg_name, ", $0") :: "r"(value) :: "volatile") - } - } - - #[cfg(not(target_arch = "aarch64"))] - () => unimplemented!(), - } - } - }; -} - -/// Raw read from system coprocessor registers -macro_rules! sys_coproc_read_raw { - ($width:ty, $asm_reg_name:tt) => { - __read_raw!($width, "mrs", $asm_reg_name); - }; -} - -/// Raw write to system coprocessor registers -macro_rules! sys_coproc_write_raw { - ($width:ty, $asm_reg_name:tt) => { - __write_raw!($width, "msr", $asm_reg_name); - }; -} - -/// Raw read from (ordinary) registers -macro_rules! read_raw { - ($width:ty, $asm_reg_name:tt) => { - __read_raw!($width, "mov", $asm_reg_name); - }; -} -/// Raw write to (ordinary) registers -macro_rules! write_raw { - ($width:ty, $asm_reg_name:tt) => { - __write_raw!($width, "mov", $asm_reg_name); - }; -} diff --git a/crate/aarch64/src/regs/mair_el1.rs b/crate/aarch64/src/regs/mair_el1.rs deleted file mode 100644 index dbd7f9d..0000000 --- a/crate/aarch64/src/regs/mair_el1.rs +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright (c) 2018 by the author(s) - * - * ============================================================================= - * - * Licensed under either of - * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) - * - MIT License (http://opensource.org/licenses/MIT) - * at your option. - * - * ============================================================================= - * - * Author(s): - * - Andre Richter - */ - -//! Memory Attribute Indirection Register - EL1 -//! -//! Provides the memory attribute encodings corresponding to the possible -//! AttrIndx values in a Long-descriptor format translation table entry for -//! stage 1 translations at EL1. - -use register::cpu::RegisterReadWrite; - -register_bitfields! {u64, - MAIR_EL1 [ - /// Attribute 7 - Attr7 OFFSET(56) NUMBITS(8) [], - - /// Attribute 6 - Attr6 OFFSET(48) NUMBITS(8) [], - - /// Attribute 5 - Attr5 OFFSET(40) NUMBITS(8) [], - - /// Attribute 4 - Attr4 OFFSET(32) NUMBITS(8) [], - - /// Attribute 3 - Attr3 OFFSET(24) NUMBITS(8) [], - - /// Attribute 2 - Attr2 OFFSET(16) NUMBITS(8) [], - - /// Attribute 1 - Attr1 OFFSET(8) NUMBITS(8) [], - - /// Attribute 0 - Attr0 OFFSET(0) NUMBITS(8) [] - ] -} - -register_bitfields! {u64, - MAIR_ATTR [ - Attr_HIGH OFFSET(4) NUMBITS(4) [ - Device = 0b0000, - Memory_OuterNonCacheable = 0b0100, - Memory_OuterWriteThrough_NonTransient_ReadAlloc_WriteAlloc = 0b1011, - Memory_OuterWriteBack_NonTransient_ReadAlloc_WriteAlloc = 0b1111 - ], - Attr_LOW_DEVICE OFFSET(0) NUMBITS(4) [ - Device_nGnRnE = 0b0000, - Device_nGnRE = 0b0100, - Device_nGRE = 0b1000, - Device_GRE = 0b1100 - ], - Attr_LOW_MEMORY OFFSET(0) NUMBITS(4) [ - InnerNonCacheable = 0b0100, - InnerWriteThrough_NonTransient_ReadAlloc_WriteAlloc = 0b1011, - InnerWriteBack_NonTransient_ReadAlloc_WriteAlloc = 0b1111 - ] - ] -} - -pub struct Reg; - -impl RegisterReadWrite for Reg { - sys_coproc_read_raw!(u64, "MAIR_EL1"); - sys_coproc_write_raw!(u64, "MAIR_EL1"); -} - -pub static MAIR_EL1: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/mod.rs b/crate/aarch64/src/regs/mod.rs deleted file mode 100644 index 11a6d3f..0000000 --- a/crate/aarch64/src/regs/mod.rs +++ /dev/null @@ -1,55 +0,0 @@ -//! Processor core registers - -#[macro_use] -mod macros; - -mod cntfrq_el0; -mod cnthctl_el2; -mod cntp_ctl_el0; -mod cntp_tval_el0; -mod cntpct_el0; -mod cntvoff_el2; -mod currentel; -mod daif; -mod elr_el2; -mod far_el1; -mod hcr_el2; -mod id_aa64mmfr0_el1; -mod mair_el1; -mod mpidr_el1; -mod sctlr_el1; -mod sp; -mod sp_el0; -mod sp_el1; -mod spsel; -mod spsr_el2; -mod tcr_el1; -mod ttbr0_el1; -mod ttbr1_el1; - -// Export only the R/W traits and the static reg definitions -pub use register::cpu::*; - -pub use self::cntfrq_el0::CNTFRQ_EL0; -pub use self::cnthctl_el2::CNTHCTL_EL2; -pub use self::cntp_ctl_el0::CNTP_CTL_EL0; -pub use self::cntp_tval_el0::CNTP_TVAL_EL0; -pub use self::cntpct_el0::CNTPCT_EL0; -pub use self::cntvoff_el2::CNTVOFF_EL2; -pub use self::currentel::CurrentEL; -pub use self::daif::DAIF; -pub use self::elr_el2::ELR_EL2; -pub use self::far_el1::FAR_EL1; -pub use self::hcr_el2::HCR_EL2; -pub use self::id_aa64mmfr0_el1::ID_AA64MMFR0_EL1; -pub use self::mair_el1::{MAIR_EL1, MAIR_ATTR}; -pub use self::mpidr_el1::MPIDR_EL1; -pub use self::sctlr_el1::SCTLR_EL1; -pub use self::sp::SP; -pub use self::sp_el0::SP_EL0; -pub use self::sp_el1::SP_EL1; -pub use self::spsel::SPSel; -pub use self::spsr_el2::SPSR_EL2; -pub use self::tcr_el1::TCR_EL1; -pub use self::ttbr0_el1::TTBR0_EL1; -pub use self::ttbr1_el1::TTBR1_EL1; diff --git a/crate/aarch64/src/regs/mpidr_el1.rs b/crate/aarch64/src/regs/mpidr_el1.rs deleted file mode 100644 index 6fbfea0..0000000 --- a/crate/aarch64/src/regs/mpidr_el1.rs +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (c) 2018 by the author(s) - * - * ============================================================================= - * - * Licensed under either of - * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) - * - MIT License (http://opensource.org/licenses/MIT) - * at your option. - * - * ============================================================================= - * - * Author(s): - * - Andre Richter - */ - -//! Multiprocessor Affinity Register - EL1 -//! -//! In a multiprocessor system, provides an additional PE -//! identification mechanism for scheduling purposes. - -use register::cpu::RegisterReadOnly; - -pub struct Reg; - -impl RegisterReadOnly for Reg { - sys_coproc_read_raw!(u64, "MPIDR_EL1"); -} - -pub static MPIDR_EL1: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/sctlr_el1.rs b/crate/aarch64/src/regs/sctlr_el1.rs deleted file mode 100644 index 1f463b4..0000000 --- a/crate/aarch64/src/regs/sctlr_el1.rs +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright (c) 2018 by the author(s) - * - * ============================================================================= - * - * Licensed under either of - * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) - * - MIT License (http://opensource.org/licenses/MIT) - * at your option. - * - * ============================================================================= - * - * Author(s): - * - Andre Richter - */ - -//! System Control Register - EL1 -//! -//! Provides top level control of the system, including its memory system, at -//! EL1 and EL0. - -use register::cpu::RegisterReadWrite; - -register_bitfields! {u32, - SCTLR_EL1 [ - /// Instruction access Cacheability control, for accesses at EL0 and - /// EL1: - /// - /// 0 All instruction access to Normal memory from EL0 and EL1 are - /// Non-cacheable for all levels of instruction and unified cache. - /// - /// If the value of SCTLR_EL1.M is 0, instruction accesses from stage - /// 1 of the EL1&0 translation regime are to Normal, Outer Shareable, - /// Inner Non-cacheable, Outer Non-cacheable memory. - /// - /// 1 This control has no effect on the Cacheability of instruction - /// access to Normal memory from EL0 and EL1. - /// - /// If the value of SCTLR_EL1.M is 0, instruction accesses from stage - /// 1 of the EL1&0 translation regime are to Normal, Outer Shareable, - /// Inner Write-Through, Outer Write-Through memory. - /// - /// When the value of the HCR_EL2.DC bit is 1, then instruction access - /// to Normal memory from EL0 and EL1 are Cacheable regardless of the - /// value of the SCTLR_EL1.I bit. - /// - /// When ARMv8.1-VHE is implemented, and the value of HCR_EL2.{E2H, TGE} - /// is {1, 1}, this bit has no effect on the PE. - /// - /// When this register has an architecturally-defined reset value, this - /// field resets to 0. - I OFFSET(12) NUMBITS(1) [ - NonCacheable = 0, - Cacheable = 1 - ], - - /// Cacheability control, for data accesses. - /// - /// 0 All data access to Normal memory from EL0 and EL1, and all Normal - /// memory accesses to the EL1&0 stage 1 translation tables, are - /// Non-cacheable for all levels of data and unified cache. - /// - /// 1 This control has no effect on the Cacheability of: - /// - Data access to Normal memory from EL0 and EL1. - /// - Normal memory accesses to the EL1&0 stage 1 translation - /// tables. - /// - /// When the value of the HCR_EL2.DC bit is 1, the PE ignores - /// SCLTR.C. This means that Non-secure EL0 and Non-secure EL1 data - /// accesses to Normal memory are Cacheable. - /// - /// When ARMv8.1-VHE is implemented, and the value of HCR_EL2.{E2H, TGE} - /// is {1, 1}, this bit has no effect on the PE. - /// - /// When this register has an architecturally-defined reset value, this - /// field resets to 0. - C OFFSET(2) NUMBITS(1) [ - NonCacheable = 0, - Cacheable = 1 - ], - - /// MMU enable for EL1 and EL0 stage 1 address translation. Possible - /// values of this bit are: - /// - /// 0 EL1 and EL0 stage 1 address translation disabled. - /// See the SCTLR_EL1.I field for the behavior of instruction accesses - /// to Normal memory. - /// 1 EL1 and EL0 stage 1 address translation enabled. - M OFFSET(0) NUMBITS(1) [ - Disable = 0, - Enable = 1 - ] - ] -} - -pub struct Reg; - -impl RegisterReadWrite for Reg { - sys_coproc_read_raw!(u32, "SCTLR_EL1"); - sys_coproc_write_raw!(u32, "SCTLR_EL1"); -} - -pub static SCTLR_EL1: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/sp.rs b/crate/aarch64/src/regs/sp.rs deleted file mode 100644 index f9f578b..0000000 --- a/crate/aarch64/src/regs/sp.rs +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2018 by the author(s) - * - * ============================================================================= - * - * Licensed under either of - * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) - * - MIT License (http://opensource.org/licenses/MIT) - * at your option. - * - * ============================================================================= - * - * Author(s): - * - Andre Richter - */ - -//! The stack pointer - -use register::cpu::RegisterReadWrite; - -pub struct Reg; - -impl RegisterReadWrite for Reg { - read_raw!(u64, "sp"); - write_raw!(u64, "sp"); -} - -pub static SP: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/sp_el0.rs b/crate/aarch64/src/regs/sp_el0.rs deleted file mode 100644 index aa82fdb..0000000 --- a/crate/aarch64/src/regs/sp_el0.rs +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2018 by the author(s) - * - * ============================================================================= - * - * Licensed under either of - * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) - * - MIT License (http://opensource.org/licenses/MIT) - * at your option. - * - * ============================================================================= - * - * Author(s): - * - Andre Richter - */ - -//! The stack pointer - EL0 -//! -//! Holds the stack pointer associated with EL0. At higher Exception levels, -//! this is used as the current stack pointer when the value of SPSel.SP is 0. - -use register::cpu::RegisterReadWrite; - -pub struct Reg; - -impl RegisterReadWrite for Reg { - sys_coproc_read_raw!(u64, "SP_EL0"); - sys_coproc_write_raw!(u64, "SP_EL0"); -} - -pub static SP_EL0: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/sp_el1.rs b/crate/aarch64/src/regs/sp_el1.rs deleted file mode 100644 index 4357412..0000000 --- a/crate/aarch64/src/regs/sp_el1.rs +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2018 by the author(s) - * - * ============================================================================= - * - * Licensed under either of - * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) - * - MIT License (http://opensource.org/licenses/MIT) - * at your option. - * - * ============================================================================= - * - * Author(s): - * - Andre Richter - */ - -//! The stack pointer - EL1 -//! -//! Holds the stack pointer associated with EL1. When executing at EL1, the -//! value of SPSel.SP determines the current stack pointer: -//! -//! SPSel.SP | current stack pointer -//! -------------------------------- -//! 0 | SP_EL0 -//! 1 | SP_EL1 - -use register::cpu::RegisterReadWrite; - -pub struct Reg; - -impl RegisterReadWrite for Reg { - sys_coproc_read_raw!(u64, "SP_EL1"); - sys_coproc_write_raw!(u64, "SP_EL1"); -} - -pub static SP_EL1: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/spsel.rs b/crate/aarch64/src/regs/spsel.rs deleted file mode 100644 index 91e3694..0000000 --- a/crate/aarch64/src/regs/spsel.rs +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright (c) 2018 by the author(s) - * - * ============================================================================= - * - * Licensed under either of - * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) - * - MIT License (http://opensource.org/licenses/MIT) - * at your option. - * - * ============================================================================= - * - * Author(s): - * - Andre Richter - */ - -//! Stack Pointer Select -//! -//! Allows the Stack Pointer to be selected between SP_EL0 and SP_ELx. - -use register::cpu::RegisterReadWrite; - -register_bitfields! {u32, - SPSel [ - /// Stack pointer to use. Possible values of this bit are: - /// - /// 0 Use SP_EL0 at all Exception levels. - /// 1 Use SP_ELx for Exception level ELx. - /// - /// When this register has an architecturally-defined reset value, this - /// field resets to 1. - SP OFFSET(0) NUMBITS(1) [ - EL0 = 0, - ELx = 1 - ] - ] -} - - -pub struct Reg; - -impl RegisterReadWrite for Reg { - sys_coproc_read_raw!(u32, "SPSEL"); - sys_coproc_write_raw!(u32, "SPSEL"); -} - -#[allow(non_upper_case_globals)] -pub static SPSel: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/spsr_el2.rs b/crate/aarch64/src/regs/spsr_el2.rs deleted file mode 100644 index 56078a4..0000000 --- a/crate/aarch64/src/regs/spsr_el2.rs +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Copyright (c) 2018 by the author(s) - * - * ============================================================================= - * - * Licensed under either of - * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) - * - MIT License (http://opensource.org/licenses/MIT) - * at your option. - * - * ============================================================================= - * - * Author(s): - * - Andre Richter - */ - -//! Saved Program Status Register - EL2 -//! -//! Holds the saved process state when an exception is taken to EL2. - -use register::cpu::RegisterReadWrite; - -register_bitfields! {u32, - SPSR_EL2 [ - /// Process state D mask. The possible values of this bit are: - /// - /// 0 Watchpoint, Breakpoint, and Software Step exceptions targeted at - /// the current Exception level are not masked. - /// - /// 1 Watchpoint, Breakpoint, and Software Step exceptions targeted at - /// the current Exception level are masked. - /// - /// When the target Exception level of the debug exception is higher - /// than the current Exception level, the exception is not masked by - /// this bit. - D OFFSET(9) NUMBITS(1) [ - Unmasked = 0, - Masked = 1 - ], - - /// SError interrupt mask bit. The possible values of this bit are: - /// - /// 0 Exception not masked. - /// 1 Exception masked. - A OFFSET(8) NUMBITS(1) [ - Unmasked = 0, - Masked = 1 - ], - - /// IRQ mask bit. The possible values of this bit are: - /// - /// 0 Exception not masked. - /// 1 Exception masked. - I OFFSET(7) NUMBITS(1) [ - Unmasked = 0, - Masked = 1 - ], - - /// FIQ mask bit. The possible values of this bit are: - /// - /// 0 Exception not masked. - /// 1 Exception masked. - F OFFSET(6) NUMBITS(1) [ - Unmasked = 0, - Masked = 1 - ], - - /// AArch64 state (Exception level and selected SP) that an exception - /// was taken from. The possible values are: - /// - /// M[3:0] | State - /// -------------- - /// 0b0000 | EL0t - /// 0b0100 | EL1t - /// 0b0101 | EL1h - /// 0b1000 | EL2t - /// 0b1001 | EL2h - /// - /// Other values are reserved, and returning to an Exception level that - /// is using AArch64 with a reserved value in this field is treated as - /// an illegal exception return. - /// - /// The bits in this field are interpreted as follows: - /// - M[3:2] holds the Exception Level. - /// - M[1] is unused and is RES 0 for all non-reserved values. - /// - M[0] is used to select the SP: - /// - 0 means the SP is always SP0. - /// - 1 means the exception SP is determined by the EL. - M OFFSET(0) NUMBITS(4) [ - EL0t = 0b0000, - EL1t = 0b0100, - EL1h = 0b0101, - EL2t = 0b1000, - EL2h = 0b1001 - ] - ] -} - -pub struct Reg; - -impl RegisterReadWrite for Reg { - sys_coproc_read_raw!(u32, "SPSR_EL2"); - sys_coproc_write_raw!(u32, "SPSR_EL2"); -} - -pub static SPSR_EL2: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/tcr_el1.rs b/crate/aarch64/src/regs/tcr_el1.rs deleted file mode 100644 index 9ebb6d7..0000000 --- a/crate/aarch64/src/regs/tcr_el1.rs +++ /dev/null @@ -1,313 +0,0 @@ -/* - * Copyright (c) 2018 by the author(s) - * - * ============================================================================= - * - * Licensed under either of - * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) - * - MIT License (http://opensource.org/licenses/MIT) - * at your option. - * - * ============================================================================= - * - * Author(s): - * - Andre Richter - */ - -//! Translation Control Register - EL1 -//! -//! The control register for stage 1 of the EL1&0 translation regime. - -use register::cpu::RegisterReadWrite; - -register_bitfields! {u64, - TCR_EL1 [ - /// Top Byte ignored - indicates whether the top byte of an address is - /// used for address match for the TTBR1_EL1 region, or ignored and used - /// for tagged addresses. Defined values are: - /// - /// 0 Top Byte used in the address calculation. - /// 1 Top Byte ignored in the address calculation. - TBI1 OFFSET(38) NUMBITS(1) [ - Used = 0, - Ignored = 1 - ], - - /// Top Byte ignored - indicates whether the top byte of an address is - /// used for address match for the TTBR0_EL1 region, or ignored and used - /// for tagged addresses. Defined values are: - /// - /// 0 Top Byte used in the address calculation. - /// 1 Top Byte ignored in the address calculation. - TBI0 OFFSET(37) NUMBITS(1) [ - Used = 0, - Ignored = 1 - ], - - /// ASID Size. Defined values are: - /// - /// 0 8 bit - the upper 8 bits of TTBR0_EL1 and TTBR1_EL1 are ignored by - /// hardware for every purpose except reading back the register, and are - /// treated as if they are all zeros for when used for allocation and matching entries in the TLB. - /// 1 16 bit - the upper 16 bits of TTBR0_EL1 and TTBR1_EL1 are used for - /// allocation and matching in the TLB. - /// - /// If the implementation has only 8 bits of ASID, this field is RES0. - AS OFFSET(36) NUMBITS(1) [ - Bits_8 = 0, - Bits_16 = 1 - ], - - /// Intermediate Physical Address Size. - /// - /// 000 32 bits, 4GiB. - /// 001 36 bits, 64GiB. - /// 010 40 bits, 1TiB. - /// 011 42 bits, 4TiB. - /// 100 44 bits, 16TiB. - /// 101 48 bits, 256TiB. - /// 110 52 bits, 4PiB - /// - /// Other values are reserved. - /// - /// The reserved values behave in the same way as the 101 or 110 - /// encoding, but software must not rely on this property as the - /// behavior of the reserved values might change in a future revision of - /// the architecture. - /// - /// The value 110 is permitted only if ARMv8.2-LPA is implemented and - /// the translation granule size is 64KiB. - /// - /// In an implementation that supports 52-bit PAs, if the value of this - /// field is not 110 , then bits[51:48] of every translation table base - /// address for the stage of translation controlled by TCR_EL1 are 0000 - /// . - IPS OFFSET(32) NUMBITS(3) [ - Bits_32 = 0b000, - Bits_36 = 0b001, - Bits_40 = 0b010, - Bits_42 = 0b011, - Bits_44 = 0b100, - Bits_48 = 0b101, - Bits_52 = 0b110 - ], - - /// Granule size for the TTBR1_EL1. - /// - /// 01 16KiB - /// 10 4KiB - /// 11 64KiB - /// - /// Other values are reserved. - /// - /// If the value is programmed to either a reserved value, or a size - /// that has not been implemented, then the hardware will treat the - /// field as if it has been programmed to an IMPLEMENTATION DEFINED - /// choice of the sizes that has been implemented for all purposes other - /// than the value read back from this register. - /// - /// It is IMPLEMENTATION DEFINED whether the value read back is the - /// value programmed or the value that corresponds to the size chosen. - TG1 OFFSET(30) NUMBITS(2) [ - KiB_4 = 0b10, - KiB_16 = 0b01, - KiB_64 = 0b11 - ], - - /// Shareability attribute for memory associated with translation table - /// walks using TTBR1_EL1. - /// - /// 00 Non-shareable - /// 10 Outer Shareable - /// 11 Inner Shareable - /// - /// Other values are reserved. - SH1 OFFSET(28) NUMBITS(2) [ - None = 0b00, - Outer = 0b10, - Inner = 0b11 - ], - - /// Outer cacheability attribute for memory associated with translation - /// table walks using TTBR1_EL1. - /// - /// 00 Normal memory, Outer Non-cacheable - /// - /// 01 Normal memory, Outer Write-Back Read-Allocate Write-Allocate - /// Cacheable - /// - /// 10 Normal memory, Outer Write-Through Read-Allocate No - /// Write-Allocate Cacheable - /// - /// 11 Normal memory, Outer Write-Back Read-Allocate No Write-Allocate - /// Cacheable - ORGN1 OFFSET(26) NUMBITS(2) [ - NonCacheable = 0b00, - WriteBack_ReadAlloc_WriteAlloc_Cacheable = 0b01, - WriteThrough_ReadAlloc_NoWriteAlloc_Cacheable = 0b10, - WriteBack_ReadAlloc_NoWriteAlloc_Cacheable = 0b11 - ], - - /// Inner cacheability attribute for memory associated with translation - /// table walks using TTBR1_EL1. - /// - /// 00 Normal memory, Inner Non-cacheable - /// - /// 01 Normal memory, Inner Write-Back Read-Allocate Write-Allocate - /// Cacheable - /// - /// 10 Normal memory, Inner Write-Through Read-Allocate No - /// Write-Allocate Cacheable - /// - /// 11 Normal memory, Inner Write-Back Read-Allocate No Write-Allocate - /// Cacheable - IRGN1 OFFSET(24) NUMBITS(2) [ - NonCacheable = 0b00, - WriteBack_ReadAlloc_WriteAlloc_Cacheable = 0b01, - WriteThrough_ReadAlloc_NoWriteAlloc_Cacheable = 0b10, - WriteBack_ReadAlloc_NoWriteAlloc_Cacheable = 0b11 - ], - - /// Translation table walk disable for translations using - /// TTBR1_EL1. This bit controls whether a translation table walk is - /// performed on a TLB miss, for an address that is translated using - /// TTBR1_EL1. The encoding of this bit is: - /// - /// 0 Perform translation table walks using TTBR1_EL1. - /// - /// 1 A TLB miss on an address that is translated using TTBR1_EL1 - /// generates a Translation fault. No translation table walk is - /// performed. - EPD1 OFFSET(23) NUMBITS(1) [ - EnableTTBR1Walks = 0, - DisableTTBR1Walks = 1 - ], - - /// Selects whether TTBR0_EL1 or TTBR1_EL1 defines the ASID. The encoding - /// of this bit is: - /// - /// 0 TTBR0_EL1.ASID defines the ASID. - /// - /// 1 TTBR1_EL1.ASID defines the ASID. - A1 OFFSET(22) NUMBITS(1) [ - UseTTBR0ASID = 0b0, - UseTTBR1ASID = 0b1 - ], - - /// The size offset of the memory region addressed by TTBR1_EL1. The - /// region size is 2^(64-T1SZ) bytes. - /// - /// The maximum and minimum possible values for T1SZ depend on the level - /// of translation table and the memory translation granule size, as - /// described in the AArch64 Virtual Memory System Architecture chapter. - T1SZ OFFSET(16) NUMBITS(6) [], - - /// Granule size for the TTBR0_EL1. - /// - /// 00 4KiB - /// 01 64KiB - /// 10 16KiB - /// - /// Other values are reserved. - /// - /// If the value is programmed to either a reserved value, or a size - /// that has not been implemented, then the hardware will treat the - /// field as if it has been programmed to an IMPLEMENTATION DEFINED - /// choice of the sizes that has been implemented for all purposes other - /// than the value read back from this register. - /// - /// It is IMPLEMENTATION DEFINED whether the value read back is the - /// value programmed or the value that corresponds to the size chosen. - TG0 OFFSET(14) NUMBITS(2) [ - KiB_4 = 0b00, - KiB_16 = 0b10, - KiB_64 = 0b01 - ], - - /// Shareability attribute for memory associated with translation table - /// walks using TTBR0_EL1. - /// - /// 00 Non-shareable - /// 10 Outer Shareable - /// 11 Inner Shareable - /// - /// Other values are reserved. - SH0 OFFSET(12) NUMBITS(2) [ - None = 0b00, - Outer = 0b10, - Inner = 0b11 - ], - - /// Outer cacheability attribute for memory associated with translation - /// table walks using TTBR0_EL1. - /// - /// 00 Normal memory, Outer Non-cacheable - /// - /// 01 Normal memory, Outer Write-Back Read-Allocate Write-Allocate - /// Cacheable - /// - /// 10 Normal memory, Outer Write-Through Read-Allocate No - /// Write-Allocate Cacheable - /// - /// 11 Normal memory, Outer Write-Back Read-Allocate No Write-Allocate - /// Cacheable - ORGN0 OFFSET(10) NUMBITS(2) [ - NonCacheable = 0b00, - WriteBack_ReadAlloc_WriteAlloc_Cacheable = 0b01, - WriteThrough_ReadAlloc_NoWriteAlloc_Cacheable = 0b10, - WriteBack_ReadAlloc_NoWriteAlloc_Cacheable = 0b11 - ], - - /// Inner cacheability attribute for memory associated with translation - /// table walks using TTBR0_EL1. - /// - /// 00 Normal memory, Inner Non-cacheable - /// - /// 01 Normal memory, Inner Write-Back Read-Allocate Write-Allocate - /// Cacheable - /// - /// 10 Normal memory, Inner Write-Through Read-Allocate No - /// Write-Allocate Cacheable - /// - /// 11 Normal memory, Inner Write-Back Read-Allocate No Write-Allocate - /// Cacheable - IRGN0 OFFSET(8) NUMBITS(2) [ - NonCacheable = 0b00, - WriteBack_ReadAlloc_WriteAlloc_Cacheable = 0b01, - WriteThrough_ReadAlloc_NoWriteAlloc_Cacheable = 0b10, - WriteBack_ReadAlloc_NoWriteAlloc_Cacheable = 0b11 - ], - - /// Translation table walk disable for translations using - /// TTBR0_EL1. This bit controls whether a translation table walk is - /// performed on a TLB miss, for an address that is translated using - /// TTBR0_EL1. The encoding of this bit is: - /// - /// 0 Perform translation table walks using TTBR0_EL1. - /// - /// 1 A TLB miss on an address that is translated using TTBR0_EL1 - /// generates a Translation fault. No translation table walk is - /// performed. - EPD0 OFFSET(7) NUMBITS(1) [ - EnableTTBR0Walks = 0, - DisableTTBR0Walks = 1 - ], - - /// The size offset of the memory region addressed by TTBR0_EL1. The - /// region size is 2^(64-T0SZ) bytes. - /// - /// The maximum and minimum possible values for T0SZ depend on the level - /// of translation table and the memory translation granule size, as - /// described in the AArch64 Virtual Memory System Architecture chapter. - T0SZ OFFSET(0) NUMBITS(6) [] - ] -} - -pub struct Reg; - -impl RegisterReadWrite for Reg { - sys_coproc_read_raw!(u64, "TCR_EL1"); - sys_coproc_write_raw!(u64, "TCR_EL1"); -} - -pub static TCR_EL1: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/ttbr0_el1.rs b/crate/aarch64/src/regs/ttbr0_el1.rs deleted file mode 100644 index a29ff79..0000000 --- a/crate/aarch64/src/regs/ttbr0_el1.rs +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (c) 2018 by the author(s) - * - * ============================================================================= - * - * Licensed under either of - * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) - * - MIT License (http://opensource.org/licenses/MIT) - * at your option. - * - * ============================================================================= - * - * Author(s): - * - Andre Richter - */ - -//! Translation Table Base Register 0 - EL1 -//! -//! Holds the base address of the translation table for the initial lookup for -//! stage 1 of the translation of an address from the lower VA range in the -//! EL1&0 translation regime, and other information for this translation regime. - -use register::cpu::RegisterReadWrite; - -register_bitfields! {u64, - TTBR0_EL1 [ - /// An ASID for the translation table base address. The TCR_EL1.A1 field - /// selects either TTBR0_EL1.ASID or TTBR1_EL1.ASID. - /// - /// If the implementation has only 8 bits of ASID, then the upper 8 bits - /// of this field are RES 0. - ASID OFFSET(48) NUMBITS(16) [], - - /// Translation table base address - BADDR OFFSET(1) NUMBITS(47) [], - - /// Common not Private - CnP OFFSET(0) NUMBITS(1) [] - ] -} - -pub struct Reg; - -impl RegisterReadWrite for Reg { - sys_coproc_read_raw!(u64, "TTBR0_EL1"); - sys_coproc_write_raw!(u64, "TTBR0_EL1"); -} - -impl Reg { - #[inline] - pub fn get_baddr(&self) -> u64 { - self.read(TTBR0_EL1::BADDR) << 1 - } - - #[inline] - pub fn set_baddr(&self, addr: u64) { - self.write(TTBR0_EL1::BADDR.val(addr >> 1)); - } -} - -pub static TTBR0_EL1: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/ttbr1_el1.rs b/crate/aarch64/src/regs/ttbr1_el1.rs deleted file mode 100644 index 7df383c..0000000 --- a/crate/aarch64/src/regs/ttbr1_el1.rs +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (c) 2018 by the author(s) - * - * ============================================================================= - * - * Licensed under either of - * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) - * - MIT License (http://opensource.org/licenses/MIT) - * at your option. - * - * ============================================================================= - * - * Author(s): - * - Andre Richter - */ - -//! Translation Table Base Register 1 - EL1 -//! -//! Holds the base address of the translation table for the initial lookup for -//! stage 1 of the translation of an address from the upper VA range in the -//! EL1&0 translation regime, and other information for this translation regime. - -use register::cpu::RegisterReadWrite; - -register_bitfields! {u64, - TTBR1_EL1 [ - /// An ASID for the translation table base address. The TCR_EL1.A1 field - /// selects either TTBR0_EL1.ASID or TTBR1_EL1.ASID. - /// - /// If the implementation has only 8 bits of ASID, then the upper 8 bits - /// of this field are RES 0. - ASID OFFSET(48) NUMBITS(16) [], - - /// Translation table base address - BADDR OFFSET(1) NUMBITS(47) [], - - /// Common not Private - CnP OFFSET(0) NUMBITS(1) [] - ] -} - -pub struct Reg; - -impl RegisterReadWrite for Reg { - sys_coproc_read_raw!(u64, "TTBR1_EL1"); - sys_coproc_write_raw!(u64, "TTBR1_EL1"); -} - -impl Reg { - #[inline] - pub fn get_baddr(&self) -> u64 { - self.read(TTBR1_EL1::BADDR) << 1 - } - - #[inline] - pub fn set_baddr(&self, addr: u64) { - self.write(TTBR1_EL1::BADDR.val(addr >> 1)); - } -} - -pub static TTBR1_EL1: Reg = Reg {}; diff --git a/kernel/Cargo.lock b/kernel/Cargo.lock index 4e2b6dc..d3dd0be 100644 --- a/kernel/Cargo.lock +++ b/kernel/Cargo.lock @@ -1,6 +1,7 @@ [[package]] name = "aarch64" -version = "0.1.0" +version = "2.2.2" +source = "git+https://github.com/equation314/aarch64#47bf5439f5a1379f0fef6272853cf684207a4e45" dependencies = [ "bare-metal 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", "bit_field 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -256,7 +257,7 @@ dependencies = [ name = "ucore" version = "0.1.0" dependencies = [ - "aarch64 0.1.0", + "aarch64 2.2.2 (git+https://github.com/equation314/aarch64)", "atags 0.1.0", "bbl 0.1.0", "bcm2837 0.1.0", @@ -362,6 +363,7 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" [metadata] +"checksum aarch64 2.2.2 (git+https://github.com/equation314/aarch64)" = "" "checksum bare-metal 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "1bdcf9294ed648c7cd29b11db06ea244005aeef50ae8f605b1a3af2940bf8f92" "checksum bit-vec 0.5.0 (git+https://github.com/AltSysrq/bit-vec.git)" = "" "checksum bit_field 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ed8765909f9009617974ab6b7d332625b320b33c326b1e9321382ef1999b5d56" diff --git a/kernel/Cargo.toml b/kernel/Cargo.toml index 20b696f..316b21c 100644 --- a/kernel/Cargo.toml +++ b/kernel/Cargo.toml @@ -44,7 +44,7 @@ riscv = { path = "../crate/riscv" } bbl = { path = "../crate/bbl" } [target.'cfg(target_arch = "aarch64")'.dependencies] -aarch64 = { path = "../crate/aarch64" } +aarch64 = { git = "https://github.com/equation314/aarch64" } atags = { path = "../crate/atags" } bcm2837 = { path = "../crate/bcm2837", features = ["use_generic_timer"] } From 4f592336ff3db6cc4e67d11128b037f98f3c8d16 Mon Sep 17 00:00:00 2001 From: equation314 Date: Sat, 1 Dec 2018 22:39:21 +0800 Subject: [PATCH 20/22] aarch64: remove test functions --- kernel/src/fs.rs | 56 ----------------------------------- kernel/src/lib.rs | 16 +--------- kernel/src/process/context.rs | 9 ------ 3 files changed, 1 insertion(+), 80 deletions(-) diff --git a/kernel/src/fs.rs b/kernel/src/fs.rs index 89e32aa..d25501b 100644 --- a/kernel/src/fs.rs +++ b/kernel/src/fs.rs @@ -40,62 +40,6 @@ pub fn show_logo() { println!("{}", LOGO); } -#[inline(always)] -fn sys_call(id: usize, arg0: usize, arg1: usize, arg2: usize, arg3: usize, arg4: usize, arg5: usize) -> i32 { - let ret: i32; - unsafe { - #[cfg(target_arch = "riscv32")] - asm!("ecall" - : "={x10}" (ret) - : "{x10}" (id), "{x11}" (arg0), "{x12}" (arg1), "{x13}" (arg2), "{x14}" (arg3), "{x15}" (arg4), "{x16}" (arg5) - : "memory" - : "volatile"); - #[cfg(target_arch = "x86_64")] - asm!("int 0x40" - : "={rax}" (ret) - : "{rax}" (id), "{rdi}" (arg0), "{rsi}" (arg1), "{rdx}" (arg2), "{rcx}" (arg3), "{r8}" (arg4), "{r9}" (arg5) - : "memory" - : "intel" "volatile"); - #[cfg(target_arch = "aarch64")] - asm!("svc 0" - : "={x0}" (ret) - : "{x8}" (id), "{x0}" (arg0), "{x1}" (arg1), "{x2}" (arg2), "{x3}" (arg3), "{x4}" (arg4), "{x5}" (arg5) - : "memory" - : "volatile"); - } - ret -} - -pub fn test_shell(prefix: &str) -> ! { - show_logo(); - loop { - print!("{}", prefix); - loop { - let c = super::arch::io::getchar(); - match c { - '\u{7f}' => { - print!("\u{7f}"); - } - 'c' => unsafe { - print!("sys_putc: "); - sys_call(30, 'A' as usize, 0, 0, 0, 0, 0); - }, - 't' => unsafe { - println!("sys_get_time: {}", sys_call(17, 0, 0, 0, 0, 0, 0)); - }, - ' '...'\u{7e}' => { - print!("{}", c); - } - '\n' | '\r' => { - print!("\n"); - break; - } - _ => {} - } - } - } -} - pub fn shell() { show_logo(); diff --git a/kernel/src/lib.rs b/kernel/src/lib.rs index 063b1d0..38bd97c 100644 --- a/kernel/src/lib.rs +++ b/kernel/src/lib.rs @@ -68,14 +68,9 @@ pub mod arch; pub fn kmain() -> ! { process::init(); - - use process::*; - processor().add(Context::new_kernel(kernel_proc2, 2333)); - processor().add(Context::new_user_test(kernel_proc3)); - unsafe { arch::interrupt::enable(); } - // fs::shell(); + fs::shell(); // thread::test::local_key(); // thread::test::unpack(); @@ -93,12 +88,3 @@ pub fn kmain() -> ! { /// It should be defined in memory mod, but in Rust `global_allocator` must be in root mod. #[global_allocator] static HEAP_ALLOCATOR: LockedHeap = LockedHeap::empty(); - - -pub extern "C" fn kernel_proc2(arg: usize) -> ! { - fs::test_shell(&format!("proc2-{}>> ", arg)); -} - -pub extern "C" fn kernel_proc3(arg: usize) -> ! { - fs::test_shell(&format!("proc3-{}$ ", arg)); -} diff --git a/kernel/src/process/context.rs b/kernel/src/process/context.rs index 088b1b8..2df7a58 100644 --- a/kernel/src/process/context.rs +++ b/kernel/src/process/context.rs @@ -33,15 +33,6 @@ impl Context { } } - pub fn new_user_test(entry: extern fn(usize) -> !) -> Self { - let ms = MemorySet::new(); - let user_stack = ::memory::alloc_stack(); - Context { - arch: unsafe { ArchContext::new_user_thread(entry as usize, user_stack.top - 8, ms.kstack_top(), false, ms.token()) }, - memory_set: ms, - } - } - /// Make a new user thread from ELF data pub fn new_user(data: &[u8]) -> Self { // Parse elf From 7855b70bc0ac147cc1ead2034dd29bb2a92370af Mon Sep 17 00:00:00 2001 From: equation314 Date: Sat, 1 Dec 2018 23:30:53 +0800 Subject: [PATCH 21/22] aarch64: add origin raspberrypi config.txt --- kernel/Makefile | 4 +- tools/raspi-firmware/config.txt | 56 +++++++++++++++++++ {riscv-pk => tools/riscv-pk}/.gitignore | 0 {riscv-pk => tools/riscv-pk}/LICENSE | 0 {riscv-pk => tools/riscv-pk}/Makefile.in | 0 {riscv-pk => tools/riscv-pk}/README.md | 0 {riscv-pk => tools/riscv-pk}/aclocal.m4 | 0 {riscv-pk => tools/riscv-pk}/bbl/bbl.ac | 0 {riscv-pk => tools/riscv-pk}/bbl/bbl.c | 0 {riscv-pk => tools/riscv-pk}/bbl/bbl.h | 0 {riscv-pk => tools/riscv-pk}/bbl/bbl.lds | 0 {riscv-pk => tools/riscv-pk}/bbl/bbl.mk.in | 0 {riscv-pk => tools/riscv-pk}/bbl/logo.c | 0 {riscv-pk => tools/riscv-pk}/bbl/payload.S | 0 {riscv-pk => tools/riscv-pk}/bbl/raw_logo.S | 0 .../riscv-pk}/bbl/riscv_logo.txt | 0 {riscv-pk => tools/riscv-pk}/config.h.in | 0 {riscv-pk => tools/riscv-pk}/configure | 0 {riscv-pk => tools/riscv-pk}/configure.ac | 0 .../riscv-pk}/dummy_payload/dummy_entry.S | 0 .../riscv-pk}/dummy_payload/dummy_payload.ac | 0 .../riscv-pk}/dummy_payload/dummy_payload.c | 0 .../riscv-pk}/dummy_payload/dummy_payload.lds | 0 .../dummy_payload/dummy_payload.mk.in | 0 {riscv-pk => tools/riscv-pk}/machine/atomic.h | 0 {riscv-pk => tools/riscv-pk}/machine/bits.h | 0 .../riscv-pk}/machine/disabled_hart_mask.h | 0 .../riscv-pk}/machine/emulation.c | 0 .../riscv-pk}/machine/emulation.h | 0 .../riscv-pk}/machine/encoding.h | 0 {riscv-pk => tools/riscv-pk}/machine/fdt.c | 0 {riscv-pk => tools/riscv-pk}/machine/fdt.h | 0 .../riscv-pk}/machine/finisher.c | 0 .../riscv-pk}/machine/finisher.h | 0 .../riscv-pk}/machine/flush_icache.c | 0 {riscv-pk => tools/riscv-pk}/machine/htif.c | 0 {riscv-pk => tools/riscv-pk}/machine/htif.h | 0 .../riscv-pk}/machine/machine.ac | 0 .../riscv-pk}/machine/machine.mk.in | 0 {riscv-pk => tools/riscv-pk}/machine/mcall.h | 0 {riscv-pk => tools/riscv-pk}/machine/mentry.S | 0 {riscv-pk => tools/riscv-pk}/machine/minit.c | 0 .../riscv-pk}/machine/misaligned_ldst.c | 0 {riscv-pk => tools/riscv-pk}/machine/mtrap.c | 0 {riscv-pk => tools/riscv-pk}/machine/mtrap.h | 0 .../riscv-pk}/machine/muldiv_emulation.c | 0 {riscv-pk => tools/riscv-pk}/machine/uart.c | 0 {riscv-pk => tools/riscv-pk}/machine/uart.h | 0 .../riscv-pk}/machine/uart16550.c | 0 .../riscv-pk}/machine/uart16550.h | 0 .../riscv-pk}/machine/unprivileged_memory.h | 0 {riscv-pk => tools/riscv-pk}/machine/vm.h | 0 .../riscv-pk}/scripts/config.guess | 0 .../riscv-pk}/scripts/config.sub | 0 .../riscv-pk}/scripts/install.sh | 0 .../riscv-pk}/scripts/mk-install-dirs.sh | 0 .../riscv-pk}/scripts/vcs-version.sh | 0 {riscv-pk => tools/riscv-pk}/util/snprintf.c | 0 {riscv-pk => tools/riscv-pk}/util/string.c | 0 {riscv-pk => tools/riscv-pk}/util/util.ac | 0 {riscv-pk => tools/riscv-pk}/util/util.mk.in | 0 61 files changed, 58 insertions(+), 2 deletions(-) create mode 100755 tools/raspi-firmware/config.txt rename {riscv-pk => tools/riscv-pk}/.gitignore (100%) rename {riscv-pk => tools/riscv-pk}/LICENSE (100%) rename {riscv-pk => tools/riscv-pk}/Makefile.in (100%) rename {riscv-pk => tools/riscv-pk}/README.md (100%) rename {riscv-pk => tools/riscv-pk}/aclocal.m4 (100%) rename {riscv-pk => tools/riscv-pk}/bbl/bbl.ac (100%) rename {riscv-pk => tools/riscv-pk}/bbl/bbl.c (100%) rename {riscv-pk => tools/riscv-pk}/bbl/bbl.h (100%) rename {riscv-pk => tools/riscv-pk}/bbl/bbl.lds (100%) rename {riscv-pk => tools/riscv-pk}/bbl/bbl.mk.in (100%) rename {riscv-pk => tools/riscv-pk}/bbl/logo.c (100%) rename {riscv-pk => tools/riscv-pk}/bbl/payload.S (100%) rename {riscv-pk => tools/riscv-pk}/bbl/raw_logo.S (100%) rename {riscv-pk => tools/riscv-pk}/bbl/riscv_logo.txt (100%) rename {riscv-pk => tools/riscv-pk}/config.h.in (100%) rename {riscv-pk => tools/riscv-pk}/configure (100%) rename {riscv-pk => tools/riscv-pk}/configure.ac (100%) rename {riscv-pk => tools/riscv-pk}/dummy_payload/dummy_entry.S (100%) rename {riscv-pk => tools/riscv-pk}/dummy_payload/dummy_payload.ac (100%) rename {riscv-pk => tools/riscv-pk}/dummy_payload/dummy_payload.c (100%) rename {riscv-pk => tools/riscv-pk}/dummy_payload/dummy_payload.lds (100%) rename {riscv-pk => tools/riscv-pk}/dummy_payload/dummy_payload.mk.in (100%) rename {riscv-pk => tools/riscv-pk}/machine/atomic.h (100%) rename {riscv-pk => tools/riscv-pk}/machine/bits.h (100%) rename {riscv-pk => tools/riscv-pk}/machine/disabled_hart_mask.h (100%) rename {riscv-pk => tools/riscv-pk}/machine/emulation.c (100%) rename {riscv-pk => tools/riscv-pk}/machine/emulation.h (100%) rename {riscv-pk => tools/riscv-pk}/machine/encoding.h (100%) rename {riscv-pk => tools/riscv-pk}/machine/fdt.c (100%) rename {riscv-pk => tools/riscv-pk}/machine/fdt.h (100%) rename {riscv-pk => tools/riscv-pk}/machine/finisher.c (100%) rename {riscv-pk => tools/riscv-pk}/machine/finisher.h (100%) rename {riscv-pk => tools/riscv-pk}/machine/flush_icache.c (100%) rename {riscv-pk => tools/riscv-pk}/machine/htif.c (100%) rename {riscv-pk => tools/riscv-pk}/machine/htif.h (100%) rename {riscv-pk => tools/riscv-pk}/machine/machine.ac (100%) rename {riscv-pk => tools/riscv-pk}/machine/machine.mk.in (100%) rename {riscv-pk => tools/riscv-pk}/machine/mcall.h (100%) rename {riscv-pk => tools/riscv-pk}/machine/mentry.S (100%) rename {riscv-pk => tools/riscv-pk}/machine/minit.c (100%) rename {riscv-pk => tools/riscv-pk}/machine/misaligned_ldst.c (100%) rename {riscv-pk => tools/riscv-pk}/machine/mtrap.c (100%) rename {riscv-pk => tools/riscv-pk}/machine/mtrap.h (100%) rename {riscv-pk => tools/riscv-pk}/machine/muldiv_emulation.c (100%) rename {riscv-pk => tools/riscv-pk}/machine/uart.c (100%) rename {riscv-pk => tools/riscv-pk}/machine/uart.h (100%) rename {riscv-pk => tools/riscv-pk}/machine/uart16550.c (100%) rename {riscv-pk => tools/riscv-pk}/machine/uart16550.h (100%) rename {riscv-pk => tools/riscv-pk}/machine/unprivileged_memory.h (100%) rename {riscv-pk => tools/riscv-pk}/machine/vm.h (100%) rename {riscv-pk => tools/riscv-pk}/scripts/config.guess (100%) rename {riscv-pk => tools/riscv-pk}/scripts/config.sub (100%) rename {riscv-pk => tools/riscv-pk}/scripts/install.sh (100%) rename {riscv-pk => tools/riscv-pk}/scripts/mk-install-dirs.sh (100%) rename {riscv-pk => tools/riscv-pk}/scripts/vcs-version.sh (100%) rename {riscv-pk => tools/riscv-pk}/util/snprintf.c (100%) rename {riscv-pk => tools/riscv-pk}/util/string.c (100%) rename {riscv-pk => tools/riscv-pk}/util/util.ac (100%) rename {riscv-pk => tools/riscv-pk}/util/util.mk.in (100%) diff --git a/kernel/Makefile b/kernel/Makefile index de0ab82..a00f1f0 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -126,7 +126,7 @@ ifeq ($(arch), riscv32) ifeq ($(board), fpga) @cp $(kernel) $@ else - @cd ../riscv-pk && \ + @cd ../tools/riscv-pk && \ mkdir -p build && \ cd build && \ ../configure \ @@ -136,7 +136,7 @@ else --host=riscv64-unknown-elf \ --with-payload=$(abspath $(kernel)) && \ make && \ - cp bbl ../../kernel/$@ + cp bbl ../../../kernel/$@ endif else ifeq ($(arch), aarch64) $(objcopy) $(kernel) --strip-all -O binary $@ diff --git a/tools/raspi-firmware/config.txt b/tools/raspi-firmware/config.txt new file mode 100755 index 0000000..06364c5 --- /dev/null +++ b/tools/raspi-firmware/config.txt @@ -0,0 +1,56 @@ +# For more options and information see +# http://rpf.io/configtxt +# Some settings may impact device functionality. See link above for details + +# uncomment if you get no picture on HDMI for a default "safe" mode +#hdmi_safe=1 + +# uncomment this if your display has a black border of unused pixels visible +# and your display can output without overscan +#disable_overscan=1 + +# uncomment the following to adjust overscan. Use positive numbers if console +# goes off screen, and negative if there is too much border +#overscan_left=16 +#overscan_right=16 +#overscan_top=16 +#overscan_bottom=16 + +# uncomment to force a console size. By default it will be display's size minus +# overscan. +#framebuffer_width=1280 +#framebuffer_height=720 + +# uncomment if hdmi display is not detected and composite is being output +#hdmi_force_hotplug=1 + +# uncomment to force a specific HDMI mode (this will force VGA) +#hdmi_group=1 +#hdmi_mode=1 + +# uncomment to force a HDMI mode rather than DVI. This can make audio work in +# DMT (computer monitor) modes +#hdmi_drive=2 + +# uncomment to increase signal to HDMI, if you have interference, blanking, or +# no display +#config_hdmi_boost=4 + +# uncomment for composite PAL +#sdtv_mode=2 + +#uncomment to overclock the arm. 700 MHz is the default. +#arm_freq=800 + +# Uncomment some or all of these to enable the optional hardware interfaces +#dtparam=i2c_arm=on +#dtparam=i2s=on +#dtparam=spi=on + +# Uncomment this to enable the lirc-rpi module +#dtoverlay=lirc-rpi + +# Additional overlays and parameters are documented /boot/overlays/README + +# Enable audio (loads snd_bcm2835) +dtparam=audio=on diff --git a/riscv-pk/.gitignore b/tools/riscv-pk/.gitignore similarity index 100% rename from riscv-pk/.gitignore rename to tools/riscv-pk/.gitignore diff --git a/riscv-pk/LICENSE b/tools/riscv-pk/LICENSE similarity index 100% rename from riscv-pk/LICENSE rename to tools/riscv-pk/LICENSE diff --git a/riscv-pk/Makefile.in b/tools/riscv-pk/Makefile.in similarity index 100% rename from riscv-pk/Makefile.in rename to tools/riscv-pk/Makefile.in diff --git a/riscv-pk/README.md b/tools/riscv-pk/README.md similarity index 100% rename from riscv-pk/README.md rename to tools/riscv-pk/README.md diff --git a/riscv-pk/aclocal.m4 b/tools/riscv-pk/aclocal.m4 similarity index 100% rename from riscv-pk/aclocal.m4 rename to tools/riscv-pk/aclocal.m4 diff --git a/riscv-pk/bbl/bbl.ac b/tools/riscv-pk/bbl/bbl.ac similarity index 100% rename from riscv-pk/bbl/bbl.ac rename to tools/riscv-pk/bbl/bbl.ac diff --git a/riscv-pk/bbl/bbl.c b/tools/riscv-pk/bbl/bbl.c similarity index 100% rename from riscv-pk/bbl/bbl.c rename to tools/riscv-pk/bbl/bbl.c diff --git a/riscv-pk/bbl/bbl.h b/tools/riscv-pk/bbl/bbl.h similarity index 100% rename from riscv-pk/bbl/bbl.h rename to tools/riscv-pk/bbl/bbl.h diff --git a/riscv-pk/bbl/bbl.lds b/tools/riscv-pk/bbl/bbl.lds similarity index 100% rename from riscv-pk/bbl/bbl.lds rename to tools/riscv-pk/bbl/bbl.lds diff --git a/riscv-pk/bbl/bbl.mk.in b/tools/riscv-pk/bbl/bbl.mk.in similarity index 100% rename from riscv-pk/bbl/bbl.mk.in rename to tools/riscv-pk/bbl/bbl.mk.in diff --git a/riscv-pk/bbl/logo.c b/tools/riscv-pk/bbl/logo.c similarity index 100% rename from riscv-pk/bbl/logo.c rename to tools/riscv-pk/bbl/logo.c diff --git a/riscv-pk/bbl/payload.S b/tools/riscv-pk/bbl/payload.S similarity index 100% rename from riscv-pk/bbl/payload.S rename to tools/riscv-pk/bbl/payload.S diff --git a/riscv-pk/bbl/raw_logo.S b/tools/riscv-pk/bbl/raw_logo.S similarity index 100% rename from riscv-pk/bbl/raw_logo.S rename to tools/riscv-pk/bbl/raw_logo.S diff --git a/riscv-pk/bbl/riscv_logo.txt b/tools/riscv-pk/bbl/riscv_logo.txt similarity index 100% rename from riscv-pk/bbl/riscv_logo.txt rename to tools/riscv-pk/bbl/riscv_logo.txt diff --git a/riscv-pk/config.h.in b/tools/riscv-pk/config.h.in similarity index 100% rename from riscv-pk/config.h.in rename to tools/riscv-pk/config.h.in diff --git a/riscv-pk/configure b/tools/riscv-pk/configure similarity index 100% rename from riscv-pk/configure rename to tools/riscv-pk/configure diff --git a/riscv-pk/configure.ac b/tools/riscv-pk/configure.ac similarity index 100% rename from riscv-pk/configure.ac rename to tools/riscv-pk/configure.ac diff --git a/riscv-pk/dummy_payload/dummy_entry.S b/tools/riscv-pk/dummy_payload/dummy_entry.S similarity index 100% rename from riscv-pk/dummy_payload/dummy_entry.S rename to tools/riscv-pk/dummy_payload/dummy_entry.S diff --git a/riscv-pk/dummy_payload/dummy_payload.ac b/tools/riscv-pk/dummy_payload/dummy_payload.ac similarity index 100% rename from riscv-pk/dummy_payload/dummy_payload.ac rename to tools/riscv-pk/dummy_payload/dummy_payload.ac diff --git a/riscv-pk/dummy_payload/dummy_payload.c b/tools/riscv-pk/dummy_payload/dummy_payload.c similarity index 100% rename from riscv-pk/dummy_payload/dummy_payload.c rename to tools/riscv-pk/dummy_payload/dummy_payload.c diff --git a/riscv-pk/dummy_payload/dummy_payload.lds b/tools/riscv-pk/dummy_payload/dummy_payload.lds similarity index 100% rename from riscv-pk/dummy_payload/dummy_payload.lds rename to tools/riscv-pk/dummy_payload/dummy_payload.lds diff --git a/riscv-pk/dummy_payload/dummy_payload.mk.in b/tools/riscv-pk/dummy_payload/dummy_payload.mk.in similarity index 100% rename from riscv-pk/dummy_payload/dummy_payload.mk.in rename to tools/riscv-pk/dummy_payload/dummy_payload.mk.in diff --git a/riscv-pk/machine/atomic.h b/tools/riscv-pk/machine/atomic.h similarity index 100% rename from riscv-pk/machine/atomic.h rename to tools/riscv-pk/machine/atomic.h diff --git a/riscv-pk/machine/bits.h b/tools/riscv-pk/machine/bits.h similarity index 100% rename from riscv-pk/machine/bits.h rename to tools/riscv-pk/machine/bits.h diff --git a/riscv-pk/machine/disabled_hart_mask.h b/tools/riscv-pk/machine/disabled_hart_mask.h similarity index 100% rename from riscv-pk/machine/disabled_hart_mask.h rename to tools/riscv-pk/machine/disabled_hart_mask.h diff --git a/riscv-pk/machine/emulation.c b/tools/riscv-pk/machine/emulation.c similarity index 100% rename from riscv-pk/machine/emulation.c rename to tools/riscv-pk/machine/emulation.c diff --git a/riscv-pk/machine/emulation.h b/tools/riscv-pk/machine/emulation.h similarity index 100% rename from riscv-pk/machine/emulation.h rename to tools/riscv-pk/machine/emulation.h diff --git a/riscv-pk/machine/encoding.h b/tools/riscv-pk/machine/encoding.h similarity index 100% rename from riscv-pk/machine/encoding.h rename to tools/riscv-pk/machine/encoding.h diff --git a/riscv-pk/machine/fdt.c b/tools/riscv-pk/machine/fdt.c similarity index 100% rename from riscv-pk/machine/fdt.c rename to tools/riscv-pk/machine/fdt.c diff --git a/riscv-pk/machine/fdt.h b/tools/riscv-pk/machine/fdt.h similarity index 100% rename from riscv-pk/machine/fdt.h rename to tools/riscv-pk/machine/fdt.h diff --git a/riscv-pk/machine/finisher.c b/tools/riscv-pk/machine/finisher.c similarity index 100% rename from riscv-pk/machine/finisher.c rename to tools/riscv-pk/machine/finisher.c diff --git a/riscv-pk/machine/finisher.h b/tools/riscv-pk/machine/finisher.h similarity index 100% rename from riscv-pk/machine/finisher.h rename to tools/riscv-pk/machine/finisher.h diff --git a/riscv-pk/machine/flush_icache.c b/tools/riscv-pk/machine/flush_icache.c similarity index 100% rename from riscv-pk/machine/flush_icache.c rename to tools/riscv-pk/machine/flush_icache.c diff --git a/riscv-pk/machine/htif.c b/tools/riscv-pk/machine/htif.c similarity index 100% rename from riscv-pk/machine/htif.c rename to tools/riscv-pk/machine/htif.c diff --git a/riscv-pk/machine/htif.h b/tools/riscv-pk/machine/htif.h similarity index 100% rename from riscv-pk/machine/htif.h rename to tools/riscv-pk/machine/htif.h diff --git a/riscv-pk/machine/machine.ac b/tools/riscv-pk/machine/machine.ac similarity index 100% rename from riscv-pk/machine/machine.ac rename to tools/riscv-pk/machine/machine.ac diff --git a/riscv-pk/machine/machine.mk.in b/tools/riscv-pk/machine/machine.mk.in similarity index 100% rename from riscv-pk/machine/machine.mk.in rename to tools/riscv-pk/machine/machine.mk.in diff --git a/riscv-pk/machine/mcall.h b/tools/riscv-pk/machine/mcall.h similarity index 100% rename from riscv-pk/machine/mcall.h rename to tools/riscv-pk/machine/mcall.h diff --git a/riscv-pk/machine/mentry.S b/tools/riscv-pk/machine/mentry.S similarity index 100% rename from riscv-pk/machine/mentry.S rename to tools/riscv-pk/machine/mentry.S diff --git a/riscv-pk/machine/minit.c b/tools/riscv-pk/machine/minit.c similarity index 100% rename from riscv-pk/machine/minit.c rename to tools/riscv-pk/machine/minit.c diff --git a/riscv-pk/machine/misaligned_ldst.c b/tools/riscv-pk/machine/misaligned_ldst.c similarity index 100% rename from riscv-pk/machine/misaligned_ldst.c rename to tools/riscv-pk/machine/misaligned_ldst.c diff --git a/riscv-pk/machine/mtrap.c b/tools/riscv-pk/machine/mtrap.c similarity index 100% rename from riscv-pk/machine/mtrap.c rename to tools/riscv-pk/machine/mtrap.c diff --git a/riscv-pk/machine/mtrap.h b/tools/riscv-pk/machine/mtrap.h similarity index 100% rename from riscv-pk/machine/mtrap.h rename to tools/riscv-pk/machine/mtrap.h diff --git a/riscv-pk/machine/muldiv_emulation.c b/tools/riscv-pk/machine/muldiv_emulation.c similarity index 100% rename from riscv-pk/machine/muldiv_emulation.c rename to tools/riscv-pk/machine/muldiv_emulation.c diff --git a/riscv-pk/machine/uart.c b/tools/riscv-pk/machine/uart.c similarity index 100% rename from riscv-pk/machine/uart.c rename to tools/riscv-pk/machine/uart.c diff --git a/riscv-pk/machine/uart.h b/tools/riscv-pk/machine/uart.h similarity index 100% rename from riscv-pk/machine/uart.h rename to tools/riscv-pk/machine/uart.h diff --git a/riscv-pk/machine/uart16550.c b/tools/riscv-pk/machine/uart16550.c similarity index 100% rename from riscv-pk/machine/uart16550.c rename to tools/riscv-pk/machine/uart16550.c diff --git a/riscv-pk/machine/uart16550.h b/tools/riscv-pk/machine/uart16550.h similarity index 100% rename from riscv-pk/machine/uart16550.h rename to tools/riscv-pk/machine/uart16550.h diff --git a/riscv-pk/machine/unprivileged_memory.h b/tools/riscv-pk/machine/unprivileged_memory.h similarity index 100% rename from riscv-pk/machine/unprivileged_memory.h rename to tools/riscv-pk/machine/unprivileged_memory.h diff --git a/riscv-pk/machine/vm.h b/tools/riscv-pk/machine/vm.h similarity index 100% rename from riscv-pk/machine/vm.h rename to tools/riscv-pk/machine/vm.h diff --git a/riscv-pk/scripts/config.guess b/tools/riscv-pk/scripts/config.guess similarity index 100% rename from riscv-pk/scripts/config.guess rename to tools/riscv-pk/scripts/config.guess diff --git a/riscv-pk/scripts/config.sub b/tools/riscv-pk/scripts/config.sub similarity index 100% rename from riscv-pk/scripts/config.sub rename to tools/riscv-pk/scripts/config.sub diff --git a/riscv-pk/scripts/install.sh b/tools/riscv-pk/scripts/install.sh similarity index 100% rename from riscv-pk/scripts/install.sh rename to tools/riscv-pk/scripts/install.sh diff --git a/riscv-pk/scripts/mk-install-dirs.sh b/tools/riscv-pk/scripts/mk-install-dirs.sh similarity index 100% rename from riscv-pk/scripts/mk-install-dirs.sh rename to tools/riscv-pk/scripts/mk-install-dirs.sh diff --git a/riscv-pk/scripts/vcs-version.sh b/tools/riscv-pk/scripts/vcs-version.sh similarity index 100% rename from riscv-pk/scripts/vcs-version.sh rename to tools/riscv-pk/scripts/vcs-version.sh diff --git a/riscv-pk/util/snprintf.c b/tools/riscv-pk/util/snprintf.c similarity index 100% rename from riscv-pk/util/snprintf.c rename to tools/riscv-pk/util/snprintf.c diff --git a/riscv-pk/util/string.c b/tools/riscv-pk/util/string.c similarity index 100% rename from riscv-pk/util/string.c rename to tools/riscv-pk/util/string.c diff --git a/riscv-pk/util/util.ac b/tools/riscv-pk/util/util.ac similarity index 100% rename from riscv-pk/util/util.ac rename to tools/riscv-pk/util/util.ac diff --git a/riscv-pk/util/util.mk.in b/tools/riscv-pk/util/util.mk.in similarity index 100% rename from riscv-pk/util/util.mk.in rename to tools/riscv-pk/util/util.mk.in From 829b7b6b135019db6871c8cc7cd7618fda2f5b00 Mon Sep 17 00:00:00 2001 From: equation314 Date: Sat, 1 Dec 2018 23:31:53 +0800 Subject: [PATCH 22/22] aarch64: update raspi-firmware/config.txt --- tools/raspi-firmware/config.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/raspi-firmware/config.txt b/tools/raspi-firmware/config.txt index 06364c5..89ebfb4 100755 --- a/tools/raspi-firmware/config.txt +++ b/tools/raspi-firmware/config.txt @@ -54,3 +54,6 @@ # Enable audio (loads snd_bcm2835) dtparam=audio=on + +kernel=kernel8.img +device_tree=