From 52251b2adb12319cbaef6906223fd639eb96a295 Mon Sep 17 00:00:00 2001 From: equation314 Date: Mon, 20 May 2019 21:51:50 +0800 Subject: [PATCH] aarch64: refactor to linear mapping TODO: * `map`, `unmap` is significantly slower than other archs * set segment permissions for kernel page table in bootloader --- bootloader/Cargo.lock | 15 +- bootloader/Cargo.toml | 2 +- bootloader/src/arch/aarch64/mod.rs | 39 ++-- kernel/Cargo.lock | 11 +- kernel/Cargo.toml | 2 +- kernel/src/arch/aarch64/board/raspi3/mod.rs | 6 +- kernel/src/arch/aarch64/consts.rs | 8 +- kernel/src/arch/aarch64/cpu.rs | 9 +- kernel/src/arch/aarch64/memory.rs | 83 +-------- kernel/src/arch/aarch64/paging.rs | 193 +++++++++----------- 10 files changed, 147 insertions(+), 221 deletions(-) diff --git a/bootloader/Cargo.lock b/bootloader/Cargo.lock index 4eaba05..2b8bee7 100644 --- a/bootloader/Cargo.lock +++ b/bootloader/Cargo.lock @@ -2,11 +2,12 @@ # It is not intended for manual editing. [[package]] name = "aarch64" -version = "2.5.0" -source = "git+https://github.com/rcore-os/aarch64#797c24f07f9d90542eb094530b6f63fe3ea7dded" +version = "2.6.0" +source = "git+https://github.com/rcore-os/aarch64#72d8ac8fdf6e41c453cda8f9fd144ae2798b579c" dependencies = [ "bit_field 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", + "cast 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "register 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "usize_conversions 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "ux 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", @@ -30,6 +31,11 @@ name = "bitflags" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "cast" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "cc" version = "1.0.31" @@ -98,7 +104,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" name = "rcore-bootloader" version = "0.1.0" dependencies = [ - "aarch64 2.5.0 (git+https://github.com/rcore-os/aarch64)", + "aarch64 2.6.0 (git+https://github.com/rcore-os/aarch64)", "bcm2837 1.0.0 (git+https://github.com/rcore-os/bcm2837)", "cc 1.0.31 (registry+https://github.com/rust-lang/crates.io-index)", "fixedvec 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", @@ -205,10 +211,11 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" [metadata] -"checksum aarch64 2.5.0 (git+https://github.com/rcore-os/aarch64)" = "" +"checksum aarch64 2.6.0 (git+https://github.com/rcore-os/aarch64)" = "" "checksum bcm2837 1.0.0 (git+https://github.com/rcore-os/bcm2837)" = "" "checksum bit_field 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ed8765909f9009617974ab6b7d332625b320b33c326b1e9321382ef1999b5d56" "checksum bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "228047a76f468627ca71776ecdebd732a3423081fcf5125585bcd7c49886ce12" +"checksum cast 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "926013f2860c46252efceabb19f4a6b308197505082c609025aa6706c011d427" "checksum cc 1.0.31 (registry+https://github.com/rust-lang/crates.io-index)" = "c9ce8bb087aacff865633f0bd5aeaed910fe2fe55b55f4739527f2e023a2e53d" "checksum fixedvec 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "7c6c16d316ccdac21a4dd648e314e76facbbaf316e83ca137d0857a9c07419d0" "checksum fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" diff --git a/bootloader/Cargo.toml b/bootloader/Cargo.toml index 7fca683..f3f3430 100644 --- a/bootloader/Cargo.toml +++ b/bootloader/Cargo.toml @@ -9,7 +9,7 @@ xmas-elf = "0.6.2" fixedvec = "0.2.3" [target.'cfg(target_arch = "aarch64")'.dependencies] -aarch64 = { git = "https://github.com/rcore-os/aarch64", version = "2.5.0" } +aarch64 = { git = "https://github.com/rcore-os/aarch64", version = "2.6.0" } bcm2837 = { git = "https://github.com/rcore-os/bcm2837", version = "1.0.0" } [build-dependencies] diff --git a/bootloader/src/arch/aarch64/mod.rs b/bootloader/src/arch/aarch64/mod.rs index 4a03460..5a70c32 100644 --- a/bootloader/src/arch/aarch64/mod.rs +++ b/bootloader/src/arch/aarch64/mod.rs @@ -1,6 +1,6 @@ -use aarch64::addr::{VirtAddr, PhysAddr}; +use aarch64::addr::{PhysAddr, VirtAddr}; use aarch64::paging::{memory_attribute::*, Page, PageTable, PageTableFlags as EF, PhysFrame}; -use aarch64::paging::{Size4KiB, Size2MiB, Size1GiB}; +use aarch64::paging::{Size1GiB, Size2MiB, Size4KiB}; use aarch64::{asm::*, barrier, regs::*}; use bcm2837::consts::RAW_IO_BASE; use core::ptr; @@ -10,12 +10,22 @@ use xmas_elf::program::{ProgramHeader64, Type}; const PAGE_SIZE: usize = 4096; const ALIGN_2MB: u64 = 0x200000; -const RECURSIVE_INDEX: usize = 0o777; -const KERNEL_OFFSET: u64 = 0xFFFF_0000_0000_0000; +const PHYSICAL_MEMORY_OFFSET: u64 = 0xFFFF_0000_0000_0000; global_asm!(include_str!("boot.S")); -fn setup_temp_page_table(start_vaddr: VirtAddr, end_vaddr: VirtAddr, offset: u64) { +/// Convert physical address to virtual address +const fn phys_to_virt(paddr: u64) -> u64 { + PHYSICAL_MEMORY_OFFSET + paddr +} + +/// Convert virtual address to physical address +const fn virt_to_phys(vaddr: u64) -> u64 { + vaddr - PHYSICAL_MEMORY_OFFSET +} + +// TODO: set segments permission +fn create_page_table(start_paddr: usize, end_paddr: usize) { #[repr(align(4096))] struct PageData([u8; PAGE_SIZE]); static mut PAGE_TABLE_LVL4: PageData = PageData([0; PAGE_SIZE]); @@ -34,13 +44,17 @@ fn setup_temp_page_table(start_vaddr: VirtAddr, end_vaddr: VirtAddr, offset: u64 let block_flags = EF::VALID | EF::AF | EF::WRITE | EF::UXN; // normal memory - for page in Page::::range_of(start_vaddr.as_u64(), end_vaddr.as_u64()) { - let paddr = PhysAddr::new(page.start_address().as_u64().wrapping_add(offset)); + for frame in PhysFrame::::range_of(start_paddr as u64, end_paddr as u64) { + let paddr = frame.start_address(); + let vaddr = VirtAddr::new(phys_to_virt(paddr.as_u64())); + let page = Page::::containing_address(vaddr); p2[page.p2_index()].set_block::(paddr, block_flags, MairNormal::attr_value()); } // device memory - for page in Page::::range_of(RAW_IO_BASE as u64, 0x4000_0000) { - let paddr = PhysAddr::new(page.start_address().as_u64()); + for frame in PhysFrame::::range_of(RAW_IO_BASE as u64, 0x4000_0000) { + let paddr = frame.start_address(); + let vaddr = VirtAddr::new(phys_to_virt(paddr.as_u64())); + let page = Page::::containing_address(vaddr); p2[page.p2_index()].set_block::(paddr, block_flags | EF::PXN, MairDevice::attr_value()); } @@ -48,8 +62,9 @@ fn setup_temp_page_table(start_vaddr: VirtAddr, end_vaddr: VirtAddr, offset: u64 p3[1].set_block::(PhysAddr::new(0x4000_0000), block_flags | EF::PXN, MairDevice::attr_value()); p4[0].set_frame(frame_lvl3, EF::default(), MairNormal::attr_value()); - p4[RECURSIVE_INDEX].set_frame(frame_lvl4, EF::default(), MairNormal::attr_value()); + // the bootloader is still running at the lower virtual address range, + // so the TTBR0_EL1 also needs to be set. ttbr_el1_write(0, frame_lvl4); ttbr_el1_write(1, frame_lvl4); tlb_invalidate_all(); @@ -118,7 +133,7 @@ pub fn map_kernel(kernel_start: usize, segments: &FixedVec) { unsafe { let src = (kernel_start as u64 + offset) as *const u8; - let dst = virt_addr.wrapping_sub(KERNEL_OFFSET) as *mut u8; + let dst = virt_to_phys(virt_addr) as *mut u8; ptr::copy(src, dst, file_size as usize); ptr::write_bytes(dst.offset(file_size as isize), 0, (mem_size - file_size) as usize); } @@ -131,6 +146,6 @@ pub fn map_kernel(kernel_start: usize, segments: &FixedVec) { } } - setup_temp_page_table(start_vaddr, end_vaddr, KERNEL_OFFSET.wrapping_neg()); + create_page_table(0, RAW_IO_BASE); enable_mmu(); } diff --git a/kernel/Cargo.lock b/kernel/Cargo.lock index 23dea29..678c485 100644 --- a/kernel/Cargo.lock +++ b/kernel/Cargo.lock @@ -2,11 +2,12 @@ # It is not intended for manual editing. [[package]] name = "aarch64" -version = "2.5.0" -source = "git+https://github.com/rcore-os/aarch64#797c24f07f9d90542eb094530b6f63fe3ea7dded" +version = "2.6.0" +source = "git+https://github.com/rcore-os/aarch64#72d8ac8fdf6e41c453cda8f9fd144ae2798b579c" dependencies = [ "bit_field 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", + "cast 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "register 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "usize_conversions 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "ux 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", @@ -43,7 +44,7 @@ name = "bcm2837" version = "1.0.0" source = "git+https://github.com/rcore-os/bcm2837#b29a8db5504b7eaa6f8adf2c3ff916d1ffd15194" dependencies = [ - "aarch64 2.5.0 (git+https://github.com/rcore-os/aarch64)", + "aarch64 2.6.0 (git+https://github.com/rcore-os/aarch64)", "volatile 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -352,7 +353,7 @@ dependencies = [ name = "rcore" version = "0.2.0" dependencies = [ - "aarch64 2.5.0 (git+https://github.com/rcore-os/aarch64)", + "aarch64 2.6.0 (git+https://github.com/rcore-os/aarch64)", "apic 0.1.0 (git+https://github.com/rcore-os/apic-rs)", "bcm2837 1.0.0 (git+https://github.com/rcore-os/bcm2837)", "bit_field 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -658,7 +659,7 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" [metadata] -"checksum aarch64 2.5.0 (git+https://github.com/rcore-os/aarch64)" = "" +"checksum aarch64 2.6.0 (git+https://github.com/rcore-os/aarch64)" = "" "checksum apic 0.1.0 (git+https://github.com/rcore-os/apic-rs)" = "" "checksum array-init 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "23589ecb866b460d3a0f1278834750268c607e8e28a1b982c907219f3178cd72" "checksum bare-metal 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "a3caf393d93b2d453e80638d0674597020cef3382ada454faacd43d1a55a735a" diff --git a/kernel/Cargo.toml b/kernel/Cargo.toml index ba20cf9..f68c68c 100644 --- a/kernel/Cargo.toml +++ b/kernel/Cargo.toml @@ -79,7 +79,7 @@ pc-keyboard = "0.5" riscv = { git = "https://github.com/rcore-os/riscv", features = ["inline-asm"] } [target.'cfg(target_arch = "aarch64")'.dependencies] -aarch64 = { git = "https://github.com/rcore-os/aarch64", version = "2.5.0" } +aarch64 = { git = "https://github.com/rcore-os/aarch64", version = "2.6.0" } bcm2837 = { git = "https://github.com/rcore-os/bcm2837", version = "1.0.0", optional = true } [target.'cfg(target_arch = "mips")'.dependencies] diff --git a/kernel/src/arch/aarch64/board/raspi3/mod.rs b/kernel/src/arch/aarch64/board/raspi3/mod.rs index 3caf828..c7e6551 100644 --- a/kernel/src/arch/aarch64/board/raspi3/mod.rs +++ b/kernel/src/arch/aarch64/board/raspi3/mod.rs @@ -1,6 +1,5 @@ //! Raspberry PI 3 Model B/B+ -use alloc::string::String; use bcm2837::atags::Atags; #[path = "../../../../drivers/gpu/fb.rs"] @@ -10,7 +9,7 @@ pub mod mailbox; pub mod serial; pub mod timer; -use fb::{ColorConfig, FramebufferInfo, FramebufferResult}; +use fb::{ColorConfig, FramebufferResult}; pub const IO_REMAP_BASE: usize = bcm2837::consts::IO_BASE; pub const IO_REMAP_END: usize = bcm2837::consts::KERNEL_OFFSET + 0x4000_1000; @@ -67,9 +66,8 @@ pub fn probe_fb_info(width: u32, height: u32, depth: u32) -> FramebufferResult { ))?; } - use crate::arch::memory; let paddr = info.bus_addr & !0xC0000000; - let vaddr = memory::ioremap(paddr as usize, info.screen_size as usize, "fb"); + let vaddr = crate::memory::phys_to_virt(paddr as usize); if vaddr == 0 { Err(format!( "cannot remap memory range [{:#x?}..{:#x?}]", diff --git a/kernel/src/arch/aarch64/consts.rs b/kernel/src/arch/aarch64/consts.rs index 3a7c500..23cdafe 100644 --- a/kernel/src/arch/aarch64/consts.rs +++ b/kernel/src/arch/aarch64/consts.rs @@ -1,7 +1,7 @@ -pub const RECURSIVE_INDEX: usize = 0o777; +pub const MEMORY_OFFSET: usize = 0; pub const KERNEL_OFFSET: usize = 0xFFFF_0000_0000_0000; -pub const KERNEL_PML4: usize = 0; +pub const PHYSICAL_MEMORY_OFFSET: usize = KERNEL_OFFSET - MEMORY_OFFSET; pub const KERNEL_HEAP_SIZE: usize = 8 * 1024 * 1024; -pub const MEMORY_OFFSET: usize = 0; + pub const USER_STACK_OFFSET: usize = 0x0000_8000_0000_0000 - USER_STACK_SIZE; -pub const USER_STACK_SIZE: usize = 1 * 1024 * 1024; +pub const USER_STACK_SIZE: usize = 8 * 1024 * 1024; diff --git a/kernel/src/arch/aarch64/cpu.rs b/kernel/src/arch/aarch64/cpu.rs index 93457ae..ae22c05 100644 --- a/kernel/src/arch/aarch64/cpu.rs +++ b/kernel/src/arch/aarch64/cpu.rs @@ -1,12 +1,13 @@ +use aarch64::{asm, regs::*}; + pub fn halt() { - unsafe { asm!("wfi" :::: "volatile") } + asm::wfi(); } pub fn id() -> usize { - // TODO: cpu id - 0 + (MPIDR_EL1.get() & 3) as usize } -pub unsafe fn exit_in_qemu(error_code: u8) -> ! { +pub unsafe fn exit_in_qemu(_error_code: u8) -> ! { unimplemented!() } diff --git a/kernel/src/arch/aarch64/memory.rs b/kernel/src/arch/aarch64/memory.rs index 3663596..1101340 100644 --- a/kernel/src/arch/aarch64/memory.rs +++ b/kernel/src/arch/aarch64/memory.rs @@ -1,9 +1,7 @@ //! Memory initialization for aarch64. -use super::paging::MMIOType; -use crate::consts::{KERNEL_OFFSET, MEMORY_OFFSET}; -use crate::memory::{init_heap, Linear, MemoryAttr, MemorySet, FRAME_ALLOCATOR}; -use aarch64::regs::*; +use crate::consts::MEMORY_OFFSET; +use crate::memory::{init_heap, virt_to_phys, FRAME_ALLOCATOR}; use log::*; use rcore_memory::PAGE_SIZE; @@ -11,7 +9,6 @@ use rcore_memory::PAGE_SIZE; pub fn init() { init_frame_allocator(); init_heap(); - remap_the_kernel(); info!("memory: init end"); } @@ -22,7 +19,7 @@ fn init_frame_allocator() { let end = super::board::probe_memory() .expect("failed to find memory map") .1; - let start = (_end as u64 + PAGE_SIZE as u64).wrapping_sub(KERNEL_OFFSET as u64) as usize; + let start = virt_to_phys(_end as usize + PAGE_SIZE); let mut ba = FRAME_ALLOCATOR.lock(); ba.insert(to_range(start, end)); info!("FrameAllocator init end"); @@ -35,79 +32,7 @@ fn init_frame_allocator() { } } -static mut KERNEL_MEMORY_SET: Option = None; - -/// remap kernel page table after all initialization. -fn remap_the_kernel() { - let offset = -(KERNEL_OFFSET as isize); - let mut ms = MemorySet::new_bare(); - ms.push( - stext as usize, - etext as usize, - MemoryAttr::default().execute().readonly(), - Linear::new(offset), - "text", - ); - ms.push( - sdata as usize, - edata as usize, - MemoryAttr::default(), - Linear::new(offset), - "data", - ); - ms.push( - srodata as usize, - erodata as usize, - MemoryAttr::default().readonly(), - Linear::new(offset), - "rodata", - ); - ms.push( - sbss as usize, - ebss as usize, - MemoryAttr::default(), - Linear::new(offset), - "bss", - ); - ms.push( - bootstack as usize, - bootstacktop as usize, - MemoryAttr::default(), - Linear::new(offset), - "kstack", - ); - - use super::board::{IO_REMAP_BASE, IO_REMAP_END}; - ms.push( - IO_REMAP_BASE, - IO_REMAP_END, - MemoryAttr::default().mmio(MMIOType::Device as u8), - Linear::new(offset), - "io_remap", - ); - - info!("{:#x?}", ms); - unsafe { ms.get_page_table_mut().activate_as_kernel() } - unsafe { KERNEL_MEMORY_SET = Some(ms) } - info!("kernel remap end"); -} - -pub fn ioremap(paddr: usize, len: usize, name: &'static str) -> usize { - let offset = -(KERNEL_OFFSET as isize); - let vaddr = paddr.wrapping_add(KERNEL_OFFSET); - if let Some(ms) = unsafe { KERNEL_MEMORY_SET.as_mut() } { - ms.push( - vaddr, - vaddr + len, - MemoryAttr::default().mmio(MMIOType::NormalNonCacheable as u8), - Linear::new(offset), - name, - ); - return vaddr; - } - 0 -} - +#[allow(dead_code)] extern "C" { fn stext(); fn etext(); diff --git a/kernel/src/arch/aarch64/paging.rs b/kernel/src/arch/aarch64/paging.rs index 4d9e4a3..edea1e0 100644 --- a/kernel/src/arch/aarch64/paging.rs +++ b/kernel/src/arch/aarch64/paging.rs @@ -1,58 +1,79 @@ //! Page table implementations for aarch64. + +use crate::memory::{alloc_frame, dealloc_frame, phys_to_virt}; use aarch64::asm::{tlb_invalidate, tlb_invalidate_all, ttbr_el1_read, ttbr_el1_write}; -use aarch64::paging::memory_attribute::*; -use aarch64::paging::{FrameAllocator, FrameDeallocator, Page, PhysFrame as Frame, Size4KiB}; use aarch64::paging::{ - Mapper, PageTable as Aarch64PageTable, PageTableEntry, PageTableFlags as EF, RecursivePageTable, + frame::PhysFrame as Frame, + mapper::{MappedPageTable, Mapper}, + memory_attribute::*, + page_table::{PageTable as Aarch64PageTable, PageTableEntry, PageTableFlags as EF}, + FrameAllocator, FrameDeallocator, Page as PageAllSizes, Size4KiB, }; use aarch64::{PhysAddr, VirtAddr}; use log::*; use rcore_memory::paging::*; -// Depends on kernel -use crate::consts::{KERNEL_OFFSET, KERNEL_PML4, RECURSIVE_INDEX}; -use crate::memory::{active_table, alloc_frame, dealloc_frame}; -pub struct ActivePageTable(RecursivePageTable); +type Page = PageAllSizes; + +pub struct PageTableImpl { + page_table: MappedPageTable<'static, fn(Frame) -> *mut Aarch64PageTable>, + root_frame: Frame, + entry: PageEntry, +} -pub struct PageEntry(PageTableEntry); +pub struct PageEntry(&'static mut PageTableEntry, Page); -impl PageTable for ActivePageTable { +impl PageTable for PageTableImpl { fn map(&mut self, addr: usize, target: usize) -> &mut Entry { let flags = EF::default(); let attr = MairNormal::attr_value(); - self.0 - .map_to( - Page::of_addr(addr as u64), - Frame::of_addr(target as u64), - flags, - attr, - &mut FrameAllocatorForAarch64, - ) - .unwrap() - .flush(); + unsafe { + self.page_table + .map_to( + Page::of_addr(addr as u64), + Frame::of_addr(target as u64), + flags, + attr, + &mut FrameAllocatorForAarch64, + ) + .unwrap() + .flush(); + } self.get_entry(addr).expect("fail to get entry") } fn unmap(&mut self, addr: usize) { - self.0.unmap(Page::of_addr(addr as u64)).unwrap().1.flush(); + self.page_table + .unmap(Page::of_addr(addr as u64)) + .unwrap() + .1 + .flush(); } fn get_entry(&mut self, vaddr: usize) -> Option<&mut Entry> { - // get p1 entry - let entry_addr = - ((vaddr >> 9) & 0o777_777_777_7770) | (RECURSIVE_INDEX << 39) | (vaddr & KERNEL_OFFSET); - Some(unsafe { &mut *(entry_addr as *mut PageEntry) }) + let page = Page::of_addr(vaddr as u64); + if let Ok(e) = self.page_table.get_entry_mut(page) { + let e = unsafe { &mut *(e as *mut PageTableEntry) }; + self.entry = PageEntry(e, page); + Some(&mut self.entry as &mut Entry) + } else { + None + } } -} -impl PageTableExt for ActivePageTable { - const TEMP_PAGE_ADDR: usize = KERNEL_OFFSET | 0xcafeb000; + fn get_page_slice_mut<'a>(&mut self, addr: usize) -> &'a mut [u8] { + let frame = self + .page_table + .translate_page(Page::of_addr(addr as u64)) + .unwrap(); + let vaddr = phys_to_virt(frame.start_address().as_u64() as usize); + unsafe { core::slice::from_raw_parts_mut(vaddr as *mut u8, 0x1000) } + } } -impl ActivePageTable { - pub unsafe fn new() -> Self { - ActivePageTable(RecursivePageTable::new(RECURSIVE_INDEX as u16)) - } +fn frame_to_page_table(frame: Frame) -> *mut Aarch64PageTable { + let vaddr = phys_to_virt(frame.start_address().as_u64() as usize); + vaddr as *mut Aarch64PageTable } #[repr(u8)] @@ -100,7 +121,8 @@ impl Entry for PageEntry { self.0.addr().as_u64() as usize } fn set_target(&mut self, target: usize) { - self.0.modify_addr(PhysAddr::new(target as u64)); + self.0 + .set_addr(PhysAddr::new(target as u64), self.0.flags(), self.0.attr()); } fn writable_shared(&self) -> bool { self.0.flags().contains(EF::WRITABLE_SHARED) @@ -163,7 +185,7 @@ impl Entry for PageEntry { 2 => MairNormalNonCacheable::attr_value(), _ => return, }; - self.0.modify_attr(attr); + self.0.set_attr(attr); } } @@ -178,40 +200,45 @@ impl PageEntry { self.0.flags().contains(EF::DIRTY) } fn as_flags(&mut self) -> &mut EF { - unsafe { &mut *(self as *mut _ as *mut EF) } + unsafe { &mut *(self.0 as *mut _ as *mut EF) } } } -#[derive(Debug)] -pub struct InactivePageTable0 { - p4_frame: Frame, +impl PageTableImpl { + /// Unsafely get the current active page table. + /// WARN: You MUST call `core::mem::forget` for it after use! + pub unsafe fn active() -> Self { + let frame = Frame::of_addr(PageTableImpl::active_token() as u64); + let table = &mut *frame_to_page_table(frame); + PageTableImpl { + page_table: MappedPageTable::new(table, frame_to_page_table), + root_frame: frame, + entry: core::mem::uninitialized(), + } + } } -impl InactivePageTable for InactivePageTable0 { - type Active = ActivePageTable; - +impl PageTableExt for PageTableImpl { fn new_bare() -> Self { let target = alloc_frame().expect("failed to allocate frame"); let frame = Frame::of_addr(target as u64); - active_table().with_temporary_map(target, |_, table: &mut Aarch64PageTable| { - table.zero(); - // set up recursive mapping for the table - table[RECURSIVE_INDEX].set_frame( - frame.clone(), - EF::default(), - MairNormal::attr_value(), - ); - }); - InactivePageTable0 { p4_frame: frame } + let table = unsafe { &mut *frame_to_page_table(frame) }; + table.zero(); + unsafe { + PageTableImpl { + page_table: MappedPageTable::new(table, frame_to_page_table), + root_frame: frame, + entry: core::mem::uninitialized(), + } + } } fn map_kernel(&mut self) { - // When the new InactivePageTable is created for the user MemorySet, it's use ttbr0 as the - // TTBR. And the kernel TTBR ttbr1 will never changed, so we needn't call map_kernel() + // kernel page table is based on TTBR1_EL1 and will nerver change. } fn token(&self) -> usize { - self.p4_frame.start_address().as_u64() as usize // as TTBRx_EL1 + self.root_frame.start_address().as_u64() as usize // as TTBR0_EL1 } unsafe fn set_token(token: usize) { @@ -225,73 +252,25 @@ impl InactivePageTable for InactivePageTable0 { fn flush_tlb() { tlb_invalidate_all(); } - - fn edit(&mut self, f: impl FnOnce(&mut Self::Active) -> T) -> T { - let target = ttbr_el1_read(1); - if self.p4_frame == target { - return f(&mut active_table()); - } - let target = target.start_address().as_u64() as usize; - active_table().with_temporary_map( - target, - |active_table, p4_table: &mut Aarch64PageTable| { - let backup = p4_table[RECURSIVE_INDEX].clone(); - let old_frame = ttbr_el1_read(0); - - // overwrite recursive mapping - p4_table[RECURSIVE_INDEX].set_frame( - self.p4_frame.clone(), - EF::default(), - MairNormal::attr_value(), - ); - ttbr_el1_write(0, self.p4_frame.clone()); - tlb_invalidate_all(); - - // execute f in the new context - let ret = f(active_table); - - // restore recursive mapping to original p4 table - p4_table[RECURSIVE_INDEX] = backup; - ttbr_el1_write(0, old_frame); - tlb_invalidate_all(); - ret - }, - ) - } -} - -impl InactivePageTable0 { - /// Activate as kernel page table (TTBR0). - /// Used in `arch::memory::remap_the_kernel()`. - pub unsafe fn activate_as_kernel(&self) { - let old_frame = ttbr_el1_read(1); - let new_frame = self.p4_frame.clone(); - debug!("switch TTBR1 {:?} -> {:?}", old_frame, new_frame); - if old_frame != new_frame { - ttbr_el1_write(0, Frame::of_addr(0)); - ttbr_el1_write(1, new_frame); - tlb_invalidate_all(); - } - } } -impl Drop for InactivePageTable0 { +impl Drop for PageTableImpl { fn drop(&mut self) { - info!("PageTable dropping: {:?}", self); - dealloc_frame(self.p4_frame.start_address().as_u64() as usize); + info!("PageTable dropping: {:?}", self.root_frame); + dealloc_frame(self.root_frame.start_address().as_u64() as usize); } } struct FrameAllocatorForAarch64; -impl FrameAllocator for FrameAllocatorForAarch64 { - fn alloc(&mut self) -> Option { +unsafe impl FrameAllocator for FrameAllocatorForAarch64 { + fn allocate_frame(&mut self) -> Option { alloc_frame().map(|addr| Frame::of_addr(addr as u64)) } } impl FrameDeallocator for FrameAllocatorForAarch64 { - fn dealloc(&mut self, frame: Frame) { + fn deallocate_frame(&mut self, frame: Frame) { dealloc_frame(frame.start_address().as_u64() as usize); } }