diff --git a/crate/aarch64/src/asm.rs b/crate/aarch64/src/asm.rs index 17e6c29..63ce23a 100644 --- a/crate/aarch64/src/asm.rs +++ b/crate/aarch64/src/asm.rs @@ -1,9 +1,9 @@ use paging::PhysFrame; -use addr::PhysAddr; +use addr::{PhysAddr, VirtAddr}; use regs::*; #[inline(always)] -pub fn tlb_invalidate() { +pub fn tlb_invalidate_all() { unsafe { asm!( "dsb ishst @@ -14,6 +14,18 @@ pub fn tlb_invalidate() { } } +#[inline(always)] +pub fn tlb_invalidate(vaddr: VirtAddr) { + unsafe { + asm!( + "dsb ishst + tlbi vaae1is, $0 + dsb ish + isb" :: "r"(vaddr.as_u64() >> 12) + ); + } +} + /// Returns the current stack pointer. #[inline(always)] pub fn sp() -> *const u8 { diff --git a/crate/aarch64/src/paging/recursive.rs b/crate/aarch64/src/paging/recursive.rs index 31d1802..79b0a0d 100644 --- a/crate/aarch64/src/paging/recursive.rs +++ b/crate/aarch64/src/paging/recursive.rs @@ -28,7 +28,7 @@ impl MapperFlush { /// Flush the page from the TLB to ensure that the newest mapping is used. pub fn flush(self) { - tlb_invalidate(); + tlb_invalidate(self.0.start_address()); } /// Don't flush the TLB and silence the “must be used” warning. @@ -232,6 +232,7 @@ impl<'a> RecursivePageTable<'a> { let page_table_ptr = next_table_page.start_address().as_mut_ptr(); let page_table: &mut PageTable = unsafe { &mut *(page_table_ptr) }; if created { + tlb_invalidate(next_table_page.start_address()); page_table.zero(); } Ok(page_table) diff --git a/kernel/src/arch/aarch64/board/raspi3/mod.rs b/kernel/src/arch/aarch64/board/raspi3/mod.rs index 2b32a10..2d13315 100644 --- a/kernel/src/arch/aarch64/board/raspi3/mod.rs +++ b/kernel/src/arch/aarch64/board/raspi3/mod.rs @@ -7,12 +7,9 @@ pub mod timer; pub mod serial; pub fn init() { - // FIXME - // assert_has_not_been_called!("board::init must be called only once"); + assert_has_not_been_called!("board::init must be called only once"); - unsafe { - serial::SERIAL_PORT.init(); - } + serial::SERIAL_PORT.lock().init(); println!("Hello Raspberry Pi!"); } diff --git a/kernel/src/arch/aarch64/board/raspi3/serial.rs b/kernel/src/arch/aarch64/board/raspi3/serial.rs index 5434071..b6c29f7 100644 --- a/kernel/src/arch/aarch64/board/raspi3/serial.rs +++ b/kernel/src/arch/aarch64/board/raspi3/serial.rs @@ -20,8 +20,7 @@ impl SerialPort { /// Init a newly created SerialPort, can only be called once. pub fn init(&mut self) { - // FIXME - // assert_has_not_been_called!("SerialPort::init must be called only once"); + assert_has_not_been_called!("SerialPort::init must be called only once"); self.mu = Some(MiniUart::new()); } @@ -71,7 +70,4 @@ impl fmt::Write for SerialPort { } } -// FIXME -// pub static SERIAL_PORT: Mutex = Mutex::new(SerialPort::new()); -pub static mut SERIAL_PORT: SerialPort = SerialPort::new(); - +pub static SERIAL_PORT: Mutex = Mutex::new(SerialPort::new()); diff --git a/kernel/src/arch/aarch64/io.rs b/kernel/src/arch/aarch64/io.rs index deda39c..1a9d1d4 100644 --- a/kernel/src/arch/aarch64/io.rs +++ b/kernel/src/arch/aarch64/io.rs @@ -4,15 +4,11 @@ use core::fmt::{Arguments, Write}; use super::board::serial::{SerialRead, SERIAL_PORT}; pub fn getchar() -> char { - // FIXME - unsafe { - SERIAL_PORT.receive() as char - } + unsafe { SERIAL_PORT.force_unlock(); } + SERIAL_PORT.lock().receive() as char } pub fn putfmt(fmt: Arguments) { - // FIXME - unsafe { - SERIAL_PORT.write_fmt(fmt).unwrap() - } + unsafe { SERIAL_PORT.force_unlock(); } + SERIAL_PORT.lock().write_fmt(fmt).unwrap() } diff --git a/kernel/src/arch/aarch64/memory.rs b/kernel/src/arch/aarch64/memory.rs index c04e9c3..3f93e31 100644 --- a/kernel/src/arch/aarch64/memory.rs +++ b/kernel/src/arch/aarch64/memory.rs @@ -7,6 +7,14 @@ use aarch64::{barrier, regs::*, addr::*, paging::PhysFrame as Frame}; /// Memory initialization. pub fn init() { + init_frame_allocator(); + init_heap(); + remap_the_kernel(); + info!("memory: init end"); +} + +/// initialize temporary paging and enable mmu immediately after boot. Serial port is disabled at this time. +pub fn init_mmu_early() { #[repr(align(4096))] struct PageData([u8; PAGE_SIZE]); static PAGE_TABLE_LVL4: PageData = PageData([0; PAGE_SIZE]); @@ -18,41 +26,6 @@ pub fn init() { let frame_lvl2 = Frame::containing_address(PhysAddr::new(&PAGE_TABLE_LVL2 as *const _ as u64)); super::paging::setup_page_table(frame_lvl4, frame_lvl3, frame_lvl2); - init_mmu(); - init_frame_allocator(); - init_heap(); - remap_the_kernel(); - - info!("memory: init end"); -} - -fn init_frame_allocator() { - use bit_allocator::BitAlloc; - use core::ops::Range; - use consts::{MEMORY_OFFSET}; - - let (start, end) = memory_map().expect("failed to find memory map"); - let mut ba = FRAME_ALLOCATOR.lock(); - ba.insert(to_range(start, end)); - info!("FrameAllocator init end"); - - /* - * @param: - * start: start address - * end: end address - * @brief: - * transform the memory address to the page number - * @retval: - * the page number range from start address to end address - */ - fn to_range(start: usize, end: usize) -> Range { - let page_start = (start - MEMORY_OFFSET) / PAGE_SIZE; - let page_end = (end - MEMORY_OFFSET - 1) / PAGE_SIZE + 1; - page_start..page_end - } -} - -fn init_mmu() { // device. MAIR_EL1.write( // Attribute 1 @@ -85,10 +58,35 @@ fn init_mmu() { // Force MMU init to complete before next instruction unsafe { barrier::isb(barrier::SY); } +} + +fn init_frame_allocator() { + use bit_allocator::BitAlloc; + use core::ops::Range; + use consts::{MEMORY_OFFSET}; + + let (start, end) = memory_map().expect("failed to find memory map"); + let mut ba = FRAME_ALLOCATOR.lock(); + ba.insert(to_range(start, end)); + info!("FrameAllocator init end"); - info!("mmu enabled"); + /* + * @param: + * start: start address + * end: end address + * @brief: + * transform the memory address to the page number + * @retval: + * the page number range from start address to end address + */ + fn to_range(start: usize, end: usize) -> Range { + let page_start = (start - MEMORY_OFFSET) / PAGE_SIZE; + let page_end = (end - MEMORY_OFFSET - 1) / PAGE_SIZE + 1; + page_start..page_end + } } +/// remap kernel page table after all initialization. fn remap_the_kernel() { let (bottom, top) = (0, bootstacktop as usize); let kstack = Stack { diff --git a/kernel/src/arch/aarch64/mod.rs b/kernel/src/arch/aarch64/mod.rs index 35613ab..ccd6230 100644 --- a/kernel/src/arch/aarch64/mod.rs +++ b/kernel/src/arch/aarch64/mod.rs @@ -17,9 +17,13 @@ pub use self::board::timer; /// The entry point of kernel #[no_mangle] // don't mangle the name of this function pub extern "C" fn rust_main() -> ! { + // Enable mmu and paging + memory::init_mmu_early(); + // Init board to enable serial port. board::init(); - ::logging::init(); // FIXME + + ::logging::init(); interrupt::init(); memory::init(); timer::init(); diff --git a/kernel/src/arch/aarch64/paging.rs b/kernel/src/arch/aarch64/paging.rs index a023237..c7c4e99 100644 --- a/kernel/src/arch/aarch64/paging.rs +++ b/kernel/src/arch/aarch64/paging.rs @@ -5,7 +5,7 @@ use memory::{active_table, alloc_frame, alloc_stack, dealloc_frame}; use ucore_memory::memory_set::*; use ucore_memory::PAGE_SIZE; use ucore_memory::paging::*; -use aarch64::asm::{tlb_invalidate, ttbr0_el1_read, ttbr0_el1_write}; +use aarch64::asm::{tlb_invalidate, tlb_invalidate_all, ttbr0_el1_read, ttbr0_el1_write}; use aarch64::{PhysAddr, VirtAddr}; use aarch64::paging::{Mapper, PageTable as Aarch64PageTable, PageTableEntry, PageTableFlags as EF, RecursivePageTable}; use aarch64::paging::{FrameAllocator, FrameDeallocator, Page, PageRange, PhysFrame as Frame, Size4KiB, Size2MiB}; @@ -123,9 +123,7 @@ pub fn setup_page_table(frame_lvl4: Frame, frame_lvl3: Frame, frame_lvl2: Frame) // } ttbr0_el1_write(frame_lvl4); - tlb_invalidate(); - - info!("setup init page table end"); + tlb_invalidate_all(); } /// map the range [start, end) as device memory, insert to the MemorySet @@ -222,7 +220,8 @@ impl ActivePageTable { impl Entry for PageEntry { fn update(&mut self) { - tlb_invalidate(); + let addr = VirtAddr::new_unchecked((self as *const _ as u64) << 9); + tlb_invalidate(addr); } fn present(&self) -> bool { self.0.flags().contains(EF::PRESENT) } @@ -314,14 +313,14 @@ impl InactivePageTable for InactivePageTable0 { // overwrite recursive mapping p4_table[RECURSIVE_INDEX].set_frame(self.p4_frame.clone(), EF::PRESENT | EF::WRITE | EF::ACCESSED | EF::PAGE_BIT); - tlb_invalidate(); + tlb_invalidate_all(); // execute f in the new context f(active_table); // restore recursive mapping to original p4 table p4_table[RECURSIVE_INDEX] = backup; - tlb_invalidate(); + tlb_invalidate_all(); }); } @@ -331,7 +330,7 @@ impl InactivePageTable for InactivePageTable0 { debug!("switch table {:?} -> {:?}", old_frame, new_frame); if old_frame != new_frame { ttbr0_el1_write(new_frame); - tlb_invalidate(); + tlb_invalidate_all(); } } @@ -341,13 +340,13 @@ impl InactivePageTable for InactivePageTable0 { debug!("switch table {:?} -> {:?}", old_frame, new_frame); if old_frame != new_frame { ttbr0_el1_write(new_frame); - tlb_invalidate(); + tlb_invalidate_all(); } f(); debug!("switch table {:?} -> {:?}", new_frame, old_frame); if old_frame != new_frame { ttbr0_el1_write(old_frame); - tlb_invalidate(); + tlb_invalidate_all(); } }