aarch64/mmu: can run on the real raspi3

crate/aarch64/src/asm.rs

@@ -1,9 +1,9 @@
 use paging::PhysFrame;
-use addr::PhysAddr;
+use addr::{PhysAddr, VirtAddr};
 use regs::*;
 
 #[inline(always)]
-pub fn tlb_invalidate() {
+pub fn tlb_invalidate_all() {
     unsafe {
         asm!(
             "dsb ishst
@@ -14,6 +14,18 @@ pub fn tlb_invalidate() {
     }
 }
 
+#[inline(always)]
+pub fn tlb_invalidate(vaddr: VirtAddr) {
+    unsafe {
+        asm!(
+            "dsb ishst
+             tlbi vaae1is, $0
+             dsb ish
+             isb" :: "r"(vaddr.as_u64() >> 12)
+        );
+    }
+}
+
 /// Returns the current stack pointer.
 #[inline(always)]
 pub fn sp() -> *const u8 {

crate/aarch64/src/paging/recursive.rs

@@ -28,7 +28,7 @@ impl<S: PageSize> MapperFlush<S> {
 
     /// Flush the page from the TLB to ensure that the newest mapping is used.
     pub fn flush(self) {
-        tlb_invalidate();
+        tlb_invalidate(self.0.start_address());
     }
 
     /// Don't flush the TLB and silence the “must be used” warning.
@@ -232,6 +232,7 @@ impl<'a> RecursivePageTable<'a> {
             let page_table_ptr = next_table_page.start_address().as_mut_ptr();
             let page_table: &mut PageTable = unsafe { &mut *(page_table_ptr) };
             if created {
+                tlb_invalidate(next_table_page.start_address());
                 page_table.zero();
             }
             Ok(page_table)

kernel/src/arch/aarch64/board/raspi3/mod.rs

@@ -7,12 +7,9 @@ pub mod timer;
 pub mod serial;
 
 pub fn init() {
-    // FIXME
-    // assert_has_not_been_called!("board::init must be called only once");
+    assert_has_not_been_called!("board::init must be called only once");
 
-    unsafe {
-        serial::SERIAL_PORT.init();
-    }
+    serial::SERIAL_PORT.lock().init();
 
     println!("Hello Raspberry Pi!");
 }

kernel/src/arch/aarch64/board/raspi3/serial.rs

@@ -20,8 +20,7 @@ impl SerialPort {
 
     /// Init a newly created SerialPort, can only be called once.
     pub fn init(&mut self) {
-        // FIXME
-        // assert_has_not_been_called!("SerialPort::init must be called only once");
+        assert_has_not_been_called!("SerialPort::init must be called only once");
         self.mu = Some(MiniUart::new());
     }
 
@@ -71,7 +70,4 @@ impl fmt::Write for SerialPort {
     }
 }
 
-// FIXME
-// pub static SERIAL_PORT: Mutex<SerialPort> = Mutex::new(SerialPort::new());
-pub static mut SERIAL_PORT: SerialPort = SerialPort::new();
-
+pub static SERIAL_PORT: Mutex<SerialPort> = Mutex::new(SerialPort::new());

kernel/src/arch/aarch64/io.rs

@@ -4,15 +4,11 @@ use core::fmt::{Arguments, Write};
 use super::board::serial::{SerialRead, SERIAL_PORT};
 
 pub fn getchar() -> char {
-    // FIXME
-    unsafe {
-        SERIAL_PORT.receive() as char
-    }
+    unsafe { SERIAL_PORT.force_unlock(); }
+    SERIAL_PORT.lock().receive() as char
 }
 
 pub fn putfmt(fmt: Arguments) {
-    // FIXME
-    unsafe {
-        SERIAL_PORT.write_fmt(fmt).unwrap()
-    }
+    unsafe { SERIAL_PORT.force_unlock(); }
+    SERIAL_PORT.lock().write_fmt(fmt).unwrap()
 }

kernel/src/arch/aarch64/memory.rs

@@ -7,6 +7,14 @@ use aarch64::{barrier, regs::*, addr::*, paging::PhysFrame as Frame};
 
 /// Memory initialization.
 pub fn init() {
+    init_frame_allocator();
+    init_heap();
+    remap_the_kernel();
+    info!("memory: init end");
+}
+
+/// initialize temporary paging and enable mmu immediately after boot. Serial port is disabled at this time.
+pub fn init_mmu_early() {
     #[repr(align(4096))]
     struct PageData([u8; PAGE_SIZE]);
     static PAGE_TABLE_LVL4: PageData = PageData([0; PAGE_SIZE]);
@@ -18,41 +26,6 @@ pub fn init() {
     let frame_lvl2 = Frame::containing_address(PhysAddr::new(&PAGE_TABLE_LVL2 as *const _ as u64));
     super::paging::setup_page_table(frame_lvl4, frame_lvl3, frame_lvl2);
 
-    init_mmu();
-    init_frame_allocator();
-    init_heap();
-    remap_the_kernel();
-
-    info!("memory: init end");
-}
-
-fn init_frame_allocator() {
-    use bit_allocator::BitAlloc;
-    use core::ops::Range;
-    use consts::{MEMORY_OFFSET};
-
-    let (start, end) = memory_map().expect("failed to find memory map");
-    let mut ba = FRAME_ALLOCATOR.lock();
-    ba.insert(to_range(start, end));
-    info!("FrameAllocator init end");
-
-    /*
-    * @param:
-    *   start: start address
-    *   end: end address
-    * @brief:
-    *   transform the memory address to the page number
-    * @retval:
-    *   the page number range from start address to end address
-    */
-    fn to_range(start: usize, end: usize) -> Range<usize> {
-        let page_start = (start - MEMORY_OFFSET) / PAGE_SIZE;
-        let page_end = (end - MEMORY_OFFSET - 1) / PAGE_SIZE + 1;
-        page_start..page_end
-    }
-}
-
-fn init_mmu() {
     // device.
     MAIR_EL1.write(
         // Attribute 1
@@ -85,10 +58,35 @@ fn init_mmu() {
 
     // Force MMU init to complete before next instruction
     unsafe { barrier::isb(barrier::SY); }
+}
+
+fn init_frame_allocator() {
+    use bit_allocator::BitAlloc;
+    use core::ops::Range;
+    use consts::{MEMORY_OFFSET};
+
+    let (start, end) = memory_map().expect("failed to find memory map");
+    let mut ba = FRAME_ALLOCATOR.lock();
+    ba.insert(to_range(start, end));
+    info!("FrameAllocator init end");
 
-    info!("mmu enabled");
+    /*
+    * @param:
+    *   start: start address
+    *   end: end address
+    * @brief:
+    *   transform the memory address to the page number
+    * @retval:
+    *   the page number range from start address to end address
+    */
+    fn to_range(start: usize, end: usize) -> Range<usize> {
+        let page_start = (start - MEMORY_OFFSET) / PAGE_SIZE;
+        let page_end = (end - MEMORY_OFFSET - 1) / PAGE_SIZE + 1;
+        page_start..page_end
+    }
 }
 
+/// remap kernel page table after all initialization.
 fn remap_the_kernel() {
     let (bottom, top) = (0, bootstacktop as usize);
     let kstack = Stack {

kernel/src/arch/aarch64/mod.rs

@@ -17,9 +17,13 @@ pub use self::board::timer;
 /// The entry point of kernel
 #[no_mangle] // don't mangle the name of this function
 pub extern "C" fn rust_main() -> ! {
+    // Enable mmu and paging
+    memory::init_mmu_early();
+
     // Init board to enable serial port.
     board::init();
-    ::logging::init(); // FIXME
+
+    ::logging::init();
     interrupt::init();
     memory::init();
     timer::init();

kernel/src/arch/aarch64/paging.rs

@@ -5,7 +5,7 @@ use memory::{active_table, alloc_frame, alloc_stack, dealloc_frame};
 use ucore_memory::memory_set::*;
 use ucore_memory::PAGE_SIZE;
 use ucore_memory::paging::*;
-use aarch64::asm::{tlb_invalidate, ttbr0_el1_read, ttbr0_el1_write};
+use aarch64::asm::{tlb_invalidate, tlb_invalidate_all, ttbr0_el1_read, ttbr0_el1_write};
 use aarch64::{PhysAddr, VirtAddr};
 use aarch64::paging::{Mapper, PageTable as Aarch64PageTable, PageTableEntry, PageTableFlags as EF, RecursivePageTable};
 use aarch64::paging::{FrameAllocator, FrameDeallocator, Page, PageRange, PhysFrame as Frame, Size4KiB, Size2MiB};
@@ -123,9 +123,7 @@ pub fn setup_page_table(frame_lvl4: Frame, frame_lvl3: Frame, frame_lvl2: Frame)
     // }
 
     ttbr0_el1_write(frame_lvl4);
-    tlb_invalidate();
-
-    info!("setup init page table end");
+    tlb_invalidate_all();
 }
 
 /// map the range [start, end) as device memory, insert to the MemorySet
@@ -222,7 +220,8 @@ impl ActivePageTable {
 
 impl Entry for PageEntry {
     fn update(&mut self) {
-        tlb_invalidate();
+        let addr = VirtAddr::new_unchecked((self as *const _ as u64) << 9);
+        tlb_invalidate(addr);
     }
 
     fn present(&self) -> bool { self.0.flags().contains(EF::PRESENT) }
@@ -314,14 +313,14 @@ impl InactivePageTable for InactivePageTable0 {
 
             // overwrite recursive mapping
             p4_table[RECURSIVE_INDEX].set_frame(self.p4_frame.clone(), EF::PRESENT | EF::WRITE | EF::ACCESSED | EF::PAGE_BIT);
-            tlb_invalidate();
+            tlb_invalidate_all();
 
             // execute f in the new context
             f(active_table);
 
             // restore recursive mapping to original p4 table
             p4_table[RECURSIVE_INDEX] = backup;
-            tlb_invalidate();
+            tlb_invalidate_all();
         });
     }
 
@@ -331,7 +330,7 @@ impl InactivePageTable for InactivePageTable0 {
         debug!("switch table {:?} -> {:?}", old_frame, new_frame);
         if old_frame != new_frame {
             ttbr0_el1_write(new_frame);
-            tlb_invalidate();
+            tlb_invalidate_all();
         }
     }
 
@@ -341,13 +340,13 @@ impl InactivePageTable for InactivePageTable0 {
         debug!("switch table {:?} -> {:?}", old_frame, new_frame);
         if old_frame != new_frame {
             ttbr0_el1_write(new_frame);
-            tlb_invalidate();
+            tlb_invalidate_all();
         }
         f();
         debug!("switch table {:?} -> {:?}", new_frame, old_frame);
         if old_frame != new_frame {
             ttbr0_el1_write(old_frame);
-            tlb_invalidate();
+            tlb_invalidate_all();
         }
     }