From c88c0193d4af782e2e181229f22604e065c8ebe6 Mon Sep 17 00:00:00 2001 From: gjz010 Date: Wed, 8 May 2019 19:36:10 +0800 Subject: [PATCH 1/3] x86_64: impl IPI and remote flush tlb Signed-off-by: WangRunji --- kernel/src/arch/x86_64/gdt.rs | 61 +++++++++++---- kernel/src/arch/x86_64/interrupt/consts.rs | 3 + kernel/src/arch/x86_64/interrupt/handler.rs | 5 ++ kernel/src/arch/x86_64/ipi.rs | 82 +++++++++++++++++++++ kernel/src/arch/x86_64/mod.rs | 5 +- kernel/src/arch/x86_64/paging.rs | 12 +++ user | 2 +- 7 files changed, 154 insertions(+), 16 deletions(-) create mode 100644 kernel/src/arch/x86_64/ipi.rs diff --git a/kernel/src/arch/x86_64/gdt.rs b/kernel/src/arch/x86_64/gdt.rs index 42c125c..8347b10 100644 --- a/kernel/src/arch/x86_64/gdt.rs +++ b/kernel/src/arch/x86_64/gdt.rs @@ -1,11 +1,14 @@ +use super::ipi::IPIEventItem; use alloc::boxed::Box; - +use alloc::vec::*; +use core::sync::atomic::{AtomicBool, Ordering}; use x86_64::registers::model_specific::Msr; use x86_64::structures::gdt::*; use x86_64::structures::tss::TaskStateSegment; use x86_64::{PrivilegeLevel, VirtAddr}; use crate::consts::MAX_CPU_NUM; +use crate::sync::{Semaphore, SpinLock as Mutex}; /// Init TSS & GDT. pub fn init() { @@ -20,22 +23,16 @@ static mut CPUS: [Option; MAX_CPU_NUM] = [ None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, - None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, - None, - // None, None, None, None, None, None, None, None, - // None, None, None, None, None, None, None, None, - // None, None, None, None, None, None, None, None, - // None, None, None, None, None, None, None, None, - // None, None, None, None, None, None, None, None, - // None, None, None, None, None, None, None, None, - // None, None, None, None, None, None, None, None, - // None, None, None, None, None, None, None, None, + None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, ]; pub struct Cpu { gdt: GlobalDescriptorTable, tss: TaskStateSegment, double_fault_stack: [u8; 0x100], + preemption_disabled: AtomicBool, // TODO: check this on timer(). This is currently unavailable since related code is in rcore_thread. + ipi_handler_queue: Mutex>, + id: usize, } impl Cpu { @@ -44,9 +41,47 @@ impl Cpu { gdt: GlobalDescriptorTable::new(), tss: TaskStateSegment::new(), double_fault_stack: [0u8; 0x100], + preemption_disabled: AtomicBool::new(false), + ipi_handler_queue: Mutex::new(vec![]), + id: 0, } } + pub fn foreach(mut f: impl FnMut(&mut Cpu)) { + unsafe { + CPUS.iter_mut() + .filter_map(|x| x.as_mut()) + .for_each(f); + } + } + pub fn get_id(&self) -> usize { + self.id + } + pub fn notify_event(&mut self, item: IPIEventItem) { + let mut queue = self.ipi_handler_queue.lock(); + queue.push(item); + } + pub fn current() -> &'static mut Cpu { + unsafe { CPUS[super::cpu::id()].as_mut().unwrap() } + } + pub fn ipi_handler(&mut self) { + let mut queue = self.ipi_handler_queue.lock(); + let mut current_events: Vec = vec![]; + ::core::mem::swap(&mut current_events, queue.as_mut()); + drop(queue); + for ev in current_events.iter() { + ev.call(); + } + } + pub fn disable_preemption(&self) -> bool { + self.preemption_disabled.swap(true, Ordering::Relaxed) + } + pub fn restore_preemption(&self, val: bool) { + self.preemption_disabled.store(val, Ordering::Relaxed); + } + pub fn can_preempt(&self) -> bool { + self.preemption_disabled.load(Ordering::Relaxed) + } unsafe fn init(&'static mut self) { use x86_64::instructions::segmentation::{load_fs, set_cs}; use x86_64::instructions::tables::load_tss; @@ -63,7 +98,7 @@ impl Cpu { self.gdt.add_entry(UCODE); self.gdt.add_entry(Descriptor::tss_segment(&self.tss)); self.gdt.load(); - + self.id = super::cpu::id(); // reload code segment register set_cs(KCODE_SELECTOR); // load TSS @@ -81,7 +116,7 @@ const KCODE: Descriptor = Descriptor::UserSegment(0x0020980000000000); // EXECUT const UCODE: Descriptor = Descriptor::UserSegment(0x0020F80000000000); // EXECUTABLE | USER_SEGMENT | USER_MODE | PRESENT | LONG_MODE const KDATA: Descriptor = Descriptor::UserSegment(0x0000920000000000); // DATA_WRITABLE | USER_SEGMENT | PRESENT const UDATA: Descriptor = Descriptor::UserSegment(0x0000F20000000000); // DATA_WRITABLE | USER_SEGMENT | USER_MODE | PRESENT - // Copied from xv6 +// Copied from xv6 const UCODE32: Descriptor = Descriptor::UserSegment(0x00cffa00_0000ffff); // EXECUTABLE | USER_SEGMENT | USER_MODE | PRESENT const UDATA32: Descriptor = Descriptor::UserSegment(0x00cff200_0000ffff); // EXECUTABLE | USER_SEGMENT | USER_MODE | PRESENT diff --git a/kernel/src/arch/x86_64/interrupt/consts.rs b/kernel/src/arch/x86_64/interrupt/consts.rs index ccf0956..fd76ec7 100644 --- a/kernel/src/arch/x86_64/interrupt/consts.rs +++ b/kernel/src/arch/x86_64/interrupt/consts.rs @@ -45,3 +45,6 @@ pub const PIRQE: u8 = 20; pub const PIRQF: u8 = 21; pub const PIRQG: u8 = 22; pub const PIRQH: u8 = 23; + +// IPI constants +pub const IPIFuncCall: u8 = 0xfc; diff --git a/kernel/src/arch/x86_64/interrupt/handler.rs b/kernel/src/arch/x86_64/interrupt/handler.rs index de70864..d11f462 100644 --- a/kernel/src/arch/x86_64/interrupt/handler.rs +++ b/kernel/src/arch/x86_64/interrupt/handler.rs @@ -109,6 +109,11 @@ pub extern "C" fn rust_trap(tf: &mut TrapFrame) { Syscall32 => syscall32(tf), InvalidOpcode => invalid_opcode(tf), DivideError | GeneralProtectionFault => error(tf), + IPIFuncCall => { + let irq = tf.trap_num as u8 - IRQ0; + super::ack(irq); // must ack before switching + super::super::gdt::Cpu::current().ipi_handler(); + } _ => panic!("Unhandled interrupt {:x}", tf.trap_num), } } diff --git a/kernel/src/arch/x86_64/ipi.rs b/kernel/src/arch/x86_64/ipi.rs new file mode 100644 index 0000000..7c0cca9 --- /dev/null +++ b/kernel/src/arch/x86_64/ipi.rs @@ -0,0 +1,82 @@ +// Interface for inter-processor interrupt. +// This module wraps inter-processor interrupt into a broadcast-calling style. + +use crate::consts::KERNEL_OFFSET; +use crate::sync::{Semaphore, SpinLock as Mutex}; +use alloc::boxed::Box; +use alloc::sync::Arc; +use apic::{LocalApic, XApic, LAPIC_ADDR}; +use lazy_static::*; +use rcore_memory::Page; +use x86_64::instructions::tlb; +use x86_64::VirtAddr; + +struct IPIInvoke<'a, A>(&'a (Fn(&A) -> ()), &'a A); + +lazy_static! { + static ref IPI_INVOKE_LOCK: Mutex<()> = Mutex::new(()); +} + +pub trait InvokeEventHandle { + fn call(&self); +} + +struct InvokeEvent { + function: fn(&A) -> (), + argument: Arc, + done_semaphore: Arc, +} + +impl InvokeEventHandle for InvokeEvent { + fn call(&self) { + let arg_ref = self.argument.as_ref(); + (self.function)(arg_ref); + self.done_semaphore.release(); + } +} + +pub type IPIEventItem = Box; + +// TODO: something fishy is going on here... +// In fact, the argument lives as long as the Arc. +fn create_item(f: fn(&A) -> (), arg: &Arc, sem: &Arc) -> IPIEventItem { + Box::new(InvokeEvent { + function: f, + argument: arg.clone(), + done_semaphore: sem.clone(), + }) +} +unsafe fn get_apic() -> XApic { + let mut lapic = unsafe { XApic::new(KERNEL_OFFSET + LAPIC_ADDR) }; + lapic +} +pub fn invoke_on_allcpu(f: fn(&A) -> (), arg: A, wait: bool) { + // Step 1: initialize + use super::interrupt::consts::IPIFuncCall; + let mut apic = unsafe { get_apic() }; + let sem = Arc::new(Semaphore::new(0)); + let arcarg = Arc::new(arg); + let mut cpu_count = 0; + // Step 2: invoke + super::gdt::Cpu::foreach(|cpu| { + let id = cpu.get_id(); + cpu_count += 1; + cpu.notify_event(create_item(f, &arcarg, &sem)); + apic.send_ipi(id as u8, IPIFuncCall); + }); + if wait { + for _ in 0..cpu_count { + sem.acquire(); + } + } +} + +// Examples of such cases. + +pub fn tlb_shootdown(tuple: &(usize, usize)) { +// debug!("CPU {}: remote tlb flush {:x?}", super::cpu::id(), tuple); + let (start_addr, end_addr) = *tuple; + for p in Page::range_of(start_addr, end_addr) { + tlb::flush(VirtAddr::new(p.start_address() as u64)); + } +} diff --git a/kernel/src/arch/x86_64/mod.rs b/kernel/src/arch/x86_64/mod.rs index f02dd60..2d0e6ca 100644 --- a/kernel/src/arch/x86_64/mod.rs +++ b/kernel/src/arch/x86_64/mod.rs @@ -14,6 +14,7 @@ pub mod paging; pub mod rand; pub mod syscall; pub mod timer; +pub mod ipi; static AP_CAN_INIT: AtomicBool = AtomicBool::new(false); @@ -51,9 +52,9 @@ pub extern "C" fn _start(boot_info: &'static BootInfo) -> ! { crate::drivers::init(); // init cpu scheduler and process manager, and add user shell app in process manager crate::process::init(); - //wake up other CPUs + // wake up other CPUs AP_CAN_INIT.store(true, Ordering::Relaxed); - //call the first main function in kernel. + // call the first main function in kernel. crate::kmain(); } diff --git a/kernel/src/arch/x86_64/paging.rs b/kernel/src/arch/x86_64/paging.rs index b280446..91fc58a 100644 --- a/kernel/src/arch/x86_64/paging.rs +++ b/kernel/src/arch/x86_64/paging.rs @@ -13,6 +13,7 @@ use x86_64::structures::paging::{ FrameAllocator, FrameDeallocator, }; use x86_64::PhysAddr; +use core::sync::atomic::Ordering; pub trait PageExt { fn of_addr(address: usize) -> Self; @@ -57,11 +58,13 @@ impl PageTable for ActivePageTable { .unwrap() .flush(); } + flush_tlb_all(addr); unsafe { &mut *(get_entry_ptr(addr, 1)) } } fn unmap(&mut self, addr: usize) { self.0.unmap(Page::of_addr(addr)).unwrap().1.flush(); + flush_tlb_all(addr); } fn get_entry(&mut self, addr: usize) -> Option<&mut Entry> { @@ -93,6 +96,7 @@ impl Entry for PageEntry { use x86_64::{instructions::tlb::flush, VirtAddr}; let addr = VirtAddr::new_unchecked((self as *const _ as u64) << 9); flush(addr); + flush_tlb_all(addr.as_u64() as usize); } fn accessed(&self) -> bool { self.0.flags().contains(EF::ACCESSED) @@ -277,3 +281,11 @@ impl FrameDeallocator for FrameAllocatorForX86 { dealloc_frame(frame.start_address().as_u64() as usize); } } + +/// Flush TLB for `vaddr` on all CPU +fn flush_tlb_all(vaddr: usize) { + if !super::AP_CAN_INIT.load(Ordering::Relaxed) { + return; + } + super::ipi::invoke_on_allcpu(super::ipi::tlb_shootdown, (vaddr, vaddr + 0x1000), false); +} diff --git a/user b/user index bb73d6e..9fb1d45 160000 --- a/user +++ b/user @@ -1 +1 @@ -Subproject commit bb73d6ecce1ab0e6fae692c51e4335772b0335d4 +Subproject commit 9fb1d459b50bc14c7ac56d9fd94b4b8485620730 From 2adbc4dcd9d96163c3eaa10c6f2a60d3ebda12aa Mon Sep 17 00:00:00 2001 From: Jiajie Chen Date: Wed, 8 May 2019 20:53:18 +0800 Subject: [PATCH 2/3] Format code and use rust/sh for rocket chip shell --- .../src/arch/riscv32/board/rocket_chip/mod.rs | 3 +- kernel/src/arch/riscv32/mod.rs | 1 - kernel/src/drivers/bus/mod.rs | 5 ++- kernel/src/fs/ioctl.rs | 2 +- kernel/src/memory.rs | 4 +-- kernel/src/shell.rs | 5 +++ kernel/src/syscall/mod.rs | 25 +++++++-------- kernel/src/syscall/net.rs | 31 ++++++++----------- 8 files changed, 38 insertions(+), 38 deletions(-) diff --git a/kernel/src/arch/riscv32/board/rocket_chip/mod.rs b/kernel/src/arch/riscv32/board/rocket_chip/mod.rs index 7e996b6..cb14a51 100644 --- a/kernel/src/arch/riscv32/board/rocket_chip/mod.rs +++ b/kernel/src/arch/riscv32/board/rocket_chip/mod.rs @@ -19,7 +19,6 @@ pub unsafe fn init_external_interrupt() { // enable all interrupts AXI_INTC_IER.write_volatile(0xffffffff); - } /// Claim and complete external interrupt by reading and writing to @@ -43,4 +42,4 @@ pub unsafe fn enable_serial_interrupt() { // Intr enable | rx reset | tx reset const UART_IE: u32 = 0x13; SERIAL_BASE.add(UART_CTRL_REG).write_volatile(UART_IE); -} \ No newline at end of file +} diff --git a/kernel/src/arch/riscv32/mod.rs b/kernel/src/arch/riscv32/mod.rs index 028d666..ff0b2c9 100644 --- a/kernel/src/arch/riscv32/mod.rs +++ b/kernel/src/arch/riscv32/mod.rs @@ -30,7 +30,6 @@ pub extern "C" fn rust_main(hartid: usize, device_tree_paddr: usize) -> ! { unsafe { cpu::set_cpu_id(hartid); - } #[cfg(feature = "board_rocket_chip")] diff --git a/kernel/src/drivers/bus/mod.rs b/kernel/src/drivers/bus/mod.rs index 56ee0cb..5a563d7 100644 --- a/kernel/src/drivers/bus/mod.rs +++ b/kernel/src/drivers/bus/mod.rs @@ -1,3 +1,6 @@ -#[cfg(any(target_arch = "x86_64", all(target_arch = "mips", feature = "board_malta")))] +#[cfg(any( + target_arch = "x86_64", + all(target_arch = "mips", feature = "board_malta") +))] pub mod pci; pub mod virtio_mmio; diff --git a/kernel/src/fs/ioctl.rs b/kernel/src/fs/ioctl.rs index 45bdebd..30652a7 100644 --- a/kernel/src/fs/ioctl.rs +++ b/kernel/src/fs/ioctl.rs @@ -37,4 +37,4 @@ pub const FIOCLEX: usize = 0x6601; // rustc using pipe and ioctl pipe file with this request id // for non-blocking/blocking IO control setting -pub const FIONBIO: usize = 0x5421; \ No newline at end of file +pub const FIONBIO: usize = 0x5421; diff --git a/kernel/src/memory.rs b/kernel/src/memory.rs index 44cdc25..d4bff95 100644 --- a/kernel/src/memory.rs +++ b/kernel/src/memory.rs @@ -13,19 +13,19 @@ //! mipssim/malta(MIPS) -- 10MB use super::HEAP_ALLOCATOR; -use core::mem; pub use crate::arch::paging::*; use crate::consts::{KERNEL_OFFSET, MEMORY_OFFSET}; +use crate::process::current_thread; use crate::sync::SpinNoIrqLock; use alloc::boxed::Box; use bitmap_allocator::BitAlloc; use buddy_system_allocator::Heap; +use core::mem; use lazy_static::*; use log::*; pub use rcore_memory::memory_set::{handler::*, MemoryArea, MemoryAttr}; use rcore_memory::paging::PageTable; use rcore_memory::*; -use crate::process::current_thread; pub type MemorySet = rcore_memory::memory_set::MemorySet; diff --git a/kernel/src/shell.rs b/kernel/src/shell.rs index f12b080..4b713bf 100644 --- a/kernel/src/shell.rs +++ b/kernel/src/shell.rs @@ -18,8 +18,13 @@ pub fn add_user_shell() { // let init_shell="/bin/busybox"; // from alpine linux // // #[cfg(not(target_arch = "x86_64"))] + #[cfg(not(feature = "board_rocket_chip"))] let init_shell = "/busybox"; //from docker-library + // fd is not available on rocket chip + #[cfg(feature = "board_rocket_chip")] + let init_shell = "/rust/sh"; + #[cfg(target_arch = "x86_64")] let init_envs = vec!["PATH=/usr/sbin:/usr/bin:/sbin:/bin:/usr/x86_64-alpine-linux-musl/bin".into()]; diff --git a/kernel/src/syscall/mod.rs b/kernel/src/syscall/mod.rs index 101dfc4..896d5d1 100644 --- a/kernel/src/syscall/mod.rs +++ b/kernel/src/syscall/mod.rs @@ -132,6 +132,14 @@ impl Syscall<'_> { SYS_DUP3 => self.sys_dup2(args[0], args[1]), // TODO: handle `flags` SYS_PIPE2 => self.sys_pipe(args[0] as *mut u32), // TODO: handle `flags` SYS_UTIMENSAT => self.unimplemented("utimensat", Ok(0)), + SYS_COPY_FILE_RANGE => self.sys_copy_file_range( + args[0], + args[1] as *mut usize, + args[2], + args[3] as *mut usize, + args[4], + args[5], + ), // io multiplexing SYS_PPOLL => { @@ -231,6 +239,7 @@ impl Syscall<'_> { args[2] as i32, args[3] as *const TimeSpec, ), + SYS_TKILL => self.unimplemented("tkill", Ok(0)), // time SYS_NANOSLEEP => self.sys_nanosleep(args[0] as *const TimeSpec), @@ -276,25 +285,15 @@ impl Syscall<'_> { args[2] as u32, args[3] as *const u8, ), - SYS_COPY_FILE_RANGE => self.sys_copy_file_range( - args[0], - args[1] as *mut usize, - args[2], - args[3] as *mut usize, - args[4], - args[5], - ), + SYS_GETRANDOM => { + self.sys_getrandom(args[0] as *mut u8, args[1] as usize, args[2] as u32) + } // custom SYS_MAP_PCI_DEVICE => self.sys_map_pci_device(args[0], args[1]), SYS_GET_PADDR => { self.sys_get_paddr(args[0] as *const u64, args[1] as *mut u64, args[2]) } - //SYS_GETRANDOM => self.unimplemented("getrandom", Err(SysError::EINVAL)), - SYS_GETRANDOM => { - self.sys_getrandom(args[0] as *mut u8, args[1] as usize, args[2] as u32) - } - SYS_TKILL => self.unimplemented("tkill", Ok(0)), _ => { let ret = match () { #[cfg(target_arch = "x86_64")] diff --git a/kernel/src/syscall/net.rs b/kernel/src/syscall/net.rs index 6a1d176..2edb9d6 100644 --- a/kernel/src/syscall/net.rs +++ b/kernel/src/syscall/net.rs @@ -185,7 +185,11 @@ impl Syscall<'_> { iovs.write_all_from_slice(&buf[..len]); let sockaddr_in = SockAddr::from(endpoint); unsafe { - sockaddr_in.write_to(&mut self.vm(), hdr.msg_name, &mut hdr.msg_namelen as *mut u32)?; + sockaddr_in.write_to( + &mut self.vm(), + hdr.msg_name, + &mut hdr.msg_namelen as *mut u32, + )?; } } result @@ -427,17 +431,13 @@ fn sockaddr_to_endpoint( Ok(Endpoint::Ip((addr, port).into())) } AddressFamily::Unix => Err(SysError::EINVAL), - AddressFamily::Packet => { - Ok(Endpoint::LinkLevel(LinkLevelEndpoint::new( - addr.addr_ll.sll_ifindex as usize, - ))) - } - AddressFamily::Netlink => { - Ok(Endpoint::Netlink(NetlinkEndpoint::new( - addr.addr_nl.nl_pid, - addr.addr_nl.nl_groups, - ))) - } + AddressFamily::Packet => Ok(Endpoint::LinkLevel(LinkLevelEndpoint::new( + addr.addr_ll.sll_ifindex as usize, + ))), + AddressFamily::Netlink => Ok(Endpoint::Netlink(NetlinkEndpoint::new( + addr.addr_nl.nl_pid, + addr.addr_nl.nl_groups, + ))), _ => Err(SysError::EINVAL), } } @@ -456,12 +456,7 @@ impl SockAddr { /// Write to user sockaddr /// Check mutability for user - unsafe fn write_to( - self, - vm: &MemorySet, - addr: *mut SockAddr, - addr_len: *mut u32, - ) -> SysResult { + unsafe fn write_to(self, vm: &MemorySet, addr: *mut SockAddr, addr_len: *mut u32) -> SysResult { // Ignore NULL if addr.is_null() { return Ok(0); From 75919dce413aab8536620bfe513ec34d6ec53413 Mon Sep 17 00:00:00 2001 From: WangRunji Date: Wed, 8 May 2019 22:46:23 +0800 Subject: [PATCH 3/3] x86_64: simplify IPI --- kernel/src/arch/x86_64/gdt.rs | 20 +++-- kernel/src/arch/x86_64/interrupt/handler.rs | 2 +- kernel/src/arch/x86_64/ipi.rs | 86 ++++++--------------- kernel/src/arch/x86_64/paging.rs | 9 ++- kernel/src/lib.rs | 1 + 5 files changed, 39 insertions(+), 79 deletions(-) diff --git a/kernel/src/arch/x86_64/gdt.rs b/kernel/src/arch/x86_64/gdt.rs index 8347b10..ffa3056 100644 --- a/kernel/src/arch/x86_64/gdt.rs +++ b/kernel/src/arch/x86_64/gdt.rs @@ -47,30 +47,28 @@ impl Cpu { } } - pub fn foreach(mut f: impl FnMut(&mut Cpu)) { + pub fn iter() -> impl Iterator { unsafe { - CPUS.iter_mut() - .filter_map(|x| x.as_mut()) - .for_each(f); + CPUS.iter() + .filter_map(|x| x.as_ref()) } } - pub fn get_id(&self) -> usize { + pub fn id(&self) -> usize { self.id } - pub fn notify_event(&mut self, item: IPIEventItem) { + pub fn notify_event(&self, item: IPIEventItem) { let mut queue = self.ipi_handler_queue.lock(); queue.push(item); } pub fn current() -> &'static mut Cpu { unsafe { CPUS[super::cpu::id()].as_mut().unwrap() } } - pub fn ipi_handler(&mut self) { + pub fn handle_ipi(&self) { let mut queue = self.ipi_handler_queue.lock(); - let mut current_events: Vec = vec![]; - ::core::mem::swap(&mut current_events, queue.as_mut()); + let handlers = core::mem::replace(queue.as_mut(), vec![]); drop(queue); - for ev in current_events.iter() { - ev.call(); + for handler in handlers { + handler(); } } pub fn disable_preemption(&self) -> bool { diff --git a/kernel/src/arch/x86_64/interrupt/handler.rs b/kernel/src/arch/x86_64/interrupt/handler.rs index d11f462..ac36e4c 100644 --- a/kernel/src/arch/x86_64/interrupt/handler.rs +++ b/kernel/src/arch/x86_64/interrupt/handler.rs @@ -112,7 +112,7 @@ pub extern "C" fn rust_trap(tf: &mut TrapFrame) { IPIFuncCall => { let irq = tf.trap_num as u8 - IRQ0; super::ack(irq); // must ack before switching - super::super::gdt::Cpu::current().ipi_handler(); + super::super::gdt::Cpu::current().handle_ipi(); } _ => panic!("Unhandled interrupt {:x}", tf.trap_num), } diff --git a/kernel/src/arch/x86_64/ipi.rs b/kernel/src/arch/x86_64/ipi.rs index 7c0cca9..b520b4f 100644 --- a/kernel/src/arch/x86_64/ipi.rs +++ b/kernel/src/arch/x86_64/ipi.rs @@ -1,82 +1,40 @@ -// Interface for inter-processor interrupt. -// This module wraps inter-processor interrupt into a broadcast-calling style. +/// Interface for inter-processor interrupt. +/// This module wraps inter-processor interrupt into a broadcast-calling style. use crate::consts::KERNEL_OFFSET; -use crate::sync::{Semaphore, SpinLock as Mutex}; -use alloc::boxed::Box; +use alloc::boxed::{Box, FnBox}; use alloc::sync::Arc; use apic::{LocalApic, XApic, LAPIC_ADDR}; -use lazy_static::*; -use rcore_memory::Page; -use x86_64::instructions::tlb; -use x86_64::VirtAddr; +use core::sync::atomic::{spin_loop_hint, AtomicU8, Ordering}; -struct IPIInvoke<'a, A>(&'a (Fn(&A) -> ()), &'a A); +pub type IPIEventItem = Box; -lazy_static! { - static ref IPI_INVOKE_LOCK: Mutex<()> = Mutex::new(()); -} - -pub trait InvokeEventHandle { - fn call(&self); -} - -struct InvokeEvent { - function: fn(&A) -> (), - argument: Arc, - done_semaphore: Arc, -} - -impl InvokeEventHandle for InvokeEvent { - fn call(&self) { - let arg_ref = self.argument.as_ref(); - (self.function)(arg_ref); - self.done_semaphore.release(); - } -} - -pub type IPIEventItem = Box; - -// TODO: something fishy is going on here... -// In fact, the argument lives as long as the Arc. -fn create_item(f: fn(&A) -> (), arg: &Arc, sem: &Arc) -> IPIEventItem { - Box::new(InvokeEvent { - function: f, - argument: arg.clone(), - done_semaphore: sem.clone(), - }) -} unsafe fn get_apic() -> XApic { let mut lapic = unsafe { XApic::new(KERNEL_OFFSET + LAPIC_ADDR) }; lapic } -pub fn invoke_on_allcpu(f: fn(&A) -> (), arg: A, wait: bool) { + +pub fn invoke_on_allcpu(f: impl Fn() + 'static, wait: bool) { // Step 1: initialize use super::interrupt::consts::IPIFuncCall; let mut apic = unsafe { get_apic() }; - let sem = Arc::new(Semaphore::new(0)); - let arcarg = Arc::new(arg); - let mut cpu_count = 0; + let func = Arc::new(f); + let cpu_count = super::gdt::Cpu::iter().count(); + let rest_count = Arc::new(AtomicU8::new(cpu_count as u8)); // Step 2: invoke - super::gdt::Cpu::foreach(|cpu| { - let id = cpu.get_id(); - cpu_count += 1; - cpu.notify_event(create_item(f, &arcarg, &sem)); - apic.send_ipi(id as u8, IPIFuncCall); - }); + for cpu in super::gdt::Cpu::iter() { + let func_clone = func.clone(); + let rest_clone = rest_count.clone(); + cpu.notify_event(Box::new(move || { + func_clone(); + rest_clone.fetch_sub(1, Ordering::Relaxed); + })); + apic.send_ipi(cpu.id() as u8, IPIFuncCall); + } if wait { - for _ in 0..cpu_count { - sem.acquire(); + // spin if remote invocation do not complete + while rest_count.load(Ordering::Relaxed) != 0 { + spin_loop_hint(); } } } - -// Examples of such cases. - -pub fn tlb_shootdown(tuple: &(usize, usize)) { -// debug!("CPU {}: remote tlb flush {:x?}", super::cpu::id(), tuple); - let (start_addr, end_addr) = *tuple; - for p in Page::range_of(start_addr, end_addr) { - tlb::flush(VirtAddr::new(p.start_address() as u64)); - } -} diff --git a/kernel/src/arch/x86_64/paging.rs b/kernel/src/arch/x86_64/paging.rs index 91fc58a..cbfb230 100644 --- a/kernel/src/arch/x86_64/paging.rs +++ b/kernel/src/arch/x86_64/paging.rs @@ -1,6 +1,7 @@ // Depends on kernel use crate::consts::KERNEL_OFFSET; use crate::memory::{active_table, alloc_frame, dealloc_frame}; +use core::sync::atomic::Ordering; use log::*; use rcore_memory::paging::*; use x86_64::instructions::tlb; @@ -12,8 +13,7 @@ use x86_64::structures::paging::{ page_table::{PageTable as x86PageTable, PageTableEntry, PageTableFlags as EF}, FrameAllocator, FrameDeallocator, }; -use x86_64::PhysAddr; -use core::sync::atomic::Ordering; +use x86_64::{VirtAddr, PhysAddr}; pub trait PageExt { fn of_addr(address: usize) -> Self; @@ -287,5 +287,8 @@ fn flush_tlb_all(vaddr: usize) { if !super::AP_CAN_INIT.load(Ordering::Relaxed) { return; } - super::ipi::invoke_on_allcpu(super::ipi::tlb_shootdown, (vaddr, vaddr + 0x1000), false); + super::ipi::invoke_on_allcpu( + move || tlb::flush(VirtAddr::new(vaddr as u64)), + false, + ); } diff --git a/kernel/src/lib.rs b/kernel/src/lib.rs index b6c482c..1b8f90b 100644 --- a/kernel/src/lib.rs +++ b/kernel/src/lib.rs @@ -6,6 +6,7 @@ #![feature(optin_builtin_traits)] #![feature(panic_info_message)] #![feature(global_asm)] +#![feature(fnbox)] #![deny(unused_must_use)] #![no_std]