diff --git a/kernel/build-rv64 b/kernel/build-rv64 index 7dd8ac6..1c66dac 100755 --- a/kernel/build-rv64 +++ b/kernel/build-rv64 @@ -12,11 +12,11 @@ set -e if [[ ${RV32} = 1 ]]; then TARGET_ARCH=riscv32 - COMPILER_RT_CFLAGS="-march=rv32ia -mabi=ilp32 -O3" + COMPILER_RT_CFLAGS="-march=rv32imac -mabi=ilp32 -O3" SFSIMG_CFLAGS="-march=rv32ia -mabi=ilp32" else TARGET_ARCH=riscv64 - COMPILER_RT_CFLAGS="-march=rv64ia -mabi=lp64 -O3" + COMPILER_RT_CFLAGS="-march=rv64imac -mabi=lp64 -O3" SFSIMG_CFLAGS="-march=rv64ia -mabi=lp64" fi UCORE_USER_IMAGE="../user/img/ucore-${TARGET_ARCH}.img" @@ -60,7 +60,7 @@ fi fi # if some crates are not exist, build for riscv32 first -if ! [[ -f $CARGO_PATH/git/checkouts/bit-vec-437fa4a002bd318d/9861a58*/src/lib.rs ]] +if ! [[ -f $CARGO_PATH/git/checkouts/bit-vec-437fa4a002bd318d/9861a58/src/lib.rs ]] then make kernel arch=riscv32 board=none fi diff --git a/kernel/build.rs b/kernel/build.rs index 3a8a541..6d4043f 100644 --- a/kernel/build.rs +++ b/kernel/build.rs @@ -10,24 +10,21 @@ fn main() { let arch: String = std::env::var("ARCH").unwrap(); match arch.as_str() { "x86_64" => { - // cc::Build::new() - // .file("src/arch/x86_64/driver/apic/lapic.c") - // .file("src/arch/x86_64/driver/keyboard/keyboard.c") - // .flag("-mcmodel=large") - // .compile("cobj"); gen_vector_asm().unwrap(); } "riscv32" => { + println!("cargo:rerun-if-changed=src/arch/riscv32/compiler_rt.c"); cc::Build::new() .file("src/arch/riscv32/compiler_rt.c") - .flag("-march=rv32ia") + .flag("-march=rv32imac") .flag("-mabi=ilp32") .flag("-Wno-builtin-declaration-mismatch") + .flag("-O3") .compile("atomic_rt"); if let Ok(file_path) = gen_sfsimg_asm() { cc::Build::new() .file(&file_path) - .flag("-march=rv32ia") + .flag("-march=rv32imac") .flag("-mabi=ilp32") .compile("sfsimg"); } diff --git a/kernel/src/arch/riscv32/atomic.patch b/kernel/src/arch/riscv32/atomic.patch index ec7ca3a..d7722b6 100644 --- a/kernel/src/arch/riscv32/atomic.patch +++ b/kernel/src/arch/riscv32/atomic.patch @@ -10,7 +10,7 @@ v: UnsafeCell, } -@@ -265,6 +268,44 @@ +@@ -265,6 +268,59 @@ pub const ATOMIC_BOOL_INIT: AtomicBool = AtomicBool::new(false); #[cfg(target_has_atomic = "8")] @@ -40,12 +40,27 @@ + /// + #[inline] + #[stable(feature = "rust1", since = "1.0.0")] -+ #[cfg(target_has_atomic = "cas")] + pub fn compare_and_swap(&self, current: bool, new: bool, order: Ordering) -> bool { -+ loop { -+ if let Ok(val) = unsafe { atomic_compare_exchange(self.v.get(), current as u32, new as u32, order, order) } { -+ return val != 0; -+ } ++ match self.compare_exchange(current, new, order, strongest_failure_ordering(order)) { ++ Ok(x) => x, ++ Err(x) => x, ++ } ++ } ++ ++ /// ++ #[inline] ++ #[stable(feature = "extended_compare_and_swap", since = "1.10.0")] ++ pub fn compare_exchange(&self, ++ current: bool, ++ new: bool, ++ success: Ordering, ++ failure: Ordering) ++ -> Result { ++ match unsafe { ++ atomic_compare_exchange(self.v.get(), current as u32, new as u32, success, failure) ++ } { ++ Ok(x) => Ok(x != 0), ++ Err(x) => Err(x != 0), + } + } +} diff --git a/kernel/src/arch/riscv32/compiler_rt.c b/kernel/src/arch/riscv32/compiler_rt.c index 57b6300..edd6105 100644 --- a/kernel/src/arch/riscv32/compiler_rt.c +++ b/kernel/src/arch/riscv32/compiler_rt.c @@ -1,60 +1,62 @@ // http://llvm.org/docs/Atomics.html#libcalls-atomic +inline void mb() { + __asm__ __volatile__("fence" ::: "memory"); +} + typedef unsigned u32; // K210 doesn't support atomic operation on 0x40000000 (io port) // We have to detect it and move it to 0x80000000 -inline u32* fix_ptr32(u32 *src) { - return (u32)src < 0x80000000? - (u32*)((u32)src + 0x40000000): - src; +inline u32* fix_ptr32(u32 *ptr) { + return (u32)ptr < 0x80000000? + (u32*)((u32)ptr + 0x40000000): + ptr; } -u32 __atomic_load_1(u32 *src) { - src = fix_ptr32(src); - u32 res = 0; - __asm__ __volatile__("amoadd.w %0, zero, (%1)" : "=r"(res) : "r"(src) : "memory"); - return res; +u32 __atomic_load_1(u32 *ptr) { + ptr = fix_ptr32(ptr); + return *ptr; } -u32 __atomic_load_2(u32 *src) { - src = fix_ptr32(src); - u32 res = 0; - __asm__ __volatile__("amoadd.w %0, zero, (%1)" : "=r"(res) : "r"(src) : "memory"); - return res; +u32 __atomic_load_2(u32 *ptr) { + ptr = fix_ptr32(ptr); + return *ptr; } -u32 __atomic_load_4(u32 *src) { - src = fix_ptr32(src); - u32 res = 0; - __asm__ __volatile__("amoadd.w %0, zero, (%1)" : "=r"(res) : "r"(src) : "memory"); - return res; +// relaxed +u32 __atomic_load_4(u32 *ptr) { + ptr = fix_ptr32(ptr); + return *ptr; } -void __atomic_store_4(u32 *dst, u32 val) { - dst = fix_ptr32(dst); - __asm__ __volatile__("amoswap.w zero, %0, (%1)" :: "r"(val), "r"(dst) : "memory"); +// release +void __atomic_store_4(u32 *ptr, u32 val) { + ptr = fix_ptr32(ptr); + mb(); + __asm__ __volatile__("amoswap.w zero, %0, (%1)" :: "r"(val), "r"(ptr) : "memory"); } -char __atomic_compare_exchange_4(u32* dst, u32* expected, u32 desired) { - dst = fix_ptr32(dst); - u32 val, expect, result; - // val = *dst - __asm__ __volatile__("lw %0, (%1)" : "=r"(expect) : "r" (expected) : "memory"); - __asm__ __volatile__("lr.w %0, (%1)" : "=r"(val) : "r"(dst) : "memory"); - - // if (val != *expected) goto fail; - if (val != expect) goto __atomic_compare_exchange_4_fail; - - // Try: *dst = desired. If success, result = 0, otherwise result != 0. - __asm__ __volatile__("sc.w %0, %1, (%2)" : "=r"(result) : "r"(desired), "r"(dst) : "memory"); - return result == 0; - - __atomic_compare_exchange_4_fail: - - // *expected should always equal to the previous value of *dst - *expected = val; - return 0; +// strong, acquire +char __atomic_compare_exchange_4(u32* ptr, u32* expected, u32 desired) { + ptr = fix_ptr32(ptr); + u32 val, expect = *expected, result, ret; + while(1) { + __asm__ __volatile__("lr.w.aq %0, (%1)" : "=r"(val) : "r"(ptr) : "memory"); + + ret = val == expect; + if(!ret) { + // *expected should always equal to the previous value of *ptr + *expected = val; + return ret; + } + + // Try: *ptr = desired. If success, result == 0, otherwise result != 0. + __asm__ __volatile__("sc.w.aq %0, %1, (%2)" : "=r"(result) : "r"(desired), "r"(ptr) : "memory"); + if(result == 0) { + return ret; + } + } } u32 __atomic_fetch_add_4(u32* ptr, u32 val) { @@ -76,43 +78,45 @@ typedef unsigned long long u64; // K210 doesn't support atomic operation on 0x40000000 (io port) // We have to detect it and move it to 0x80000000 -inline u64* fix_ptr64(u64 *src) { - return (u64)src < 0x80000000? - (u64*)((u64)src + 0x40000000): - src; +inline u64* fix_ptr64(u64 *ptr) { + return (u64)ptr < 0x80000000? + (u64*)((u64)ptr + 0x40000000): + ptr; } -u64 __atomic_load_8(u64 *src) { - src = fix_ptr64(src); - u64 res = 0; - __asm__ __volatile__("amoadd.d %0, zero, (%1)" : "=r"(res) : "r"(src) : "memory"); - return res; +// relaxed +u64 __atomic_load_8(u64 *ptr) { + ptr = fix_ptr64(ptr); + return *ptr; } -void __atomic_store_8(u64 *dst, u64 val) { - dst = fix_ptr64(dst); - __asm__ __volatile__("amoswap.d zero, %0, (%1)" :: "r"(val), "r"(dst) : "memory"); +// release +void __atomic_store_8(u64 *ptr, u64 val) { + ptr = fix_ptr64(ptr); + mb(); + __asm__ __volatile__("amoswap.d zero, %0, (%1)" :: "r"(val), "r"(ptr) : "memory"); } -char __atomic_compare_exchange_8(u64* dst, u64* expected, u64 desired) { - dst = fix_ptr64(dst); - u64 val, expect, result; - // val = *dst - __asm__ __volatile__("ld %0, (%1)" : "=r"(expect) : "r" (expected) : "memory"); - __asm__ __volatile__("lr.d %0, (%1)" : "=r"(val) : "r"(dst) : "memory"); - - // if (val != *expected) goto fail; - if (val != expect) goto __atomic_compare_exchange_8_fail; - - // Try: *dst = desired. If success, result = 0, otherwise result != 0. - __asm__ __volatile__("sc.d %0, %1, (%2)" : "=r"(result) : "r"(desired), "r"(dst) : "memory"); - return result == 0; - - __atomic_compare_exchange_8_fail: - - // *expected should always equal to the previous value of *dst - *expected = val; - return 0; +// strong, acquire +char __atomic_compare_exchange_8(u64* ptr, u64* expected, u64 desired) { + ptr = fix_ptr64(ptr); + u64 val, expect = *expected, result, ret; + while(1) { + __asm__ __volatile__("lr.d.aq %0, (%1)" : "=r"(val) : "r"(ptr) : "memory"); + + ret = val == expect; + if(!ret) { + // *expected should always equal to the previous value of *ptr + *expected = val; + return ret; + } + + // Try: *ptr = desired. If success, result == 0, otherwise result != 0. + __asm__ __volatile__("sc.d.aq %0, %1, (%2)" : "=r"(result) : "r"(desired), "r"(ptr) : "memory"); + if(result == 0) { + return ret; + } + } } u64 __atomic_fetch_add_8(u64* ptr, u64 val) {