From 14c3ec67236b2d90c553d2575950369aa6fa43c5 Mon Sep 17 00:00:00 2001 From: Jesse Taube Date: Tue, 9 Jul 2024 13:39:34 -0400 Subject: [PATCH 01/47] RISC-V: pi: Force hidden visibility for all symbol references Eliminate all GOT entries in the .pi section, by forcing hidden visibility for all symbol references, which informs the compiler that such references will be resolved at link time without the need for allocating GOT entries. Include linux/hidden.h in Makefile, like arm64, for the hidden visibility attribute. Signed-off-by: Jesse Taube Reviewed-by: Alexandre Ghiti Link: https://lore.kernel.org/r/20240709173937.510084-2-jesse@rivosinc.com Signed-off-by: Palmer Dabbelt --- arch/riscv/kernel/pi/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/riscv/kernel/pi/Makefile b/arch/riscv/kernel/pi/Makefile index 50bc5ef7dd2f..1ef7584be0c3 100644 --- a/arch/riscv/kernel/pi/Makefile +++ b/arch/riscv/kernel/pi/Makefile @@ -5,6 +5,7 @@ KBUILD_CFLAGS := $(subst $(CC_FLAGS_FTRACE),,$(KBUILD_CFLAGS)) -fpie \ -Os -DDISABLE_BRANCH_PROFILING $(DISABLE_STACKLEAK_PLUGIN) \ $(call cc-option,-mbranch-protection=none) \ -I$(srctree)/scripts/dtc/libfdt -fno-stack-protector \ + -include $(srctree)/include/linux/hidden.h \ -D__DISABLE_EXPORTS -ffreestanding \ -fno-asynchronous-unwind-tables -fno-unwind-tables \ $(call cc-option,-fno-addrsig) From d57e19fcbf3f7492974e78cd1dbaf85c67d198ce Mon Sep 17 00:00:00 2001 From: Jesse Taube Date: Tue, 9 Jul 2024 13:39:35 -0400 Subject: [PATCH 02/47] RISC-V: lib: Add pi aliases for string functions memset, strcmp, and strncmp are all used in the __pi_ section, add SYM_FUNC_ALIAS for them. When KASAN is enabled in __pi___memset is also needed. Suggested-by: Charlie Jenkins Signed-off-by: Jesse Taube Reviewed-by: Charlie Jenkins Reviewed-by: Alexandre Ghiti Link: https://lore.kernel.org/r/20240709173937.510084-3-jesse@rivosinc.com Signed-off-by: Palmer Dabbelt --- arch/riscv/lib/memset.S | 2 ++ arch/riscv/lib/strcmp.S | 1 + arch/riscv/lib/strncmp.S | 1 + 3 files changed, 4 insertions(+) diff --git a/arch/riscv/lib/memset.S b/arch/riscv/lib/memset.S index 35f358e70bdb..da23b8347e2d 100644 --- a/arch/riscv/lib/memset.S +++ b/arch/riscv/lib/memset.S @@ -111,3 +111,5 @@ SYM_FUNC_START(__memset) ret SYM_FUNC_END(__memset) SYM_FUNC_ALIAS_WEAK(memset, __memset) +SYM_FUNC_ALIAS(__pi_memset, __memset) +SYM_FUNC_ALIAS(__pi___memset, __memset) diff --git a/arch/riscv/lib/strcmp.S b/arch/riscv/lib/strcmp.S index 687b2bea5c43..bc73325b2fd1 100644 --- a/arch/riscv/lib/strcmp.S +++ b/arch/riscv/lib/strcmp.S @@ -120,3 +120,4 @@ strcmp_zbb: .option pop #endif SYM_FUNC_END(strcmp) +SYM_FUNC_ALIAS(__pi_strcmp, strcmp) diff --git a/arch/riscv/lib/strncmp.S b/arch/riscv/lib/strncmp.S index aba5b3148621..b36325a57f6a 100644 --- a/arch/riscv/lib/strncmp.S +++ b/arch/riscv/lib/strncmp.S @@ -136,3 +136,4 @@ strncmp_zbb: .option pop #endif SYM_FUNC_END(strncmp) +SYM_FUNC_ALIAS(__pi_strncmp, strncmp) From b3311827155aca72498b8a0dfd0f2499b70d39b8 Mon Sep 17 00:00:00 2001 From: Jesse Taube Date: Tue, 9 Jul 2024 13:39:36 -0400 Subject: [PATCH 03/47] RISC-V: pi: Add kernel/pi/pi.h Add pi.h header for declarations of the kernel/pi prefixed functions and any other related declarations. Suggested-by: Charlie Jenkins Signed-off-by: Jesse Taube Reviewed-by: Alexandre Ghiti Link: https://lore.kernel.org/r/20240709173937.510084-4-jesse@rivosinc.com Signed-off-by: Palmer Dabbelt --- arch/riscv/kernel/pi/cmdline_early.c | 10 ++-------- arch/riscv/kernel/pi/fdt_early.c | 7 +------ arch/riscv/kernel/pi/pi.h | 17 +++++++++++++++++ 3 files changed, 20 insertions(+), 14 deletions(-) create mode 100644 arch/riscv/kernel/pi/pi.h diff --git a/arch/riscv/kernel/pi/cmdline_early.c b/arch/riscv/kernel/pi/cmdline_early.c index f6d4dedffb84..fbcdc9e4e143 100644 --- a/arch/riscv/kernel/pi/cmdline_early.c +++ b/arch/riscv/kernel/pi/cmdline_early.c @@ -6,15 +6,9 @@ #include #include -static char early_cmdline[COMMAND_LINE_SIZE]; +#include "pi.h" -/* - * Declare the functions that are exported (but prefixed) here so that LLVM - * does not complain it lacks the 'static' keyword (which, if added, makes - * LLVM complain because the function is actually unused in this file). - */ -u64 set_satp_mode_from_cmdline(uintptr_t dtb_pa); -bool set_nokaslr_from_cmdline(uintptr_t dtb_pa); +static char early_cmdline[COMMAND_LINE_SIZE]; static char *get_early_cmdline(uintptr_t dtb_pa) { diff --git a/arch/riscv/kernel/pi/fdt_early.c b/arch/riscv/kernel/pi/fdt_early.c index 899610e042ab..40ee299702bf 100644 --- a/arch/riscv/kernel/pi/fdt_early.c +++ b/arch/riscv/kernel/pi/fdt_early.c @@ -3,12 +3,7 @@ #include #include -/* - * Declare the functions that are exported (but prefixed) here so that LLVM - * does not complain it lacks the 'static' keyword (which, if added, makes - * LLVM complain because the function is actually unused in this file). - */ -u64 get_kaslr_seed(uintptr_t dtb_pa); +#include "pi.h" u64 get_kaslr_seed(uintptr_t dtb_pa) { diff --git a/arch/riscv/kernel/pi/pi.h b/arch/riscv/kernel/pi/pi.h new file mode 100644 index 000000000000..493c8cb7c0e6 --- /dev/null +++ b/arch/riscv/kernel/pi/pi.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _RISCV_PI_H_ +#define _RISCV_PI_H_ + +#include + +/* + * The following functions are exported (but prefixed). Declare them here so + * that LLVM does not complain it lacks the 'static' keyword (which, if + * added, makes LLVM complain because the function is unused). + */ + +u64 get_kaslr_seed(uintptr_t dtb_pa); +bool set_nokaslr_from_cmdline(uintptr_t dtb_pa); +u64 set_satp_mode_from_cmdline(uintptr_t dtb_pa); + +#endif /* _RISCV_PI_H_ */ From 945302df3de156fc367d3b537cec76d4aea0b0d1 Mon Sep 17 00:00:00 2001 From: Jesse Taube Date: Tue, 9 Jul 2024 13:39:37 -0400 Subject: [PATCH 04/47] RISC-V: Use Zkr to seed KASLR base address Parse the device tree for Zkr in the isa string. If Zkr is present, use it to seed the kernel base address. On an ACPI system, as of this commit, there is no easy way to check if Zkr is present. Blindly running the instruction isn't an option as; we have to be able to trust the firmware. Signed-off-by: Jesse Taube Reviewed-by: Charlie Jenkins Reviewed-by: Alexandre Ghiti Tested-by: Zong Li Reviewed-by: Conor Dooley Link: https://lore.kernel.org/r/20240709173937.510084-5-jesse@rivosinc.com Signed-off-by: Palmer Dabbelt --- arch/riscv/kernel/pi/Makefile | 3 +- arch/riscv/kernel/pi/archrandom_early.c | 30 +++++ arch/riscv/kernel/pi/fdt_early.c | 160 ++++++++++++++++++++++++ arch/riscv/kernel/pi/pi.h | 3 + arch/riscv/mm/init.c | 5 +- 5 files changed, 199 insertions(+), 2 deletions(-) create mode 100644 arch/riscv/kernel/pi/archrandom_early.c diff --git a/arch/riscv/kernel/pi/Makefile b/arch/riscv/kernel/pi/Makefile index 1ef7584be0c3..d5bf1bc7de62 100644 --- a/arch/riscv/kernel/pi/Makefile +++ b/arch/riscv/kernel/pi/Makefile @@ -17,6 +17,7 @@ KBUILD_CFLAGS += -mcmodel=medany CFLAGS_cmdline_early.o += -D__NO_FORTIFY CFLAGS_lib-fdt_ro.o += -D__NO_FORTIFY +CFLAGS_fdt_early.o += -D__NO_FORTIFY $(obj)/%.pi.o: OBJCOPYFLAGS := --prefix-symbols=__pi_ \ --remove-section=.note.gnu.property \ @@ -33,5 +34,5 @@ $(obj)/string.o: $(srctree)/lib/string.c FORCE $(obj)/ctype.o: $(srctree)/lib/ctype.c FORCE $(call if_changed_rule,cc_o_c) -obj-y := cmdline_early.pi.o fdt_early.pi.o string.pi.o ctype.pi.o lib-fdt.pi.o lib-fdt_ro.pi.o +obj-y := cmdline_early.pi.o fdt_early.pi.o string.pi.o ctype.pi.o lib-fdt.pi.o lib-fdt_ro.pi.o archrandom_early.pi.o extra-y := $(patsubst %.pi.o,%.o,$(obj-y)) diff --git a/arch/riscv/kernel/pi/archrandom_early.c b/arch/riscv/kernel/pi/archrandom_early.c new file mode 100644 index 000000000000..3f05d3cf3b7b --- /dev/null +++ b/arch/riscv/kernel/pi/archrandom_early.c @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include +#include + +#include "pi.h" + +/* + * To avoid rewriting code include asm/archrandom.h and create macros + * for the functions that won't be included. + */ +#undef riscv_has_extension_unlikely +#define riscv_has_extension_likely(...) false +#undef pr_err_once +#define pr_err_once(...) + +#include + +u64 get_kaslr_seed_zkr(const uintptr_t dtb_pa) +{ + unsigned long seed = 0; + + if (!fdt_early_match_extension_isa((const void *)dtb_pa, "zkr")) + return 0; + + if (!csr_seed_long(&seed)) + return 0; + + return seed; +} diff --git a/arch/riscv/kernel/pi/fdt_early.c b/arch/riscv/kernel/pi/fdt_early.c index 40ee299702bf..9bdee2fafe47 100644 --- a/arch/riscv/kernel/pi/fdt_early.c +++ b/arch/riscv/kernel/pi/fdt_early.c @@ -2,6 +2,7 @@ #include #include #include +#include #include "pi.h" @@ -23,3 +24,162 @@ u64 get_kaslr_seed(uintptr_t dtb_pa) *prop = 0; return ret; } + +/** + * fdt_device_is_available - check if a device is available for use + * + * @fdt: pointer to the device tree blob + * @node: offset of the node whose property to find + * + * Returns true if the status property is absent or set to "okay" or "ok", + * false otherwise + */ +static bool fdt_device_is_available(const void *fdt, int node) +{ + const char *status; + int statlen; + + status = fdt_getprop(fdt, node, "status", &statlen); + if (!status) + return true; + + if (statlen > 0) { + if (!strcmp(status, "okay") || !strcmp(status, "ok")) + return true; + } + + return false; +} + +/* Copy of fdt_nodename_eq_ */ +static int fdt_node_name_eq(const void *fdt, int offset, + const char *s) +{ + int olen; + int len = strlen(s); + const char *p = fdt_get_name(fdt, offset, &olen); + + if (!p || olen < len) + /* short match */ + return 0; + + if (memcmp(p, s, len) != 0) + return 0; + + if (p[len] == '\0') + return 1; + else if (!memchr(s, '@', len) && (p[len] == '@')) + return 1; + else + return 0; +} + +/** + * isa_string_contains - check if isa string contains an extension + * + * @isa_str: isa string to search + * @ext_name: the extension to search for + * + * Returns true if the extension is in the given isa string, + * false otherwise + */ +static bool isa_string_contains(const char *isa_str, const char *ext_name) +{ + size_t i, single_end, len = strlen(ext_name); + char ext_end; + + /* Error must contain rv32/64 */ + if (strlen(isa_str) < 4) + return false; + + if (len == 1) { + single_end = strcspn(isa_str, "sSxXzZ"); + /* Search for single chars between rv32/64 and multi-letter extensions */ + for (i = 4; i < single_end; i++) { + if (tolower(isa_str[i]) == ext_name[0]) + return true; + } + return false; + } + + /* Skip to start of multi-letter extensions */ + isa_str = strpbrk(isa_str, "sSxXzZ"); + while (isa_str) { + if (strncasecmp(isa_str, ext_name, len) == 0) { + ext_end = isa_str[len]; + /* Check if matches the whole extension. */ + if (ext_end == '\0' || ext_end == '_') + return true; + } + /* Multi-letter extensions must be split from other multi-letter + * extensions with an "_", the end of a multi-letter extension will + * either be the null character or the "_" at the start of the next + * multi-letter extension. + */ + isa_str = strchr(isa_str, '_'); + if (isa_str) + isa_str++; + } + + return false; +} + +/** + * early_cpu_isa_ext_available - check if cpu node has an extension + * + * @fdt: pointer to the device tree blob + * @node: offset of the cpu node + * @ext_name: the extension to search for + * + * Returns true if the cpu node has the extension, + * false otherwise + */ +static bool early_cpu_isa_ext_available(const void *fdt, int node, const char *ext_name) +{ + const void *prop; + int len; + + prop = fdt_getprop(fdt, node, "riscv,isa-extensions", &len); + if (prop && fdt_stringlist_contains(prop, len, ext_name)) + return true; + + prop = fdt_getprop(fdt, node, "riscv,isa", &len); + if (prop && isa_string_contains(prop, ext_name)) + return true; + + return false; +} + +/** + * fdt_early_match_extension_isa - check if all cpu nodes have an extension + * + * @fdt: pointer to the device tree blob + * @ext_name: the extension to search for + * + * Returns true if the all available the cpu nodes have the extension, + * false otherwise + */ +bool fdt_early_match_extension_isa(const void *fdt, const char *ext_name) +{ + int node, parent; + bool ret = false; + + parent = fdt_path_offset(fdt, "/cpus"); + if (parent < 0) + return false; + + fdt_for_each_subnode(node, fdt, parent) { + if (!fdt_node_name_eq(fdt, node, "cpu")) + continue; + + if (!fdt_device_is_available(fdt, node)) + continue; + + if (!early_cpu_isa_ext_available(fdt, node, ext_name)) + return false; + + ret = true; + } + + return ret; +} diff --git a/arch/riscv/kernel/pi/pi.h b/arch/riscv/kernel/pi/pi.h index 493c8cb7c0e6..21141d84fea6 100644 --- a/arch/riscv/kernel/pi/pi.h +++ b/arch/riscv/kernel/pi/pi.h @@ -11,7 +11,10 @@ */ u64 get_kaslr_seed(uintptr_t dtb_pa); +u64 get_kaslr_seed_zkr(const uintptr_t dtb_pa); bool set_nokaslr_from_cmdline(uintptr_t dtb_pa); u64 set_satp_mode_from_cmdline(uintptr_t dtb_pa); +bool fdt_early_match_extension_isa(const void *fdt, const char *ext_name); + #endif /* _RISCV_PI_H_ */ diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c index e3218d65f21d..5551403a85ae 100644 --- a/arch/riscv/mm/init.c +++ b/arch/riscv/mm/init.c @@ -1042,6 +1042,7 @@ static void __init pt_ops_set_late(void) #ifdef CONFIG_RANDOMIZE_BASE extern bool __init __pi_set_nokaslr_from_cmdline(uintptr_t dtb_pa); extern u64 __init __pi_get_kaslr_seed(uintptr_t dtb_pa); +extern u64 __init __pi_get_kaslr_seed_zkr(const uintptr_t dtb_pa); static int __init print_nokaslr(char *p) { @@ -1062,10 +1063,12 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa) #ifdef CONFIG_RANDOMIZE_BASE if (!__pi_set_nokaslr_from_cmdline(dtb_pa)) { - u64 kaslr_seed = __pi_get_kaslr_seed(dtb_pa); + u64 kaslr_seed = __pi_get_kaslr_seed_zkr(dtb_pa); u32 kernel_size = (uintptr_t)(&_end) - (uintptr_t)(&_start); u32 nr_pos; + if (kaslr_seed == 0) + kaslr_seed = __pi_get_kaslr_seed(dtb_pa); /* * Compute the number of positions available: we are limited * by the early page table that only has one PUD and we must From f15c21a3de1b5321ba4ae2c6c8f2e63b839d49fa Mon Sep 17 00:00:00 2001 From: Ryo Takakura Date: Thu, 18 Jul 2024 18:36:59 +0900 Subject: [PATCH 05/47] RISC-V: Enable IPI CPU Backtrace Add arch_trigger_cpumask_backtrace() which is a generic infrastructure for sampling other CPUs' backtrace using IPI. The feature is used when lockups are detected or in case of oops/panic if parameters are set accordingly. Below is the case of oops with the oops_all_cpu_backtrace enabled. $ sysctl kernel.oops_all_cpu_backtrace=1 triggering oops shows: [ 212.214237] NMI backtrace for cpu 1 [ 212.214390] CPU: 1 PID: 610 Comm: in:imklog Tainted: G OE 6.10.0-rc6 #1 [ 212.214570] Hardware name: riscv-virtio,qemu (DT) [ 212.214690] epc : fallback_scalar_usercopy+0x8/0xdc [ 212.214809] ra : _copy_to_user+0x20/0x40 [ 212.214913] epc : ffffffff80c3a930 ra : ffffffff8059ba7e sp : ff20000000eabb50 [ 212.215061] gp : ffffffff82066f90 tp : ff6000008e958000 t0 : 3463303866660000 [ 212.215210] t1 : 000000000000005b t2 : 3463303866666666 s0 : ff20000000eabb60 [ 212.215358] s1 : 0000000000000386 a0 : 00007ff6e81df926 a1 : ff600000824df800 [ 212.215505] a2 : 000000000000003f a3 : 7fffffffffffffc0 a4 : 0000000000000000 [ 212.215651] a5 : 000000000000003f a6 : 0000000000000000 a7 : 0000000000000000 [ 212.215857] s2 : ff600000824df800 s3 : ffffffff82066cc0 s4 : 0000000000001c1a [ 212.216074] s5 : ffffffff8206a5a8 s6 : 00007ff6e81df926 s7 : ffffffff8206a5a0 [ 212.216278] s8 : ff600000824df800 s9 : ffffffff81e25de0 s10: 000000000000003f [ 212.216471] s11: ffffffff8206a59d t3 : ff600000824df812 t4 : ff600000824df812 [ 212.216651] t5 : ff600000824df818 t6 : 0000000000040000 [ 212.216796] status: 0000000000040120 badaddr: 0000000000000000 cause: 8000000000000001 [ 212.217035] [] fallback_scalar_usercopy+0x8/0xdc [ 212.217207] [] syslog_print+0x1f4/0x2b2 [ 212.217362] [] do_syslog.part.0+0x94/0x2d8 [ 212.217502] [] do_syslog+0x66/0x88 [ 212.217636] [] kmsg_read+0x44/0x5c [ 212.217764] [] proc_reg_read+0x7a/0xa8 [ 212.217952] [] vfs_read+0xb0/0x24e [ 212.218090] [] ksys_read+0x64/0xe4 [ 212.218264] [] __riscv_sys_read+0x20/0x2c [ 212.218453] [] do_trap_ecall_u+0x60/0x1d4 [ 212.218664] [] ret_from_exception+0x0/0x64 Signed-off-by: Ryo Takakura Link: https://lore.kernel.org/r/20240718093659.158912-1-takakura@valinux.co.jp Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/irq.h | 5 +++++ arch/riscv/kernel/smp.c | 16 ++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/arch/riscv/include/asm/irq.h b/arch/riscv/include/asm/irq.h index 8e10a94430a2..8330d16b05b5 100644 --- a/arch/riscv/include/asm/irq.h +++ b/arch/riscv/include/asm/irq.h @@ -12,6 +12,11 @@ #include +#ifdef CONFIG_SMP +void arch_trigger_cpumask_backtrace(const cpumask_t *mask, int exclude_cpu); +#define arch_trigger_cpumask_backtrace arch_trigger_cpumask_backtrace +#endif + void riscv_set_intc_hwnode_fn(struct fwnode_handle *(*fn)(void)); struct fwnode_handle *riscv_get_intc_hwnode(void); diff --git a/arch/riscv/kernel/smp.c b/arch/riscv/kernel/smp.c index 8e6eb64459af..9b047899791c 100644 --- a/arch/riscv/kernel/smp.c +++ b/arch/riscv/kernel/smp.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -33,6 +34,7 @@ enum ipi_message_type { IPI_CPU_CRASH_STOP, IPI_IRQ_WORK, IPI_TIMER, + IPI_CPU_BACKTRACE, IPI_MAX }; @@ -136,6 +138,9 @@ static irqreturn_t handle_IPI(int irq, void *data) tick_receive_broadcast(); break; #endif + case IPI_CPU_BACKTRACE: + nmi_cpu_backtrace(get_irq_regs()); + break; default: pr_warn("CPU%d: unhandled IPI%d\n", smp_processor_id(), ipi); break; @@ -203,6 +208,7 @@ static const char * const ipi_names[] = { [IPI_CPU_CRASH_STOP] = "CPU stop (for crash dump) interrupts", [IPI_IRQ_WORK] = "IRQ work interrupts", [IPI_TIMER] = "Timer broadcast interrupts", + [IPI_CPU_BACKTRACE] = "CPU backtrace interrupts", }; void show_ipi_stats(struct seq_file *p, int prec) @@ -323,3 +329,13 @@ void arch_smp_send_reschedule(int cpu) send_ipi_single(cpu, IPI_RESCHEDULE); } EXPORT_SYMBOL_GPL(arch_smp_send_reschedule); + +static void riscv_backtrace_ipi(cpumask_t *mask) +{ + send_ipi_mask(mask, IPI_CPU_BACKTRACE); +} + +void arch_trigger_cpumask_backtrace(const cpumask_t *mask, int exclude_cpu) +{ + nmi_trigger_cpumask_backtrace(mask, exclude_cpu, riscv_backtrace_ipi); +} From c6ebf2c528470a09be77d0d9df2c6617ea037ac5 Mon Sep 17 00:00:00 2001 From: Ying Sun Date: Thu, 11 Jul 2024 08:32:36 +0000 Subject: [PATCH 06/47] riscv/kexec_file: Fix relocation type R_RISCV_ADD16 and R_RISCV_SUB16 unknown Runs on the kernel with CONFIG_RISCV_ALTERNATIVE enabled: kexec -sl vmlinux Error: kexec_image: Unknown rela relocation: 34 kexec_image: Error loading purgatory ret=-8 and kexec_image: Unknown rela relocation: 38 kexec_image: Error loading purgatory ret=-8 The purgatory code uses the 16-bit addition and subtraction relocation type, but not handled, resulting in kexec_file_load failure. So add handle to arch_kexec_apply_relocations_add(). Tested on RISC-V64 Qemu-virt, issue fixed. Co-developed-by: Petr Tesarik Signed-off-by: Petr Tesarik Signed-off-by: Ying Sun Reviewed-by: Andrew Jones Link: https://lore.kernel.org/r/20240711083236.2859632-1-sunying@isrc.iscas.ac.cn Signed-off-by: Palmer Dabbelt --- arch/riscv/kernel/elf_kexec.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/arch/riscv/kernel/elf_kexec.c b/arch/riscv/kernel/elf_kexec.c index 11c0d2e0becf..3c37661801f9 100644 --- a/arch/riscv/kernel/elf_kexec.c +++ b/arch/riscv/kernel/elf_kexec.c @@ -451,6 +451,12 @@ int arch_kexec_apply_relocations_add(struct purgatory_info *pi, *(u32 *)loc = CLEAN_IMM(CJTYPE, *(u32 *)loc) | ENCODE_CJTYPE_IMM(val - addr); break; + case R_RISCV_ADD16: + *(u16 *)loc += val; + break; + case R_RISCV_SUB16: + *(u16 *)loc -= val; + break; case R_RISCV_ADD32: *(u32 *)loc += val; break; From 0e3f3649d44bf1b388a7613ade14c29cbdedf075 Mon Sep 17 00:00:00 2001 From: Jinjie Ruan Date: Wed, 3 Jul 2024 10:27:32 +0800 Subject: [PATCH 07/47] riscv: Enable generic CPU vulnerabilites support Currently x86, ARM and ARM64 support generic CPU vulnerabilites, but RISC-V not, such as: # cd /sys/devices/system/cpu/vulnerabilities/ x86: # cat spec_store_bypass Mitigation: Speculative Store Bypass disabled via prctl and seccomp # cat meltdown Not affected ARM64: # cat spec_store_bypass Mitigation: Speculative Store Bypass disabled via prctl and seccomp # cat meltdown Mitigation: PTI RISC-V: # cat /sys/devices/system/cpu/vulnerabilities # ... No such file or directory As SiFive RISC-V Core IP offerings are not affected by Meltdown and Spectre, it can use the default weak function as below: # cat spec_store_bypass Not affected # cat meltdown Not affected Link: https://www.sifive.cn/blog/sifive-statement-on-meltdown-and-spectre Signed-off-by: Jinjie Ruan Link: https://lore.kernel.org/r/20240703022732.2068316-1-ruanjinjie@huawei.com Signed-off-by: Palmer Dabbelt --- arch/riscv/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 0f3cd7c3a436..4871220061c4 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -92,6 +92,7 @@ config RISCV select GENERIC_ATOMIC64 if !64BIT select GENERIC_CLOCKEVENTS_BROADCAST if SMP select GENERIC_CPU_DEVICES + select GENERIC_CPU_VULNERABILITIES select GENERIC_EARLY_IOREMAP select GENERIC_ENTRY select GENERIC_GETTIMEOFDAY if HAVE_GENERIC_VDSO From 097c72e1f2b57d8b4d0b18c5a768a2028d666475 Mon Sep 17 00:00:00 2001 From: Charlie Jenkins Date: Mon, 29 Jul 2024 14:00:02 -0700 Subject: [PATCH 08/47] riscv: Add license to fence.h Add a missing license to fence.h. Signed-off-by: Charlie Jenkins Link: https://lore.kernel.org/r/20240729-riscv_fence_license-v1-1-7d5648069640@rivosinc.com Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/fence.h | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/riscv/include/asm/fence.h b/arch/riscv/include/asm/fence.h index 6bcd80325dfc..182db7930edc 100644 --- a/arch/riscv/include/asm/fence.h +++ b/arch/riscv/include/asm/fence.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ #ifndef _ASM_RISCV_FENCE_H #define _ASM_RISCV_FENCE_H From 4ffc8a3422986470972b4487f02e381692269098 Mon Sep 17 00:00:00 2001 From: Charlie Jenkins Date: Mon, 29 Jul 2024 14:00:03 -0700 Subject: [PATCH 09/47] riscv: Add license to vmalloc.h Add a missing license to vmalloc.h. Signed-off-by: Charlie Jenkins Link: https://lore.kernel.org/r/20240729-riscv_fence_license-v1-2-7d5648069640@rivosinc.com Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/vmalloc.h | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/riscv/include/asm/vmalloc.h b/arch/riscv/include/asm/vmalloc.h index 51f6dfe19745..fefe94dc98e2 100644 --- a/arch/riscv/include/asm/vmalloc.h +++ b/arch/riscv/include/asm/vmalloc.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ #ifndef _ASM_RISCV_VMALLOC_H #define _ASM_RISCV_VMALLOC_H From aa3457f22f00b9c9039672b1ff942fd72a5330d8 Mon Sep 17 00:00:00 2001 From: Nam Cao Date: Fri, 7 Jun 2024 22:22:06 +0200 Subject: [PATCH 10/47] riscv: cleanup XIP_FIXUP macro The XIP_FIXUP macro is used to fix addresses early during boot before MMU: generated code "thinks" the data section is in ROM while it is actually in RAM. So this macro corrects the addresses in the data section. This macro determines if the address needs to be fixed by checking if it is within the range starting from ROM address up to the size of (2 * XIP_OFFSET). This means if the kernel size is bigger than (2 * XIP_OFFSET), some addresses would not be fixed up. XIP kernel can still work if the above scenario does not happen. But this macro is obviously incorrect. Rewrite this macro to only fix up addresses within the data section. Signed-off-by: Nam Cao Reviewed-by: Alexandre Ghiti Link: https://lore.kernel.org/r/95f50a4ec8204ec4fcbf2a80c9addea0e0609e3b.1717789719.git.namcao@linutronix.de Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/pgtable.h | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index 089f3c9f56a3..6e16ee77619f 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -142,11 +142,14 @@ #ifdef CONFIG_XIP_KERNEL #define XIP_FIXUP(addr) ({ \ + extern char _sdata[], _start[], _end[]; \ + uintptr_t __rom_start_data = CONFIG_XIP_PHYS_ADDR \ + + (uintptr_t)&_sdata - (uintptr_t)&_start; \ + uintptr_t __rom_end_data = CONFIG_XIP_PHYS_ADDR \ + + (uintptr_t)&_end - (uintptr_t)&_start; \ uintptr_t __a = (uintptr_t)(addr); \ - (__a >= CONFIG_XIP_PHYS_ADDR && \ - __a < CONFIG_XIP_PHYS_ADDR + XIP_OFFSET * 2) ? \ - __a - CONFIG_XIP_PHYS_ADDR + CONFIG_PHYS_RAM_BASE - XIP_OFFSET :\ - __a; \ + (__a >= __rom_start_data && __a < __rom_end_data) ? \ + __a - __rom_start_data + CONFIG_PHYS_RAM_BASE : __a; \ }) #else #define XIP_FIXUP(addr) (addr) From f2df5b4fdd74a3490c35498de935ebf4f9b7c382 Mon Sep 17 00:00:00 2001 From: Nam Cao Date: Fri, 7 Jun 2024 22:22:07 +0200 Subject: [PATCH 11/47] riscv: don't export va_kernel_pa_offset in vmcoreinfo for XIP kernel The crash utility uses va_kernel_pa_offset to translate virtual addresses. This is incorrect in the case of XIP kernel, because va_kernel_pa_offset is not the virtual-physical address offset (yes, the name is misleading; this variable will be removed for XIP in a following commit). Stop exporting this variable for XIP kernel. The replacement is to be determined, note it as a TODO for now. Signed-off-by: Nam Cao Reviewed-by: Alexandre Ghiti Link: https://lore.kernel.org/r/8f8760d3f9a11af4ea0acbc247e4f49ff5d317e9.1717789719.git.namcao@linutronix.de Signed-off-by: Palmer Dabbelt --- arch/riscv/kernel/vmcore_info.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/arch/riscv/kernel/vmcore_info.c b/arch/riscv/kernel/vmcore_info.c index 6d7a22522d63..d5e448aa90e7 100644 --- a/arch/riscv/kernel/vmcore_info.c +++ b/arch/riscv/kernel/vmcore_info.c @@ -19,6 +19,13 @@ void arch_crash_save_vmcoreinfo(void) #endif #endif vmcoreinfo_append_str("NUMBER(KERNEL_LINK_ADDR)=0x%lx\n", KERNEL_LINK_ADDR); +#ifdef CONFIG_XIP_KERNEL + /* TODO: Communicate with crash-utility developers on the information to + * export. The XIP case is more complicated, because the virtual-physical + * address offset depends on whether the address is in ROM or in RAM. + */ +#else vmcoreinfo_append_str("NUMBER(va_kernel_pa_offset)=0x%lx\n", kernel_map.va_kernel_pa_offset); +#endif } From 5cf089672119808c2f5b7035c91adcc0cc7287e1 Mon Sep 17 00:00:00 2001 From: Nam Cao Date: Fri, 7 Jun 2024 22:22:08 +0200 Subject: [PATCH 12/47] riscv: replace misleading va_kernel_pa_offset on XIP kernel On XIP kernel, the name "va_kernel_pa_offset" is misleading: unlike "normal" kernel, it is not the virtual-physical address offset of kernel mapping, it is the offset of kernel mapping's first virtual address to first physical address in DRAM, which is not meaningful because the kernel's first physical address is not in DRAM. For XIP kernel, there are 2 different offsets because the read-only part of the kernel resides in ROM while the rest is in RAM. The offset to ROM is in kernel_map.va_kernel_xip_pa_offset, while the offset to RAM is not stored anywhere: it is calculated on-the-fly. Remove this confusing "va_kernel_pa_offset" and add "va_kernel_xip_data_pa_offset" as its replacement. This new variable is the offset of virtual mapping of the kernel's data portion to the corresponding physical addresses. With the introduction of this new variable, also rename va_kernel_xip_pa_offset -> va_kernel_xip_text_pa_offset to make it clear that this one is about the .text section. Signed-off-by: Nam Cao Reviewed-by: Alexandre Ghiti Link: https://lore.kernel.org/r/84e5d005c1386d88d7b2531e0b6707ec5352ee54.1717789719.git.namcao@linutronix.de Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/page.h | 29 +++++++++++++++++++++-------- arch/riscv/mm/init.c | 6 ++++-- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/arch/riscv/include/asm/page.h b/arch/riscv/include/asm/page.h index 7ede2111c591..4d7a8d5bb2d2 100644 --- a/arch/riscv/include/asm/page.h +++ b/arch/riscv/include/asm/page.h @@ -112,11 +112,13 @@ struct kernel_mapping { /* Offset between linear mapping virtual address and kernel load address */ unsigned long va_pa_offset; /* Offset between kernel mapping virtual address and kernel load address */ - unsigned long va_kernel_pa_offset; - unsigned long va_kernel_xip_pa_offset; #ifdef CONFIG_XIP_KERNEL + unsigned long va_kernel_xip_text_pa_offset; + unsigned long va_kernel_xip_data_pa_offset; uintptr_t xiprom; uintptr_t xiprom_sz; +#else + unsigned long va_kernel_pa_offset; #endif }; @@ -134,12 +136,18 @@ extern phys_addr_t phys_ram_base; #else void *linear_mapping_pa_to_va(unsigned long x); #endif + +#ifdef CONFIG_XIP_KERNEL #define kernel_mapping_pa_to_va(y) ({ \ unsigned long _y = (unsigned long)(y); \ - (IS_ENABLED(CONFIG_XIP_KERNEL) && _y < phys_ram_base) ? \ - (void *)(_y + kernel_map.va_kernel_xip_pa_offset) : \ - (void *)(_y + kernel_map.va_kernel_pa_offset + XIP_OFFSET); \ + (_y < phys_ram_base) ? \ + (void *)(_y + kernel_map.va_kernel_xip_text_pa_offset) : \ + (void *)(_y + kernel_map.va_kernel_xip_data_pa_offset); \ }) +#else +#define kernel_mapping_pa_to_va(y) ((void *)((unsigned long)(y) + kernel_map.va_kernel_pa_offset)) +#endif + #define __pa_to_va_nodebug(x) linear_mapping_pa_to_va(x) #ifndef CONFIG_DEBUG_VIRTUAL @@ -147,12 +155,17 @@ void *linear_mapping_pa_to_va(unsigned long x); #else phys_addr_t linear_mapping_va_to_pa(unsigned long x); #endif + +#ifdef CONFIG_XIP_KERNEL #define kernel_mapping_va_to_pa(y) ({ \ unsigned long _y = (unsigned long)(y); \ - (IS_ENABLED(CONFIG_XIP_KERNEL) && _y < kernel_map.virt_addr + XIP_OFFSET) ? \ - (_y - kernel_map.va_kernel_xip_pa_offset) : \ - (_y - kernel_map.va_kernel_pa_offset - XIP_OFFSET); \ + (_y < kernel_map.virt_addr + XIP_OFFSET) ? \ + (_y - kernel_map.va_kernel_xip_text_pa_offset) : \ + (_y - kernel_map.va_kernel_xip_data_pa_offset); \ }) +#else +#define kernel_mapping_va_to_pa(y) ((unsigned long)(y) - kernel_map.va_kernel_pa_offset) +#endif #define __va_to_pa_nodebug(x) ({ \ unsigned long _x = x; \ diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c index eb0649a61b4c..c2d63808d7ab 100644 --- a/arch/riscv/mm/init.c +++ b/arch/riscv/mm/init.c @@ -1098,11 +1098,14 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa) kernel_map.phys_addr = (uintptr_t)CONFIG_PHYS_RAM_BASE; kernel_map.size = (uintptr_t)(&_end) - (uintptr_t)(&_start); - kernel_map.va_kernel_xip_pa_offset = kernel_map.virt_addr - kernel_map.xiprom; + kernel_map.va_kernel_xip_text_pa_offset = kernel_map.virt_addr - kernel_map.xiprom; + kernel_map.va_kernel_xip_data_pa_offset = kernel_map.virt_addr - kernel_map.phys_addr + + (uintptr_t)&_sdata - (uintptr_t)&_start; #else kernel_map.page_offset = _AC(CONFIG_PAGE_OFFSET, UL); kernel_map.phys_addr = (uintptr_t)(&_start); kernel_map.size = (uintptr_t)(&_end) - kernel_map.phys_addr; + kernel_map.va_kernel_pa_offset = kernel_map.virt_addr - kernel_map.phys_addr; #endif #if defined(CONFIG_64BIT) && !defined(CONFIG_XIP_KERNEL) @@ -1124,7 +1127,6 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa) */ kernel_map.va_pa_offset = IS_ENABLED(CONFIG_64BIT) ? 0UL : PAGE_OFFSET - kernel_map.phys_addr; - kernel_map.va_kernel_pa_offset = kernel_map.virt_addr - kernel_map.phys_addr; /* * The default maximal physical memory size is KERN_VIRT_SIZE for 32-bit From e4eac34feda4959fad754004435512b24af73fe6 Mon Sep 17 00:00:00 2001 From: Nam Cao Date: Fri, 7 Jun 2024 22:22:09 +0200 Subject: [PATCH 13/47] riscv: drop the use of XIP_OFFSET in XIP_FIXUP_OFFSET XIP_OFFSET is the hard-coded offset of writable data section within the kernel. By hard-coding this value, the read-only section of the kernel (which is placed before the writable data section) is restricted in size. As a preparation to remove this hard-coded macro XIP_OFFSET entirely, stop using XIP_OFFSET in XIP_FIXUP_OFFSET. Instead, use CONFIG_PHYS_RAM_BASE and _sdata to do the same thing. While at it, also add a description for XIP_FIXUP_OFFSET. Signed-off-by: Nam Cao Reviewed-by: Alexandre Ghiti Link: https://lore.kernel.org/r/dba0409518b14ee83b346e099b1f7f934daf7b74.1717789719.git.namcao@linutronix.de Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/xip_fixup.h | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/arch/riscv/include/asm/xip_fixup.h b/arch/riscv/include/asm/xip_fixup.h index b65bf6306f69..9ed2cfae09e0 100644 --- a/arch/riscv/include/asm/xip_fixup.h +++ b/arch/riscv/include/asm/xip_fixup.h @@ -9,8 +9,19 @@ #ifdef CONFIG_XIP_KERNEL .macro XIP_FIXUP_OFFSET reg - REG_L t0, _xip_fixup + /* Fix-up address in Flash into address in RAM early during boot before + * MMU is up. Because generated code "thinks" data is in Flash, but it + * is actually in RAM (actually data is also in Flash, but Flash is + * read-only, thus we need to use the data residing in RAM). + * + * The start of data in Flash is _sdata and the start of data in RAM is + * CONFIG_PHYS_RAM_BASE. So this fix-up essentially does this: + * reg += CONFIG_PHYS_RAM_BASE - _start + */ + li t0, CONFIG_PHYS_RAM_BASE add \reg, \reg, t0 + la t0, _sdata + sub \reg, \reg, t0 .endm .macro XIP_FIXUP_FLASH_OFFSET reg la t0, __data_loc @@ -19,7 +30,6 @@ add \reg, \reg, t0 .endm -_xip_fixup: .dword CONFIG_PHYS_RAM_BASE - CONFIG_XIP_PHYS_ADDR - XIP_OFFSET _xip_phys_offset: .dword CONFIG_XIP_PHYS_ADDR + XIP_OFFSET #else .macro XIP_FIXUP_OFFSET reg From 23311f57ee132b2584915d38955b6cf125e76022 Mon Sep 17 00:00:00 2001 From: Nam Cao Date: Fri, 7 Jun 2024 22:22:10 +0200 Subject: [PATCH 14/47] riscv: drop the use of XIP_OFFSET in XIP_FIXUP_FLASH_OFFSET XIP_OFFSET is the hard-coded offset of writable data section within the kernel. By hard-coding this value, the read-only section of the kernel (which is placed before the writable data section) is restricted in size. As a preparation to remove this hard-coded macro XIP_OFFSET entirely, stop using XIP_OFFSET in XIP_FIXUP_FLASH_OFFSET. Instead, use __data_loc and _sdata to do the same thing. While at it, also add a description for XIP_FIXUP_FLASH_OFFSET. Signed-off-by: Nam Cao Reviewed-by: Alexandre Ghiti Link: https://lore.kernel.org/r/7b3319657edd1822f3457e7e7c07aaa326cc2f87.1717789719.git.namcao@linutronix.de Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/xip_fixup.h | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/arch/riscv/include/asm/xip_fixup.h b/arch/riscv/include/asm/xip_fixup.h index 9ed2cfae09e0..f3d56299bc22 100644 --- a/arch/riscv/include/asm/xip_fixup.h +++ b/arch/riscv/include/asm/xip_fixup.h @@ -24,13 +24,21 @@ sub \reg, \reg, t0 .endm .macro XIP_FIXUP_FLASH_OFFSET reg + /* In linker script, at the transition from read-only section to + * writable section, the VMA is increased while LMA remains the same. + * (See in linker script how _sdata, __data_loc and LOAD_OFFSET is + * changed) + * + * Consequently, early during boot before MMU is up, the generated code + * reads the "writable" section at wrong addresses, because VMA is used + * by compiler to generate code, but the data is located in Flash using + * LMA. + */ + la t0, _sdata + sub \reg, \reg, t0 la t0, __data_loc - REG_L t1, _xip_phys_offset - sub \reg, \reg, t1 add \reg, \reg, t0 .endm - -_xip_phys_offset: .dword CONFIG_XIP_PHYS_ADDR + XIP_OFFSET #else .macro XIP_FIXUP_OFFSET reg .endm From 75fdf791dff08470c035996f5d3e92c64491ce19 Mon Sep 17 00:00:00 2001 From: Nam Cao Date: Fri, 7 Jun 2024 22:22:11 +0200 Subject: [PATCH 15/47] riscv: drop the use of XIP_OFFSET in kernel_mapping_va_to_pa() XIP_OFFSET is the hard-coded offset of writable data section within the kernel. By hard-coding this value, the read-only section of the kernel (which is placed before the writable data section) is restricted in size. As a preparation to remove this hard-coded macro XIP_OFFSET entirely, remove the use of XIP_OFFSET in kernel_mapping_va_to_pa(). The macro XIP_OFFSET is used in this case to check if the virtual address is mapped to Flash or to RAM. The same check can be done with kernel_map.xiprom_sz. Signed-off-by: Nam Cao Reviewed-by: Alexandre Ghiti Link: https://lore.kernel.org/r/644c13d9467525a06f5d63d157875a35b2edb4bc.1717789719.git.namcao@linutronix.de Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/page.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/riscv/include/asm/page.h b/arch/riscv/include/asm/page.h index 4d7a8d5bb2d2..32d308a3355f 100644 --- a/arch/riscv/include/asm/page.h +++ b/arch/riscv/include/asm/page.h @@ -159,7 +159,7 @@ phys_addr_t linear_mapping_va_to_pa(unsigned long x); #ifdef CONFIG_XIP_KERNEL #define kernel_mapping_va_to_pa(y) ({ \ unsigned long _y = (unsigned long)(y); \ - (_y < kernel_map.virt_addr + XIP_OFFSET) ? \ + (_y < kernel_map.virt_addr + kernel_map.xiprom_sz) ? \ (_y - kernel_map.va_kernel_xip_text_pa_offset) : \ (_y - kernel_map.va_kernel_xip_data_pa_offset); \ }) From a7cfb999433ad3a1aa7ca86ecdaf3e061ab7076a Mon Sep 17 00:00:00 2001 From: Nam Cao Date: Fri, 7 Jun 2024 22:22:12 +0200 Subject: [PATCH 16/47] riscv: drop the use of XIP_OFFSET in create_kernel_page_table() XIP_OFFSET is the hard-coded offset of writable data section within the kernel. By hard-coding this value, the read-only section of the kernel (which is placed before the writable data section) is restricted in size. As a preparation to remove this hard-coded value entirely, stop using XIP_OFFSET in create_kernel_page_table(). Instead use _sdata and _start to do the same thing. Signed-off-by: Nam Cao Reviewed-by: Alexandre Ghiti Link: https://lore.kernel.org/r/4ea3f222a7eb9f91c04b155ff2e4d3ef19158acc.1717789719.git.namcao@linutronix.de Signed-off-by: Palmer Dabbelt --- arch/riscv/mm/init.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c index c2d63808d7ab..31b23f84494f 100644 --- a/arch/riscv/mm/init.c +++ b/arch/riscv/mm/init.c @@ -917,7 +917,7 @@ static void __init relocate_kernel(void) static void __init create_kernel_page_table(pgd_t *pgdir, __always_unused bool early) { - uintptr_t va, end_va; + uintptr_t va, start_va, end_va; /* Map the flash resident part */ end_va = kernel_map.virt_addr + kernel_map.xiprom_sz; @@ -927,10 +927,11 @@ static void __init create_kernel_page_table(pgd_t *pgdir, PMD_SIZE, PAGE_KERNEL_EXEC); /* Map the data in RAM */ + start_va = kernel_map.virt_addr + (uintptr_t)&_sdata - (uintptr_t)&_start; end_va = kernel_map.virt_addr + kernel_map.size; - for (va = kernel_map.virt_addr + XIP_OFFSET; va < end_va; va += PMD_SIZE) + for (va = start_va; va < end_va; va += PMD_SIZE) create_pgd_mapping(pgdir, va, - kernel_map.phys_addr + (va - (kernel_map.virt_addr + XIP_OFFSET)), + kernel_map.phys_addr + (va - start_va), PMD_SIZE, PAGE_KERNEL); } #else From b635a84bde6f0b94944b5d960f4e6a52ac3555c6 Mon Sep 17 00:00:00 2001 From: Nam Cao Date: Fri, 7 Jun 2024 22:22:13 +0200 Subject: [PATCH 17/47] riscv: remove limit on the size of read-only section for XIP kernel XIP_OFFSET is the hard-coded offset of writable data section within the kernel. By hard-coding this value, the read-only section of the kernel (which is placed before the writable data section) is restricted in size. This causes build failures if the kernel gets too big [1]. Remove this limit. Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202404211031.J6l2AfJk-lkp@intel.com [1] Signed-off-by: Nam Cao Reviewed-by: Alexandre Ghiti Link: https://lore.kernel.org/r/3bf3a77be10ebb0d8086c028500baa16e7a8e648.1717789719.git.namcao@linutronix.de Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/pgtable.h | 7 ------- arch/riscv/include/asm/set_memory.h | 2 +- arch/riscv/kernel/vmlinux-xip.lds.S | 5 +++-- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index 6e16ee77619f..ede2f921967c 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -107,13 +107,6 @@ #endif -#ifdef CONFIG_XIP_KERNEL -#define XIP_OFFSET SZ_32M -#define XIP_OFFSET_MASK (SZ_32M - 1) -#else -#define XIP_OFFSET 0 -#endif - #ifndef __ASSEMBLY__ #include diff --git a/arch/riscv/include/asm/set_memory.h b/arch/riscv/include/asm/set_memory.h index ec11001c3fe0..ab92fc84e1fc 100644 --- a/arch/riscv/include/asm/set_memory.h +++ b/arch/riscv/include/asm/set_memory.h @@ -46,7 +46,7 @@ bool kernel_page_present(struct page *page); #endif /* __ASSEMBLY__ */ -#ifdef CONFIG_STRICT_KERNEL_RWX +#if defined(CONFIG_STRICT_KERNEL_RWX) || defined(CONFIG_XIP_KERNEL) #ifdef CONFIG_64BIT #define SECTION_ALIGN (1 << 21) #else diff --git a/arch/riscv/kernel/vmlinux-xip.lds.S b/arch/riscv/kernel/vmlinux-xip.lds.S index 8c3daa1b0531..a7611789bad5 100644 --- a/arch/riscv/kernel/vmlinux-xip.lds.S +++ b/arch/riscv/kernel/vmlinux-xip.lds.S @@ -14,6 +14,7 @@ #include #include #include +#include OUTPUT_ARCH(riscv) ENTRY(_start) @@ -65,10 +66,10 @@ SECTIONS * From this point, stuff is considered writable and will be copied to RAM */ __data_loc = ALIGN(PAGE_SIZE); /* location in file */ - . = KERNEL_LINK_ADDR + XIP_OFFSET; /* location in memory */ + . = ALIGN(SECTION_ALIGN); /* location in memory */ #undef LOAD_OFFSET -#define LOAD_OFFSET (KERNEL_LINK_ADDR + XIP_OFFSET - (__data_loc & XIP_OFFSET_MASK)) +#define LOAD_OFFSET (KERNEL_LINK_ADDR + _sdata - __data_loc) _sdata = .; /* Start of data section */ _data = .; From 7587a3602bf2f12e798ea955b17521948ddb56a5 Mon Sep 17 00:00:00 2001 From: Changbin Du Date: Tue, 11 Jun 2024 12:09:47 +0800 Subject: [PATCH 18/47] riscv: vdso: do not strip debugging info for vdso.so.dbg The vdso.so.dbg is a debug version of vdso and could be used for debugging purpose. For example, perf-annotate requires debugging info to show source lines. So let's keep its debugging info. Signed-off-by: Changbin Du Reviewed-by: Cyril Bur Reviewed-by: Alexandre Ghiti Link: https://lore.kernel.org/r/20240611040947.3024710-1-changbin.du@huawei.com Signed-off-by: Palmer Dabbelt --- arch/riscv/kernel/vdso/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/riscv/kernel/vdso/Makefile b/arch/riscv/kernel/vdso/Makefile index f7ef8ad9b550..960feb1526ca 100644 --- a/arch/riscv/kernel/vdso/Makefile +++ b/arch/riscv/kernel/vdso/Makefile @@ -45,7 +45,7 @@ $(obj)/vdso.o: $(obj)/vdso.so # link rule for the .so file, .lds has to be first $(obj)/vdso.so.dbg: $(obj)/vdso.lds $(obj-vdso) FORCE $(call if_changed,vdsold) -LDFLAGS_vdso.so.dbg = -shared -S -soname=linux-vdso.so.1 \ +LDFLAGS_vdso.so.dbg = -shared -soname=linux-vdso.so.1 \ --build-id=sha1 --hash-style=both --eh-frame-hdr # strip rule for the .so file From d6a1928134a1c626ff369129d7a80951b2949a48 Mon Sep 17 00:00:00 2001 From: Stuart Menefy Date: Mon, 24 Jun 2024 13:17:23 +0100 Subject: [PATCH 19/47] riscv: Remove redundant restriction on memory size The original reason for reserving the top 4GiB of the direct map (space for modules/BPF/kernel) hasn't applied since the address map was reworked for KASAN. Signed-off-by: Stuart Menefy Reviewed-by: Alexandre Ghiti Link: https://lore.kernel.org/r/20240624121723.2186279-1-stuart.menefy@codasip.com Signed-off-by: Palmer Dabbelt --- arch/riscv/mm/init.c | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c index f43139db82a9..0e56725377a4 100644 --- a/arch/riscv/mm/init.c +++ b/arch/riscv/mm/init.c @@ -1132,13 +1132,7 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa) kernel_map.va_pa_offset = IS_ENABLED(CONFIG_64BIT) ? 0UL : PAGE_OFFSET - kernel_map.phys_addr; - /* - * The default maximal physical memory size is KERN_VIRT_SIZE for 32-bit - * kernel, whereas for 64-bit kernel, the end of the virtual address - * space is occupied by the modules/BPF/kernel mappings which reduces - * the available size of the linear mapping. - */ - memory_limit = KERN_VIRT_SIZE - (IS_ENABLED(CONFIG_64BIT) ? SZ_4G : 0); + memory_limit = KERN_VIRT_SIZE; /* Sanity check alignment and size */ BUG_ON((PAGE_OFFSET % PGDIR_SIZE) != 0); From 22ab08955ea13be04a8efd20cc30890e0afaa49c Mon Sep 17 00:00:00 2001 From: Jinjie Ruan Date: Mon, 8 Jul 2024 11:28:46 +0800 Subject: [PATCH 20/47] riscv: Fix fp alignment bug in perf_callchain_user() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The standard RISC-V calling convention said: "The stack grows downward and the stack pointer is always kept 16-byte aligned". So perf_callchain_user() should check whether 16-byte aligned for fp. Link: https://riscv.org/wp-content/uploads/2015/01/riscv-calling.pdf Fixes: dbeb90b0c1eb ("riscv: Add perf callchain support") Signed-off-by: Jinjie Ruan Cc: Björn Töpel Link: https://lore.kernel.org/r/20240708032847.2998158-2-ruanjinjie@huawei.com Signed-off-by: Palmer Dabbelt --- arch/riscv/kernel/perf_callchain.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/riscv/kernel/perf_callchain.c b/arch/riscv/kernel/perf_callchain.c index 3348a61de7d9..2932791e9388 100644 --- a/arch/riscv/kernel/perf_callchain.c +++ b/arch/riscv/kernel/perf_callchain.c @@ -62,7 +62,7 @@ void perf_callchain_user(struct perf_callchain_entry_ctx *entry, perf_callchain_store(entry, regs->epc); fp = user_backtrace(entry, fp, regs->ra); - while (fp && !(fp & 0x3) && entry->nr < entry->max_stack) + while (fp && !(fp & 0x7) && entry->nr < entry->max_stack) fp = user_backtrace(entry, fp, 0); } From 1a7483318274d0ec60f160e604c2a1dbce27fc0a Mon Sep 17 00:00:00 2001 From: Jinjie Ruan Date: Mon, 8 Jul 2024 11:28:47 +0800 Subject: [PATCH 21/47] riscv: stacktrace: Add USER_STACKTRACE support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently, userstacktrace is unsupported for riscv. So use the perf_callchain_user() code as blueprint to implement the arch_stack_walk_user() which add userstacktrace support on riscv. Meanwhile, we can use arch_stack_walk_user() to simplify the implementation of perf_callchain_user(). A ftrace test case is shown as below: # cd /sys/kernel/debug/tracing # echo 1 > options/userstacktrace # echo 1 > options/sym-userobj # echo 1 > events/sched/sched_process_fork/enable # cat trace ...... bash-178 [000] ...1. 97.968395: sched_process_fork: comm=bash pid=178 child_comm=bash child_pid=231 bash-178 [000] ...1. 97.970075: => /lib/libc.so.6[+0xb5090] Also a simple perf test is ok as below: # perf record -e cpu-clock --call-graph fp top # perf report --call-graph ..... [[31m 66.54%[[m 0.00% top [kernel.kallsyms] [k] ret_from_exception | ---ret_from_exception | |--[[31m58.97%[[m--do_trap_ecall_u | | | |--[[31m17.34%[[m--__riscv_sys_read | | ksys_read | | | | | --[[31m16.88%[[m--vfs_read | | | | | |--[[31m10.90%[[m--seq_read Signed-off-by: Jinjie Ruan Tested-by: Jinjie Ruan Cc: Björn Töpel Link: https://lore.kernel.org/r/20240708032847.2998158-3-ruanjinjie@huawei.com Signed-off-by: Palmer Dabbelt --- arch/riscv/Kconfig | 1 + arch/riscv/kernel/perf_callchain.c | 46 ++---------------------------- arch/riscv/kernel/stacktrace.c | 43 ++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 43 deletions(-) diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 0f3cd7c3a436..443bac99c94a 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -200,6 +200,7 @@ config RISCV select THREAD_INFO_IN_TASK select TRACE_IRQFLAGS_SUPPORT select UACCESS_MEMCPY if !MMU + select USER_STACKTRACE_SUPPORT select ZONE_DMA32 if 64BIT config CLANG_SUPPORTS_DYNAMIC_FTRACE diff --git a/arch/riscv/kernel/perf_callchain.c b/arch/riscv/kernel/perf_callchain.c index 2932791e9388..c7468af77c66 100644 --- a/arch/riscv/kernel/perf_callchain.c +++ b/arch/riscv/kernel/perf_callchain.c @@ -6,37 +6,9 @@ #include -/* - * Get the return address for a single stackframe and return a pointer to the - * next frame tail. - */ -static unsigned long user_backtrace(struct perf_callchain_entry_ctx *entry, - unsigned long fp, unsigned long reg_ra) +static bool fill_callchain(void *entry, unsigned long pc) { - struct stackframe buftail; - unsigned long ra = 0; - unsigned long __user *user_frame_tail = - (unsigned long __user *)(fp - sizeof(struct stackframe)); - - /* Check accessibility of one struct frame_tail beyond */ - if (!access_ok(user_frame_tail, sizeof(buftail))) - return 0; - if (__copy_from_user_inatomic(&buftail, user_frame_tail, - sizeof(buftail))) - return 0; - - if (reg_ra != 0) - ra = reg_ra; - else - ra = buftail.ra; - - fp = buftail.fp; - if (ra != 0) - perf_callchain_store(entry, ra); - else - return 0; - - return fp; + return perf_callchain_store(entry, pc) == 0; } /* @@ -56,19 +28,7 @@ static unsigned long user_backtrace(struct perf_callchain_entry_ctx *entry, void perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) { - unsigned long fp = 0; - - fp = regs->s0; - perf_callchain_store(entry, regs->epc); - - fp = user_backtrace(entry, fp, regs->ra); - while (fp && !(fp & 0x7) && entry->nr < entry->max_stack) - fp = user_backtrace(entry, fp, 0); -} - -static bool fill_callchain(void *entry, unsigned long pc) -{ - return perf_callchain_store(entry, pc) == 0; + arch_stack_walk_user(fill_callchain, entry, regs); } void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, diff --git a/arch/riscv/kernel/stacktrace.c b/arch/riscv/kernel/stacktrace.c index c6d5de22463f..153a2db4c5fa 100644 --- a/arch/riscv/kernel/stacktrace.c +++ b/arch/riscv/kernel/stacktrace.c @@ -162,3 +162,46 @@ noinline noinstr void arch_stack_walk(stack_trace_consume_fn consume_entry, void { walk_stackframe(task, regs, consume_entry, cookie); } + +/* + * Get the return address for a single stackframe and return a pointer to the + * next frame tail. + */ +static unsigned long unwind_user_frame(stack_trace_consume_fn consume_entry, + void *cookie, unsigned long fp, + unsigned long reg_ra) +{ + struct stackframe buftail; + unsigned long ra = 0; + unsigned long __user *user_frame_tail = + (unsigned long __user *)(fp - sizeof(struct stackframe)); + + /* Check accessibility of one struct frame_tail beyond */ + if (!access_ok(user_frame_tail, sizeof(buftail))) + return 0; + if (__copy_from_user_inatomic(&buftail, user_frame_tail, + sizeof(buftail))) + return 0; + + ra = reg_ra ? : buftail.ra; + + fp = buftail.fp; + if (!ra || !consume_entry(cookie, ra)) + return 0; + + return fp; +} + +void arch_stack_walk_user(stack_trace_consume_fn consume_entry, void *cookie, + const struct pt_regs *regs) +{ + unsigned long fp = 0; + + fp = regs->s0; + if (!consume_entry(cookie, regs->epc)) + return; + + fp = unwind_user_frame(consume_entry, cookie, fp, regs->ra); + while (fp && !(fp & 0x7)) + fp = unwind_user_frame(consume_entry, cookie, fp, 0); +} From 6868d12e0205765c1296c7c5e81e515ec015ab89 Mon Sep 17 00:00:00 2001 From: Jisheng Zhang Date: Wed, 10 Jul 2024 00:05:21 +0800 Subject: [PATCH 22/47] riscv: errata: sifive: Use SYM_*() assembly macros MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ENTRY()/END() macros are deprecated and we should make use of the new SYM_*() macros [1] for better annotation of symbols. Replace the deprecated ones with the new ones. [1] https://docs.kernel.org/core-api/asm-annotations.html Signed-off-by: Jisheng Zhang Reviewed-By: Clément Léger Link: https://lore.kernel.org/r/20240709160536.3690-2-jszhang@kernel.org Signed-off-by: Palmer Dabbelt --- arch/riscv/errata/sifive/errata_cip_453.S | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/riscv/errata/sifive/errata_cip_453.S b/arch/riscv/errata/sifive/errata_cip_453.S index f1b9623fe1de..b1f7b636fe9a 100644 --- a/arch/riscv/errata/sifive/errata_cip_453.S +++ b/arch/riscv/errata/sifive/errata_cip_453.S @@ -21,7 +21,7 @@ 1: .endm -ENTRY(sifive_cip_453_page_fault_trp) +SYM_FUNC_START(sifive_cip_453_page_fault_trp) ADD_SIGN_EXT a0, t0, t1 #ifdef CONFIG_MMU la t0, do_page_fault @@ -29,10 +29,10 @@ ENTRY(sifive_cip_453_page_fault_trp) la t0, do_trap_unknown #endif jr t0 -END(sifive_cip_453_page_fault_trp) +SYM_FUNC_END(sifive_cip_453_page_fault_trp) -ENTRY(sifive_cip_453_insn_fault_trp) +SYM_FUNC_START(sifive_cip_453_insn_fault_trp) ADD_SIGN_EXT a0, t0, t1 la t0, do_trap_insn_fault jr t0 -END(sifive_cip_453_insn_fault_trp) +SYM_FUNC_END(sifive_cip_453_insn_fault_trp) From 7c9d980e46703029e9f149ecf26c6a263d8923b6 Mon Sep 17 00:00:00 2001 From: Jisheng Zhang Date: Wed, 10 Jul 2024 00:05:22 +0800 Subject: [PATCH 23/47] riscv: select ARCH_USE_SYM_ANNOTATIONS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now, riscv has been converted to the new style SYM_ assembler annotations. So select ARCH_USE_SYM_ANNOTATIONS to ensure the deprecated macros such as ENTRY(), END(), WEAK() and so on are not available and we don't regress. Signed-off-by: Jisheng Zhang Reviewed-By: Clément Léger Link: https://lore.kernel.org/r/20240709160536.3690-3-jszhang@kernel.org Signed-off-by: Palmer Dabbelt --- arch/riscv/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 0f3cd7c3a436..e97bf2a06c31 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -68,6 +68,7 @@ config RISCV select ARCH_USE_CMPXCHG_LOCKREF if 64BIT select ARCH_USE_MEMTEST select ARCH_USE_QUEUED_RWLOCKS + select ARCH_USE_SYM_ANNOTATIONS select ARCH_USES_CFI_TRAPS if CFI_CLANG select ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH if MMU select ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT if MMU From a6efe33cc5945c8d435d2441ecc4e0ca7c49e040 Mon Sep 17 00:00:00 2001 From: Alexandre Ghiti Date: Wed, 17 Jul 2024 08:01:22 +0200 Subject: [PATCH 24/47] riscv: Add ISA extension parsing for Svvptc Add support to parse the Svvptc string in the riscv,isa string. Signed-off-by: Alexandre Ghiti Reviewed-by: Conor Dooley Link: https://lore.kernel.org/r/20240717060125.139416-2-alexghiti@rivosinc.com Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/hwcap.h | 1 + arch/riscv/kernel/cpufeature.c | 1 + 2 files changed, 2 insertions(+) diff --git a/arch/riscv/include/asm/hwcap.h b/arch/riscv/include/asm/hwcap.h index e17d0078a651..6dd0dd8beb30 100644 --- a/arch/riscv/include/asm/hwcap.h +++ b/arch/riscv/include/asm/hwcap.h @@ -81,6 +81,7 @@ #define RISCV_ISA_EXT_ZTSO 72 #define RISCV_ISA_EXT_ZACAS 73 #define RISCV_ISA_EXT_XANDESPMU 74 +#define RISCV_ISA_EXT_SVVPTC 75 #define RISCV_ISA_EXT_XLINUXENVCFG 127 diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c index 5ef48cb20ee1..60780d246743 100644 --- a/arch/riscv/kernel/cpufeature.c +++ b/arch/riscv/kernel/cpufeature.c @@ -305,6 +305,7 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = { __RISCV_ISA_EXT_DATA(svnapot, RISCV_ISA_EXT_SVNAPOT), __RISCV_ISA_EXT_DATA(svpbmt, RISCV_ISA_EXT_SVPBMT), __RISCV_ISA_EXT_DATA(xandespmu, RISCV_ISA_EXT_XANDESPMU), + __RISCV_ISA_EXT_DATA(svvptc, RISCV_ISA_EXT_SVVPTC), }; const size_t riscv_isa_ext_count = ARRAY_SIZE(riscv_isa_ext); From d25599b5933fb5f89d4b4c720564d613a795f502 Mon Sep 17 00:00:00 2001 From: Alexandre Ghiti Date: Wed, 17 Jul 2024 08:01:23 +0200 Subject: [PATCH 25/47] dt-bindings: riscv: Add Svvptc ISA extension description Add description for the Svvptc ISA extension which was ratified recently. Signed-off-by: Alexandre Ghiti Acked-by: Conor Dooley Link: https://lore.kernel.org/r/20240717060125.139416-3-alexghiti@rivosinc.com Signed-off-by: Palmer Dabbelt --- Documentation/devicetree/bindings/riscv/extensions.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Documentation/devicetree/bindings/riscv/extensions.yaml b/Documentation/devicetree/bindings/riscv/extensions.yaml index 468c646247aa..b52375bea512 100644 --- a/Documentation/devicetree/bindings/riscv/extensions.yaml +++ b/Documentation/devicetree/bindings/riscv/extensions.yaml @@ -171,6 +171,13 @@ properties: memory types as ratified in the 20191213 version of the privileged ISA specification. + - const: svvptc + description: + The standard Svvptc supervisor-level extension for + address-translation cache behaviour with respect to invalid entries + as ratified at commit 4a69197e5617 ("Update to ratified state") of + riscv-svvptc. + - const: zacas description: | The Zacas extension for Atomic Compare-and-Swap (CAS) instructions From 503638e0babf364061bc50fca5103b00a56cc50a Mon Sep 17 00:00:00 2001 From: Alexandre Ghiti Date: Wed, 17 Jul 2024 08:01:24 +0200 Subject: [PATCH 26/47] riscv: Stop emitting preventive sfence.vma for new vmalloc mappings In 6.5, we removed the vmalloc fault path because that can't work (see [1] [2]). Then in order to make sure that new page table entries were seen by the page table walker, we had to preventively emit a sfence.vma on all harts [3] but this solution is very costly since it relies on IPI. And even there, we could end up in a loop of vmalloc faults if a vmalloc allocation is done in the IPI path (for example if it is traced, see [4]), which could result in a kernel stack overflow. Those preventive sfence.vma needed to be emitted because: - if the uarch caches invalid entries, the new mapping may not be observed by the page table walker and an invalidation may be needed. - if the uarch does not cache invalid entries, a reordered access could "miss" the new mapping and traps: in that case, we would actually only need to retry the access, no sfence.vma is required. So this patch removes those preventive sfence.vma and actually handles the possible (and unlikely) exceptions. And since the kernel stacks mappings lie in the vmalloc area, this handling must be done very early when the trap is taken, at the very beginning of handle_exception: this also rules out the vmalloc allocations in the fault path. Link: https://lore.kernel.org/linux-riscv/20230531093817.665799-1-bjorn@kernel.org/ [1] Link: https://lore.kernel.org/linux-riscv/20230801090927.2018653-1-dylan@andestech.com [2] Link: https://lore.kernel.org/linux-riscv/20230725132246.817726-1-alexghiti@rivosinc.com/ [3] Link: https://lore.kernel.org/lkml/20200508144043.13893-1-joro@8bytes.org/ [4] Signed-off-by: Alexandre Ghiti Reviewed-by: Yunhui Cui Link: https://lore.kernel.org/r/20240717060125.139416-4-alexghiti@rivosinc.com Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/cacheflush.h | 18 +++++- arch/riscv/include/asm/thread_info.h | 7 +++ arch/riscv/kernel/asm-offsets.c | 7 +++ arch/riscv/kernel/entry.S | 87 ++++++++++++++++++++++++++++ arch/riscv/mm/init.c | 2 + 5 files changed, 120 insertions(+), 1 deletion(-) diff --git a/arch/riscv/include/asm/cacheflush.h b/arch/riscv/include/asm/cacheflush.h index ce79c558a4c8..8de73f91bfa3 100644 --- a/arch/riscv/include/asm/cacheflush.h +++ b/arch/riscv/include/asm/cacheflush.h @@ -46,7 +46,23 @@ do { \ } while (0) #ifdef CONFIG_64BIT -#define flush_cache_vmap(start, end) flush_tlb_kernel_range(start, end) +extern u64 new_vmalloc[NR_CPUS / sizeof(u64) + 1]; +extern char _end[]; +#define flush_cache_vmap flush_cache_vmap +static inline void flush_cache_vmap(unsigned long start, unsigned long end) +{ + if (is_vmalloc_or_module_addr((void *)start)) { + int i; + + /* + * We don't care if concurrently a cpu resets this value since + * the only place this can happen is in handle_exception() where + * an sfence.vma is emitted. + */ + for (i = 0; i < ARRAY_SIZE(new_vmalloc); ++i) + new_vmalloc[i] = -1ULL; + } +} #define flush_cache_vmap_early(start, end) local_flush_tlb_kernel_range(start, end) #endif diff --git a/arch/riscv/include/asm/thread_info.h b/arch/riscv/include/asm/thread_info.h index 5d473343634b..0ddf1123b5ba 100644 --- a/arch/riscv/include/asm/thread_info.h +++ b/arch/riscv/include/asm/thread_info.h @@ -60,6 +60,13 @@ struct thread_info { void *scs_base; void *scs_sp; #endif +#ifdef CONFIG_64BIT + /* + * Used in handle_exception() to save a0, a1 and a2 before knowing if we + * can access the kernel stack. + */ + unsigned long a0, a1, a2; +#endif }; #ifdef CONFIG_SHADOW_CALL_STACK diff --git a/arch/riscv/kernel/asm-offsets.c b/arch/riscv/kernel/asm-offsets.c index b09ca5f944f7..e94180ba432f 100644 --- a/arch/riscv/kernel/asm-offsets.c +++ b/arch/riscv/kernel/asm-offsets.c @@ -36,6 +36,8 @@ void asm_offsets(void) OFFSET(TASK_THREAD_S9, task_struct, thread.s[9]); OFFSET(TASK_THREAD_S10, task_struct, thread.s[10]); OFFSET(TASK_THREAD_S11, task_struct, thread.s[11]); + + OFFSET(TASK_TI_CPU, task_struct, thread_info.cpu); OFFSET(TASK_TI_FLAGS, task_struct, thread_info.flags); OFFSET(TASK_TI_PREEMPT_COUNT, task_struct, thread_info.preempt_count); OFFSET(TASK_TI_KERNEL_SP, task_struct, thread_info.kernel_sp); @@ -43,6 +45,11 @@ void asm_offsets(void) #ifdef CONFIG_SHADOW_CALL_STACK OFFSET(TASK_TI_SCS_SP, task_struct, thread_info.scs_sp); #endif +#ifdef CONFIG_64BIT + OFFSET(TASK_TI_A0, task_struct, thread_info.a0); + OFFSET(TASK_TI_A1, task_struct, thread_info.a1); + OFFSET(TASK_TI_A2, task_struct, thread_info.a2); +#endif OFFSET(TASK_TI_CPU_NUM, task_struct, thread_info.cpu); OFFSET(TASK_THREAD_F0, task_struct, thread.fstate.f[0]); diff --git a/arch/riscv/kernel/entry.S b/arch/riscv/kernel/entry.S index 68a24cf9481a..d80b90f99bc1 100644 --- a/arch/riscv/kernel/entry.S +++ b/arch/riscv/kernel/entry.S @@ -19,6 +19,79 @@ .section .irqentry.text, "ax" +.macro new_vmalloc_check + REG_S a0, TASK_TI_A0(tp) + csrr a0, CSR_CAUSE + /* Exclude IRQs */ + blt a0, zero, _new_vmalloc_restore_context_a0 + + REG_S a1, TASK_TI_A1(tp) + /* Only check new_vmalloc if we are in page/protection fault */ + li a1, EXC_LOAD_PAGE_FAULT + beq a0, a1, _new_vmalloc_kernel_address + li a1, EXC_STORE_PAGE_FAULT + beq a0, a1, _new_vmalloc_kernel_address + li a1, EXC_INST_PAGE_FAULT + bne a0, a1, _new_vmalloc_restore_context_a1 + +_new_vmalloc_kernel_address: + /* Is it a kernel address? */ + csrr a0, CSR_TVAL + bge a0, zero, _new_vmalloc_restore_context_a1 + + /* Check if a new vmalloc mapping appeared that could explain the trap */ + REG_S a2, TASK_TI_A2(tp) + /* + * Computes: + * a0 = &new_vmalloc[BIT_WORD(cpu)] + * a1 = BIT_MASK(cpu) + */ + REG_L a2, TASK_TI_CPU(tp) + /* + * Compute the new_vmalloc element position: + * (cpu / 64) * 8 = (cpu >> 6) << 3 + */ + srli a1, a2, 6 + slli a1, a1, 3 + la a0, new_vmalloc + add a0, a0, a1 + /* + * Compute the bit position in the new_vmalloc element: + * bit_pos = cpu % 64 = cpu - (cpu / 64) * 64 = cpu - (cpu >> 6) << 6 + * = cpu - ((cpu >> 6) << 3) << 3 + */ + slli a1, a1, 3 + sub a1, a2, a1 + /* Compute the "get mask": 1 << bit_pos */ + li a2, 1 + sll a1, a2, a1 + + /* Check the value of new_vmalloc for this cpu */ + REG_L a2, 0(a0) + and a2, a2, a1 + beq a2, zero, _new_vmalloc_restore_context + + /* Atomically reset the current cpu bit in new_vmalloc */ + amoxor.d a0, a1, (a0) + + /* Only emit a sfence.vma if the uarch caches invalid entries */ + ALTERNATIVE("sfence.vma", "nop", 0, RISCV_ISA_EXT_SVVPTC, 1) + + REG_L a0, TASK_TI_A0(tp) + REG_L a1, TASK_TI_A1(tp) + REG_L a2, TASK_TI_A2(tp) + csrw CSR_SCRATCH, x0 + sret + +_new_vmalloc_restore_context: + REG_L a2, TASK_TI_A2(tp) +_new_vmalloc_restore_context_a1: + REG_L a1, TASK_TI_A1(tp) +_new_vmalloc_restore_context_a0: + REG_L a0, TASK_TI_A0(tp) +.endm + + SYM_CODE_START(handle_exception) /* * If coming from userspace, preserve the user thread pointer and load @@ -30,6 +103,20 @@ SYM_CODE_START(handle_exception) .Lrestore_kernel_tpsp: csrr tp, CSR_SCRATCH + +#ifdef CONFIG_64BIT + /* + * The RISC-V kernel does not eagerly emit a sfence.vma after each + * new vmalloc mapping, which may result in exceptions: + * - if the uarch caches invalid entries, the new mapping would not be + * observed by the page table walker and an invalidation is needed. + * - if the uarch does not cache invalid entries, a reordered access + * could "miss" the new mapping and traps: in that case, we only need + * to retry the access, no sfence.vma is required. + */ + new_vmalloc_check +#endif + REG_S sp, TASK_TI_KERNEL_SP(tp) #ifdef CONFIG_VMAP_STACK diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c index e3218d65f21d..337dab7d5fd7 100644 --- a/arch/riscv/mm/init.c +++ b/arch/riscv/mm/init.c @@ -36,6 +36,8 @@ #include "../kernel/head.h" +u64 new_vmalloc[NR_CPUS / sizeof(u64) + 1]; + struct kernel_mapping kernel_map __ro_after_init; EXPORT_SYMBOL(kernel_map); #ifdef CONFIG_XIP_KERNEL From 7a21b2e370dab780ddb3aa80f2a4c8ff97bddccc Mon Sep 17 00:00:00 2001 From: Alexandre Ghiti Date: Wed, 17 Jul 2024 08:01:25 +0200 Subject: [PATCH 27/47] riscv: Stop emitting preventive sfence.vma for new userspace mappings with Svvptc The preventive sfence.vma were emitted because new mappings must be made visible to the page table walker but Svvptc guarantees that it will happen within a bounded timeframe, so no need to sfence.vma for the uarchs that implement this extension, we will then take gratuitous (but very unlikely) page faults, similarly to x86 and arm64. This allows to drastically reduce the number of sfence.vma emitted: * Ubuntu boot to login: Before: ~630k sfence.vma After: ~200k sfence.vma * ltp - mmapstress01 Before: ~45k After: ~6.3k * lmbench - lat_pagefault Before: ~665k After: 832 (!) * lmbench - lat_mmap Before: ~546k After: 718 (!) Signed-off-by: Alexandre Ghiti Link: https://lore.kernel.org/r/20240717060125.139416-5-alexghiti@rivosinc.com Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/pgtable.h | 16 +++++++++++++++- arch/riscv/mm/pgtable.c | 13 +++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index aad8b8ca51f1..4ed85329317d 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -476,6 +476,9 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf, struct vm_area_struct *vma, unsigned long address, pte_t *ptep, unsigned int nr) { + asm goto(ALTERNATIVE("nop", "j %l[svvptc]", 0, RISCV_ISA_EXT_SVVPTC, 1) + : : : : svvptc); + /* * The kernel assumes that TLBs don't cache invalid entries, but * in RISC-V, SFENCE.VMA specifies an ordering constraint, not a @@ -485,12 +488,23 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf, */ while (nr--) local_flush_tlb_page(address + nr * PAGE_SIZE); + +svvptc:; + /* + * Svvptc guarantees that the new valid pte will be visible within + * a bounded timeframe, so when the uarch does not cache invalid + * entries, we don't have to do anything. + */ } #define update_mmu_cache(vma, addr, ptep) \ update_mmu_cache_range(NULL, vma, addr, ptep, 1) #define __HAVE_ARCH_UPDATE_MMU_TLB -#define update_mmu_tlb update_mmu_cache +static inline void update_mmu_tlb(struct vm_area_struct *vma, + unsigned long address, pte_t *ptep) +{ + flush_tlb_range(vma, address, address + PAGE_SIZE); +} static inline void update_mmu_cache_pmd(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp) diff --git a/arch/riscv/mm/pgtable.c b/arch/riscv/mm/pgtable.c index 533ec9055fa0..4ae67324f992 100644 --- a/arch/riscv/mm/pgtable.c +++ b/arch/riscv/mm/pgtable.c @@ -9,6 +9,9 @@ int ptep_set_access_flags(struct vm_area_struct *vma, unsigned long address, pte_t *ptep, pte_t entry, int dirty) { + asm goto(ALTERNATIVE("nop", "j %l[svvptc]", 0, RISCV_ISA_EXT_SVVPTC, 1) + : : : : svvptc); + if (!pte_same(ptep_get(ptep), entry)) __set_pte_at(vma->vm_mm, ptep, entry); /* @@ -16,6 +19,16 @@ int ptep_set_access_flags(struct vm_area_struct *vma, * the case that the PTE changed and the spurious fault case. */ return true; + +svvptc: + if (!pte_same(ptep_get(ptep), entry)) { + __set_pte_at(vma->vm_mm, ptep, entry); + /* Here only not svadu is impacted */ + flush_tlb_page(vma, address); + return true; + } + + return false; } int ptep_test_and_clear_young(struct vm_area_struct *vma, From 5c178472af247c7b50f962495bb7462ba453b9fb Mon Sep 17 00:00:00 2001 From: Jisheng Zhang Date: Sat, 6 Jul 2024 01:02:10 +0800 Subject: [PATCH 28/47] riscv: define ILLEGAL_POINTER_VALUE for 64bit This is used in poison.h for poison pointer offset. Based on current SV39, SV48 and SV57 vm layout, 0xdead000000000000 is a proper value that is not mappable, this can avoid potentially turning an oops to an expolit. Signed-off-by: Jisheng Zhang Fixes: fbe934d69eb7 ("RISC-V: Build Infrastructure") Cc: stable@vger.kernel.org Link: https://lore.kernel.org/r/20240705170210.3236-1-jszhang@kernel.org Signed-off-by: Palmer Dabbelt --- arch/riscv/Kconfig | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 4871220061c4..31d941792369 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -320,6 +320,11 @@ config GENERIC_HWEIGHT config FIX_EARLYCON_MEM def_bool MMU +config ILLEGAL_POINTER_VALUE + hex + default 0 if 32BIT + default 0xdead000000000000 if 64BIT + config PGTABLE_LEVELS int default 5 if 64BIT From 1e206fad765b293aa169ec08917761021f52399a Mon Sep 17 00:00:00 2001 From: Xiao Wang Date: Mon, 8 Jul 2024 20:12:24 +0800 Subject: [PATCH 29/47] drivers/perf: riscv: Remove redundant macro check The macro CONFIG_RISCV_PMU must have been defined when riscv_pmu.c gets compiled, so this patch removes the redundant check. Signed-off-by: Xiao Wang Reviewed-by: Atish Patra Link: https://lore.kernel.org/r/20240708121224.1148154-1-xiao.w.wang@intel.com Signed-off-by: Palmer Dabbelt --- drivers/perf/riscv_pmu.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/perf/riscv_pmu.c b/drivers/perf/riscv_pmu.c index 0a02e85a8951..7644147d50b4 100644 --- a/drivers/perf/riscv_pmu.c +++ b/drivers/perf/riscv_pmu.c @@ -39,7 +39,6 @@ void arch_perf_update_userpage(struct perf_event *event, userpg->cap_user_time_short = 0; userpg->cap_user_rdpmc = riscv_perf_user_access(event); -#ifdef CONFIG_RISCV_PMU /* * The counters are 64-bit but the priv spec doesn't mandate all the * bits to be implemented: that's why, counter width can vary based on @@ -47,7 +46,6 @@ void arch_perf_update_userpage(struct perf_event *event, */ if (userpg->cap_user_rdpmc) userpg->pmc_width = to_riscv_pmu(event->pmu)->ctr_get_width(event->hw.idx) + 1; -#endif do { rd = sched_clock_read_begin(&seq); From cea9d27705d62984faf6137963c10bf26b967996 Mon Sep 17 00:00:00 2001 From: Jinjie Ruan Date: Thu, 11 Jul 2024 19:15:08 +0800 Subject: [PATCH 30/47] riscv: Remove unused _TIF_WORK_MASK Since commit f0bddf50586d ("riscv: entry: Convert to generic entry"), _TIF_WORK_MASK is no longer used, so remove it. Fixes: f0bddf50586d ("riscv: entry: Convert to generic entry") Signed-off-by: Jinjie Ruan Reviewed-by: Guo Ren Reviewed-by: Andy Chiu Link: https://lore.kernel.org/r/20240711111508.1373322-1-ruanjinjie@huawei.com Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/thread_info.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/arch/riscv/include/asm/thread_info.h b/arch/riscv/include/asm/thread_info.h index fca5c6be2b81..5b09dc874151 100644 --- a/arch/riscv/include/asm/thread_info.h +++ b/arch/riscv/include/asm/thread_info.h @@ -112,8 +112,4 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src); #define _TIF_UPROBE (1 << TIF_UPROBE) #define _TIF_RISCV_V_DEFER_RESTORE (1 << TIF_RISCV_V_DEFER_RESTORE) -#define _TIF_WORK_MASK \ - (_TIF_NOTIFY_RESUME | _TIF_SIGPENDING | _TIF_NEED_RESCHED | \ - _TIF_NOTIFY_SIGNAL | _TIF_UPROBE) - #endif /* _ASM_RISCV_THREAD_INFO_H */ From 1845d381f28063a3b68e9e148d5a7f01d6be8721 Mon Sep 17 00:00:00 2001 From: Steffen Persvold Date: Sun, 7 Jul 2024 02:35:15 +0200 Subject: [PATCH 31/47] riscv: cacheinfo: Add back init_cache_level() function commit 5944ce092b97 (arch_topology: Build cacheinfo from primary CPU) removed the init_cache_level() function from arch/riscv/kernel/cacheinfo.c and relies on the init_cpu_topology() function in drivers/base/arch_topology.c to call fetch_cache_info() which in turn calls init_of_cache_level() to populate the cache hierarchy information. However, init_cpu_topology() is only called from smpboot.c:smp_prepare_cpus() and thus only available when CONFIG_SMP is defined. To support non-SMP enabled kernels to still detect cache hierarchy, we add back the init_cache_level() function. The init_level_allocate_ci() function handles this gracefully on SMP-enabled kernels anyway where fetch_cache_info() is called from init_cpu_topology() earlier in the boot phase. Signed-off-by: Steffen Persvold Link: https://lore.kernel.org/r/20240707003515.5058-1-spersvold@gmail.com Signed-off-by: Palmer Dabbelt --- arch/riscv/kernel/cacheinfo.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/arch/riscv/kernel/cacheinfo.c b/arch/riscv/kernel/cacheinfo.c index d6c108c50cba..b320b1d9aa01 100644 --- a/arch/riscv/kernel/cacheinfo.c +++ b/arch/riscv/kernel/cacheinfo.c @@ -71,6 +71,11 @@ static void ci_leaf_init(struct cacheinfo *this_leaf, this_leaf->type = type; } +int init_cache_level(unsigned int cpu) +{ + return init_of_cache_level(cpu); +} + int populate_cache_leaves(unsigned int cpu) { struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); From 8f1534e7440382d118c3d655d3a6014128b2086d Mon Sep 17 00:00:00 2001 From: Jisheng Zhang Date: Sun, 21 Jul 2024 01:06:59 +0800 Subject: [PATCH 32/47] riscv: avoid Imbalance in RAS Inspired by[1], modify the code to remove the code of modifying ra to avoid imbalance RAS (return address stack) which may lead to incorret predictions on return. Link: https://lore.kernel.org/linux-riscv/20240607061335.2197383-1-cyrilbur@tenstorrent.com/ [1] Signed-off-by: Jisheng Zhang Reviewed-by: Cyril Bur Link: https://lore.kernel.org/r/20240720170659.1522-1-jszhang@kernel.org Signed-off-by: Palmer Dabbelt --- arch/riscv/kernel/entry.S | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/riscv/kernel/entry.S b/arch/riscv/kernel/entry.S index fc41472d512e..c200d329d4bd 100644 --- a/arch/riscv/kernel/entry.S +++ b/arch/riscv/kernel/entry.S @@ -326,8 +326,8 @@ SYM_CODE_START(ret_from_fork) jalr s0 1: move a0, sp /* pt_regs */ - la ra, ret_from_exception - tail syscall_exit_to_user_mode + call syscall_exit_to_user_mode + j ret_from_exception SYM_CODE_END(ret_from_fork) #ifdef CONFIG_IRQ_STACKS From 983f12149942a58c66e9db7638a10667aae0d958 Mon Sep 17 00:00:00 2001 From: Jinjie Ruan Date: Sat, 27 Jul 2024 14:34:38 +0800 Subject: [PATCH 33/47] RISC-V: Implement kgdb_roundup_cpus() to enable future NMI Roundup Until now, the generic weak kgdb_roundup_cpus() has been used for kgdb on RISCV. A custom one allows to debug CPUs that are stuck with interrupts disabled with NMI support in the future. And using an IPI is better than the generic one since it avoids the potential situation described in the generic kgdb_call_nmi_hook(). As Andrew pointed out, once there is NMI support, we can easily extend this and the CPU backtrace support to use NMIs. After this patch, the kgdb test show that: # echo g > /proc/sysrq-trigger [2]kdb> btc btc: cpu status: Currently on cpu 2 Available cpus: 0-1(-), 2, 3(-) Stack traceback for pid 0 0xffffffff81c13a40 0 0 1 0 - 0xffffffff81c14510 swapper/0 CPU: 0 PID: 0 Comm: swapper/0 Not tainted 6.10.0-g3120273055b6-dirty #51 Hardware name: riscv-virtio,qemu (DT) Call Trace: [] dump_backtrace+0x28/0x30 [] show_stack+0x38/0x44 [] dump_stack_lvl+0x58/0x7a [] dump_stack+0x18/0x20 [] kgdb_cpu_enter+0x682/0x6b2 [] kgdb_nmicallback+0xa0/0xac [] handle_IPI+0x9c/0x120 [] handle_percpu_devid_irq+0xa4/0x1e4 [] generic_handle_domain_irq+0x28/0x36 [] ipi_mux_process+0xe8/0x110 [] imsic_handle_irq+0xf8/0x13a [] generic_handle_domain_irq+0x28/0x36 [] riscv_intc_aia_irq+0x2e/0x40 [] handle_riscv_irq+0x54/0x86 [] call_on_irq_stack+0x32/0x40 Rebased on Ryo Takakura's "RISC-V: Enable IPI CPU Backtrace" patch. Signed-off-by: Jinjie Ruan Reviewed-by: Andrew Jones Link: https://lore.kernel.org/r/20240727063438.886155-1-ruanjinjie@huawei.com Signed-off-by: Palmer Dabbelt --- arch/riscv/kernel/smp.c | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/arch/riscv/kernel/smp.c b/arch/riscv/kernel/smp.c index 9b047899791c..c180a647a30e 100644 --- a/arch/riscv/kernel/smp.c +++ b/arch/riscv/kernel/smp.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -35,6 +36,7 @@ enum ipi_message_type { IPI_IRQ_WORK, IPI_TIMER, IPI_CPU_BACKTRACE, + IPI_KGDB_ROUNDUP, IPI_MAX }; @@ -115,6 +117,7 @@ void arch_irq_work_raise(void) static irqreturn_t handle_IPI(int irq, void *data) { + unsigned int cpu = smp_processor_id(); int ipi = irq - ipi_virq_base; switch (ipi) { @@ -128,7 +131,7 @@ static irqreturn_t handle_IPI(int irq, void *data) ipi_stop(); break; case IPI_CPU_CRASH_STOP: - ipi_cpu_crash_stop(smp_processor_id(), get_irq_regs()); + ipi_cpu_crash_stop(cpu, get_irq_regs()); break; case IPI_IRQ_WORK: irq_work_run(); @@ -141,8 +144,11 @@ static irqreturn_t handle_IPI(int irq, void *data) case IPI_CPU_BACKTRACE: nmi_cpu_backtrace(get_irq_regs()); break; + case IPI_KGDB_ROUNDUP: + kgdb_nmicallback(cpu, get_irq_regs()); + break; default: - pr_warn("CPU%d: unhandled IPI%d\n", smp_processor_id(), ipi); + pr_warn("CPU%d: unhandled IPI%d\n", cpu, ipi); break; } @@ -209,6 +215,7 @@ static const char * const ipi_names[] = { [IPI_IRQ_WORK] = "IRQ work interrupts", [IPI_TIMER] = "Timer broadcast interrupts", [IPI_CPU_BACKTRACE] = "CPU backtrace interrupts", + [IPI_KGDB_ROUNDUP] = "KGDB roundup interrupts", }; void show_ipi_stats(struct seq_file *p, int prec) @@ -339,3 +346,19 @@ void arch_trigger_cpumask_backtrace(const cpumask_t *mask, int exclude_cpu) { nmi_trigger_cpumask_backtrace(mask, exclude_cpu, riscv_backtrace_ipi); } + +#ifdef CONFIG_KGDB +void kgdb_roundup_cpus(void) +{ + int this_cpu = raw_smp_processor_id(); + int cpu; + + for_each_online_cpu(cpu) { + /* No need to roundup ourselves */ + if (cpu == this_cpu) + continue; + + send_ipi_single(cpu, IPI_KGDB_ROUNDUP); + } +} +#endif From e36ddf3226864e095c5f18a7d46feb1e75fe91b2 Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Tue, 30 Jul 2024 17:37:26 +0200 Subject: [PATCH 34/47] riscv: defconfig: Disable RZ/Five peripheral support There is not much point in keeping support for RZ/Five peripherals enabled, as the RZ/Five platform option (ARCH_R9A07G043) is gated behind NONPORTABLE. Hence drop all config options that enable built-in or modular support for peripherals found on RZ/Five SoCs. Disable USB_XHCI_RCAR explicitly, as its value defaults to the value of ARCH_RENESAS, which is still enabled. Signed-off-by: Geert Uytterhoeven Reviewed-by: Lad Prabhakar Link: https://lore.kernel.org/r/89ad70c7d6e8078208fecfd41dc03f6028531729.1722353710.git.geert+renesas@glider.be Signed-off-by: Palmer Dabbelt --- arch/riscv/configs/defconfig | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/arch/riscv/configs/defconfig b/arch/riscv/configs/defconfig index 0d678325444f..b51ef6cd1e39 100644 --- a/arch/riscv/configs/defconfig +++ b/arch/riscv/configs/defconfig @@ -137,12 +137,10 @@ CONFIG_VIRTIO_NET=y CONFIG_MACB=y CONFIG_E1000E=y CONFIG_R8169=y -CONFIG_RAVB=y CONFIG_STMMAC_ETH=m CONFIG_MICREL_PHY=y CONFIG_MICROSEMI_PHY=y CONFIG_MOTORCOMM_PHY=y -CONFIG_CAN_RCAR_CANFD=m CONFIG_INPUT_MOUSEDEV=y CONFIG_KEYBOARD_SUN4I_LRADC=m CONFIG_SERIAL_8250=y @@ -150,7 +148,6 @@ CONFIG_SERIAL_8250_CONSOLE=y CONFIG_SERIAL_8250_DW=y CONFIG_SERIAL_OF_PLATFORM=y CONFIG_SERIAL_EARLYCON_RISCV_SBI=y -CONFIG_SERIAL_SH_SCI=y CONFIG_VIRTIO_CONSOLE=y CONFIG_HW_RANDOM=y CONFIG_HW_RANDOM_VIRTIO=y @@ -159,11 +156,9 @@ CONFIG_I2C=y CONFIG_I2C_CHARDEV=m CONFIG_I2C_DESIGNWARE_PLATFORM=y CONFIG_I2C_MV64XXX=m -CONFIG_I2C_RIIC=y CONFIG_SPI=y CONFIG_SPI_CADENCE_QUADSPI=m CONFIG_SPI_PL022=m -CONFIG_SPI_RSPI=m CONFIG_SPI_SIFIVE=y CONFIG_SPI_SUN6I=y # CONFIG_PTP_1588_CLOCK is not set @@ -172,7 +167,6 @@ CONFIG_POWER_RESET_GPIO_RESTART=y CONFIG_SENSORS_SFCTEMP=m CONFIG_CPU_THERMAL=y CONFIG_DEVFREQ_THERMAL=y -CONFIG_RZG2L_THERMAL=y CONFIG_WATCHDOG=y CONFIG_SUNXI_WATCHDOG=y CONFIG_MFD_AXP20X_I2C=y @@ -201,11 +195,11 @@ CONFIG_USB=y CONFIG_USB_OTG=y CONFIG_USB_XHCI_HCD=y CONFIG_USB_XHCI_PLATFORM=y +# CONFIG_USB_XHCI_RCAR is not set CONFIG_USB_EHCI_HCD=y CONFIG_USB_EHCI_HCD_PLATFORM=y CONFIG_USB_OHCI_HCD=y CONFIG_USB_OHCI_HCD_PLATFORM=y -CONFIG_USB_RENESAS_USBHS=m CONFIG_USB_STORAGE=y CONFIG_USB_UAS=y CONFIG_USB_CDNS_SUPPORT=m @@ -217,7 +211,6 @@ CONFIG_USB_MUSB_HDRC=m CONFIG_USB_MUSB_SUNXI=m CONFIG_NOP_USB_XCEIV=m CONFIG_USB_GADGET=y -CONFIG_USB_RENESAS_USBHS_UDC=m CONFIG_USB_CONFIGFS=m CONFIG_USB_CONFIGFS_SERIAL=y CONFIG_USB_CONFIGFS_ACM=y @@ -235,7 +228,6 @@ CONFIG_MMC_SDHCI_PLTFM=y CONFIG_MMC_SDHCI_OF_DWCMSHC=y CONFIG_MMC_SDHCI_CADENCE=y CONFIG_MMC_SPI=y -CONFIG_MMC_SDHI=y CONFIG_MMC_DW=y CONFIG_MMC_DW_STARFIVE=y CONFIG_MMC_SUNXI=y @@ -250,7 +242,6 @@ CONFIG_VIRTIO_INPUT=y CONFIG_VIRTIO_MMIO=y CONFIG_CLK_SOPHGO_CV1800=y CONFIG_SUN8I_DE2_CCU=m -CONFIG_RENESAS_OSTM=y CONFIG_SUN50I_IOMMU=y CONFIG_RPMSG_CHAR=y CONFIG_RPMSG_CTRL=y @@ -258,7 +249,6 @@ CONFIG_RPMSG_VIRTIO=y CONFIG_PM_DEVFREQ=y CONFIG_IIO=y CONFIG_PHY_SUN4I_USB=m -CONFIG_PHY_RCAR_GEN3_USB2=y CONFIG_PHY_STARFIVE_JH7110_DPHY_RX=m CONFIG_PHY_STARFIVE_JH7110_PCIE=m CONFIG_PHY_STARFIVE_JH7110_USB=m From 3cc754c237e9552ee0f2f6c2f37b3454bee39e67 Mon Sep 17 00:00:00 2001 From: Jinjie Ruan Date: Wed, 4 Sep 2024 09:33:44 +0800 Subject: [PATCH 35/47] riscv: Use LIST_HEAD() to simplify code list_head can be initialized automatically with LIST_HEAD() instead of calling INIT_LIST_HEAD(). Signed-off-by: Jinjie Ruan Tested-by: Charlie Jenkins Reviewed-by: Charlie Jenkins Link: https://lore.kernel.org/r/20240904013344.2026738-1-ruanjinjie@huawei.com Signed-off-by: Palmer Dabbelt --- arch/riscv/kernel/module.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/arch/riscv/kernel/module.c b/arch/riscv/kernel/module.c index 906f9a3a5d65..1cd461f3d872 100644 --- a/arch/riscv/kernel/module.c +++ b/arch/riscv/kernel/module.c @@ -787,8 +787,8 @@ int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab, int res; unsigned int num_relocations = sechdrs[relsec].sh_size / sizeof(*rel); struct hlist_head *relocation_hashtable; - struct list_head used_buckets_list; unsigned int hashtable_bits; + LIST_HEAD(used_buckets_list); hashtable_bits = initialize_relocation_hashtable(num_relocations, &relocation_hashtable); @@ -796,8 +796,6 @@ int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab, if (!relocation_hashtable) return -ENOMEM; - INIT_LIST_HEAD(&used_buckets_list); - pr_debug("Applying relocate section %u to %u\n", relsec, sechdrs[relsec].sh_info); From 594ffcf4efe5094876f5b549a36262416104cd3d Mon Sep 17 00:00:00 2001 From: Charlie Jenkins Date: Wed, 7 Aug 2024 17:27:42 -0700 Subject: [PATCH 36/47] riscv: Make riscv_isa_vendor_ext_andes array static Since this array is only used in this file, it should be static. Signed-off-by: Charlie Jenkins Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202407241530.ej5SVgX1-lkp@intel.com/ Reviewed-by: Alexandre Ghiti Link: https://lore.kernel.org/r/20240807-make_andes_static-v1-1-b64bf4c3d941@rivosinc.com Signed-off-by: Palmer Dabbelt --- arch/riscv/kernel/vendor_extensions/andes.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/riscv/kernel/vendor_extensions/andes.c b/arch/riscv/kernel/vendor_extensions/andes.c index ec688c88456a..51f302b6d503 100644 --- a/arch/riscv/kernel/vendor_extensions/andes.c +++ b/arch/riscv/kernel/vendor_extensions/andes.c @@ -8,7 +8,7 @@ #include /* All Andes vendor extensions supported in Linux */ -const struct riscv_isa_ext_data riscv_isa_vendor_ext_andes[] = { +static const struct riscv_isa_ext_data riscv_isa_vendor_ext_andes[] = { __RISCV_ISA_EXT_DATA(xandespmu, RISCV_ISA_VENDOR_EXT_XANDESPMU), }; From 11c2dbd7f2415731ac8213d3de0dbd5f5dd0ddc7 Mon Sep 17 00:00:00 2001 From: Charlie Jenkins Date: Thu, 8 Aug 2024 11:41:40 -0700 Subject: [PATCH 37/47] selftests: riscv: Allow mmap test to compile on 32-bit Macros needed for 32-bit compilations were hidden behind 64-bit riscv ifdefs. Fix the 32-bit compilations by moving macros to allow the memory_layout test to run on 32-bit. Signed-off-by: Charlie Jenkins Fixes: 73d05262a2ca ("selftests: riscv: Generalize mm selftests") Link: https://lore.kernel.org/r/20240808-mmap_tests__fixes-v1-1-b1344b642a84@rivosinc.com Signed-off-by: Palmer Dabbelt --- tools/testing/selftests/riscv/mm/mmap_test.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/riscv/mm/mmap_test.h b/tools/testing/selftests/riscv/mm/mmap_test.h index 3b29ca3bb3d4..1c3313c201d5 100644 --- a/tools/testing/selftests/riscv/mm/mmap_test.h +++ b/tools/testing/selftests/riscv/mm/mmap_test.h @@ -48,11 +48,11 @@ uint32_t random_addresses[] = { }; #endif -// Only works on 64 bit -#if __riscv_xlen == 64 #define PROT (PROT_READ | PROT_WRITE) #define FLAGS (MAP_PRIVATE | MAP_ANONYMOUS) +// Only works on 64 bit +#if __riscv_xlen == 64 /* mmap must return a value that doesn't use more bits than the hint address. */ static inline unsigned long get_max_value(unsigned long input) { @@ -80,6 +80,8 @@ static inline unsigned long get_max_value(unsigned long input) }) #endif /* __riscv_xlen == 64 */ +#define TEST_MMAPS do { } while (0) + static inline int memory_layout(void) { void *value1 = mmap(NULL, sizeof(int), PROT, FLAGS, 0, 0); From 048e2906d4caf57018e92f49c9a0f998ebb83f9b Mon Sep 17 00:00:00 2001 From: Yunhui Cui Date: Tue, 25 Jun 2024 11:05:02 +0800 Subject: [PATCH 38/47] riscv: Randomize lower bits of stack address Implement arch_align_stack() to randomize the lower bits of the stack address. Signed-off-by: Yunhui Cui Link: https://lore.kernel.org/r/20240625030502.68988-1-cuiyunhui@bytedance.com Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/exec.h | 8 ++++++++ arch/riscv/kernel/process.c | 9 +++++++++ 2 files changed, 17 insertions(+) create mode 100644 arch/riscv/include/asm/exec.h diff --git a/arch/riscv/include/asm/exec.h b/arch/riscv/include/asm/exec.h new file mode 100644 index 000000000000..07d9942682e0 --- /dev/null +++ b/arch/riscv/include/asm/exec.h @@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef __ASM_EXEC_H +#define __ASM_EXEC_H + +extern unsigned long arch_align_stack(unsigned long sp); + +#endif /* __ASM_EXEC_H */ diff --git a/arch/riscv/kernel/process.c b/arch/riscv/kernel/process.c index e4bc61c4e58a..e3142d8a6e28 100644 --- a/arch/riscv/kernel/process.c +++ b/arch/riscv/kernel/process.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -26,6 +27,7 @@ #include #include #include +#include #if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_STACKPROTECTOR_PER_TASK) #include @@ -99,6 +101,13 @@ void show_regs(struct pt_regs *regs) dump_backtrace(regs, NULL, KERN_DEFAULT); } +unsigned long arch_align_stack(unsigned long sp) +{ + if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space) + sp -= get_random_u32_below(PAGE_SIZE); + return sp & ~0xf; +} + #ifdef CONFIG_COMPAT static bool compat_mode_supported __read_mostly; From 21d98d658f9e5967dc30c321bc258b50740c6665 Mon Sep 17 00:00:00 2001 From: Hanjun Guo Date: Sun, 11 Aug 2024 11:18:04 +0800 Subject: [PATCH 39/47] ACPI: RISCV: Make acpi_numa_get_nid() to be static acpi_numa_get_nid() is only called in acpi_numa.c for riscv, no need to add it in head file, so make it static and remove related functions in the asm/acpi.h. Spotted by doing some cleanup for arm64 ACPI. Signed-off-by: Hanjun Guo Reviewed-by: Haibo Xu Link: https://lore.kernel.org/r/20240811031804.3347298-1-guohanjun@huawei.com Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/acpi.h | 2 -- arch/riscv/kernel/acpi_numa.c | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/arch/riscv/include/asm/acpi.h b/arch/riscv/include/asm/acpi.h index e0a1f84404f3..6e13695120bc 100644 --- a/arch/riscv/include/asm/acpi.h +++ b/arch/riscv/include/asm/acpi.h @@ -91,10 +91,8 @@ static inline void acpi_get_cbo_block_size(struct acpi_table_header *table, #endif /* CONFIG_ACPI */ #ifdef CONFIG_ACPI_NUMA -int acpi_numa_get_nid(unsigned int cpu); void acpi_map_cpus_to_nodes(void); #else -static inline int acpi_numa_get_nid(unsigned int cpu) { return NUMA_NO_NODE; } static inline void acpi_map_cpus_to_nodes(void) { } #endif /* CONFIG_ACPI_NUMA */ diff --git a/arch/riscv/kernel/acpi_numa.c b/arch/riscv/kernel/acpi_numa.c index 0231482d6946..a93c773b60e6 100644 --- a/arch/riscv/kernel/acpi_numa.c +++ b/arch/riscv/kernel/acpi_numa.c @@ -30,7 +30,7 @@ static int acpi_early_node_map[NR_CPUS] __initdata = { NUMA_NO_NODE }; -int __init acpi_numa_get_nid(unsigned int cpu) +static int __init acpi_numa_get_nid(unsigned int cpu) { return acpi_early_node_map[cpu]; } From 58ff537109ac863d4ec83baf8413b17dcc10101c Mon Sep 17 00:00:00 2001 From: Samuel Holland Date: Wed, 31 Jul 2024 20:36:59 -0700 Subject: [PATCH 40/47] riscv: Omit optimized string routines when using KASAN The optimized string routines are implemented in assembly, so they are not instrumented for use with KASAN. Fall back to the C version of the routines in order to improve KASAN coverage. This fixes the kasan_strings() unit test. Signed-off-by: Samuel Holland Reviewed-by: Alexandre Ghiti Tested-by: Alexandre Ghiti Link: https://lore.kernel.org/r/20240801033725.28816-2-samuel.holland@sifive.com Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/string.h | 2 ++ arch/riscv/kernel/riscv_ksyms.c | 3 --- arch/riscv/lib/Makefile | 2 ++ arch/riscv/lib/strcmp.S | 1 + arch/riscv/lib/strlen.S | 1 + arch/riscv/lib/strncmp.S | 1 + arch/riscv/purgatory/Makefile | 2 ++ 7 files changed, 9 insertions(+), 3 deletions(-) diff --git a/arch/riscv/include/asm/string.h b/arch/riscv/include/asm/string.h index a96b1fea24fe..5ba77f60bf0b 100644 --- a/arch/riscv/include/asm/string.h +++ b/arch/riscv/include/asm/string.h @@ -19,6 +19,7 @@ extern asmlinkage void *__memcpy(void *, const void *, size_t); extern asmlinkage void *memmove(void *, const void *, size_t); extern asmlinkage void *__memmove(void *, const void *, size_t); +#if !(defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)) #define __HAVE_ARCH_STRCMP extern asmlinkage int strcmp(const char *cs, const char *ct); @@ -27,6 +28,7 @@ extern asmlinkage __kernel_size_t strlen(const char *); #define __HAVE_ARCH_STRNCMP extern asmlinkage int strncmp(const char *cs, const char *ct, size_t count); +#endif /* For those files which don't want to check by kasan. */ #if defined(CONFIG_KASAN) && !defined(__SANITIZE_ADDRESS__) diff --git a/arch/riscv/kernel/riscv_ksyms.c b/arch/riscv/kernel/riscv_ksyms.c index a72879b4249a..5ab1c7e1a6ed 100644 --- a/arch/riscv/kernel/riscv_ksyms.c +++ b/arch/riscv/kernel/riscv_ksyms.c @@ -12,9 +12,6 @@ EXPORT_SYMBOL(memset); EXPORT_SYMBOL(memcpy); EXPORT_SYMBOL(memmove); -EXPORT_SYMBOL(strcmp); -EXPORT_SYMBOL(strlen); -EXPORT_SYMBOL(strncmp); EXPORT_SYMBOL(__memset); EXPORT_SYMBOL(__memcpy); EXPORT_SYMBOL(__memmove); diff --git a/arch/riscv/lib/Makefile b/arch/riscv/lib/Makefile index 2b369f51b0a5..8eec6b69a875 100644 --- a/arch/riscv/lib/Makefile +++ b/arch/riscv/lib/Makefile @@ -3,9 +3,11 @@ lib-y += delay.o lib-y += memcpy.o lib-y += memset.o lib-y += memmove.o +ifeq ($(CONFIG_KASAN_GENERIC)$(CONFIG_KASAN_SW_TAGS),) lib-y += strcmp.o lib-y += strlen.o lib-y += strncmp.o +endif lib-y += csum.o ifeq ($(CONFIG_MMU), y) lib-$(CONFIG_RISCV_ISA_V) += uaccess_vector.o diff --git a/arch/riscv/lib/strcmp.S b/arch/riscv/lib/strcmp.S index 687b2bea5c43..542301a67a2f 100644 --- a/arch/riscv/lib/strcmp.S +++ b/arch/riscv/lib/strcmp.S @@ -120,3 +120,4 @@ strcmp_zbb: .option pop #endif SYM_FUNC_END(strcmp) +EXPORT_SYMBOL(strcmp) diff --git a/arch/riscv/lib/strlen.S b/arch/riscv/lib/strlen.S index 8ae3064e45ff..962983b73251 100644 --- a/arch/riscv/lib/strlen.S +++ b/arch/riscv/lib/strlen.S @@ -131,3 +131,4 @@ strlen_zbb: #endif SYM_FUNC_END(strlen) SYM_FUNC_ALIAS(__pi_strlen, strlen) +EXPORT_SYMBOL(strlen) diff --git a/arch/riscv/lib/strncmp.S b/arch/riscv/lib/strncmp.S index aba5b3148621..0f359ea2f55b 100644 --- a/arch/riscv/lib/strncmp.S +++ b/arch/riscv/lib/strncmp.S @@ -136,3 +136,4 @@ strncmp_zbb: .option pop #endif SYM_FUNC_END(strncmp) +EXPORT_SYMBOL(strncmp) diff --git a/arch/riscv/purgatory/Makefile b/arch/riscv/purgatory/Makefile index f11945ee2490..fb9c917c9b45 100644 --- a/arch/riscv/purgatory/Makefile +++ b/arch/riscv/purgatory/Makefile @@ -1,7 +1,9 @@ # SPDX-License-Identifier: GPL-2.0 purgatory-y := purgatory.o sha256.o entry.o string.o ctype.o memcpy.o memset.o +ifeq ($(CONFIG_KASAN_GENERIC)$(CONFIG_KASAN_SW_TAGS),) purgatory-y += strcmp.o strlen.o strncmp.o +endif targets += $(purgatory-y) PURGATORY_OBJS = $(addprefix $(obj)/,$(purgatory-y)) From 77514915b72c51ebc1c30a67a54d4a90ca2a4a39 Mon Sep 17 00:00:00 2001 From: Samuel Holland Date: Wed, 31 Jul 2024 20:37:00 -0700 Subject: [PATCH 41/47] riscv: Enable bitops instrumentation Instead of implementing the bitops functions directly in assembly, provide the arch_-prefixed versions and use the wrappers from asm-generic to add instrumentation. This improves KASAN coverage and fixes the kasan_bitops_generic() unit test. Signed-off-by: Samuel Holland Reviewed-by: Alexandre Ghiti Tested-by: Alexandre Ghiti Link: https://lore.kernel.org/r/20240801033725.28816-3-samuel.holland@sifive.com Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/bitops.h | 43 ++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/arch/riscv/include/asm/bitops.h b/arch/riscv/include/asm/bitops.h index 71af9ecfcfcb..fae152ea0508 100644 --- a/arch/riscv/include/asm/bitops.h +++ b/arch/riscv/include/asm/bitops.h @@ -222,44 +222,44 @@ legacy: #define __NOT(x) (~(x)) /** - * test_and_set_bit - Set a bit and return its old value + * arch_test_and_set_bit - Set a bit and return its old value * @nr: Bit to set * @addr: Address to count from * * This operation may be reordered on other architectures than x86. */ -static inline int test_and_set_bit(int nr, volatile unsigned long *addr) +static inline int arch_test_and_set_bit(int nr, volatile unsigned long *addr) { return __test_and_op_bit(or, __NOP, nr, addr); } /** - * test_and_clear_bit - Clear a bit and return its old value + * arch_test_and_clear_bit - Clear a bit and return its old value * @nr: Bit to clear * @addr: Address to count from * * This operation can be reordered on other architectures other than x86. */ -static inline int test_and_clear_bit(int nr, volatile unsigned long *addr) +static inline int arch_test_and_clear_bit(int nr, volatile unsigned long *addr) { return __test_and_op_bit(and, __NOT, nr, addr); } /** - * test_and_change_bit - Change a bit and return its old value + * arch_test_and_change_bit - Change a bit and return its old value * @nr: Bit to change * @addr: Address to count from * * This operation is atomic and cannot be reordered. * It also implies a memory barrier. */ -static inline int test_and_change_bit(int nr, volatile unsigned long *addr) +static inline int arch_test_and_change_bit(int nr, volatile unsigned long *addr) { return __test_and_op_bit(xor, __NOP, nr, addr); } /** - * set_bit - Atomically set a bit in memory + * arch_set_bit - Atomically set a bit in memory * @nr: the bit to set * @addr: the address to start counting from * @@ -270,13 +270,13 @@ static inline int test_and_change_bit(int nr, volatile unsigned long *addr) * Note that @nr may be almost arbitrarily large; this function is not * restricted to acting on a single-word quantity. */ -static inline void set_bit(int nr, volatile unsigned long *addr) +static inline void arch_set_bit(int nr, volatile unsigned long *addr) { __op_bit(or, __NOP, nr, addr); } /** - * clear_bit - Clears a bit in memory + * arch_clear_bit - Clears a bit in memory * @nr: Bit to clear * @addr: Address to start counting from * @@ -284,13 +284,13 @@ static inline void set_bit(int nr, volatile unsigned long *addr) * on non x86 architectures, so if you are writing portable code, * make sure not to rely on its reordering guarantees. */ -static inline void clear_bit(int nr, volatile unsigned long *addr) +static inline void arch_clear_bit(int nr, volatile unsigned long *addr) { __op_bit(and, __NOT, nr, addr); } /** - * change_bit - Toggle a bit in memory + * arch_change_bit - Toggle a bit in memory * @nr: Bit to change * @addr: Address to start counting from * @@ -298,40 +298,40 @@ static inline void clear_bit(int nr, volatile unsigned long *addr) * Note that @nr may be almost arbitrarily large; this function is not * restricted to acting on a single-word quantity. */ -static inline void change_bit(int nr, volatile unsigned long *addr) +static inline void arch_change_bit(int nr, volatile unsigned long *addr) { __op_bit(xor, __NOP, nr, addr); } /** - * test_and_set_bit_lock - Set a bit and return its old value, for lock + * arch_test_and_set_bit_lock - Set a bit and return its old value, for lock * @nr: Bit to set * @addr: Address to count from * * This operation is atomic and provides acquire barrier semantics. * It can be used to implement bit locks. */ -static inline int test_and_set_bit_lock( +static inline int arch_test_and_set_bit_lock( unsigned long nr, volatile unsigned long *addr) { return __test_and_op_bit_ord(or, __NOP, nr, addr, .aq); } /** - * clear_bit_unlock - Clear a bit in memory, for unlock + * arch_clear_bit_unlock - Clear a bit in memory, for unlock * @nr: the bit to set * @addr: the address to start counting from * * This operation is atomic and provides release barrier semantics. */ -static inline void clear_bit_unlock( +static inline void arch_clear_bit_unlock( unsigned long nr, volatile unsigned long *addr) { __op_bit_ord(and, __NOT, nr, addr, .rl); } /** - * __clear_bit_unlock - Clear a bit in memory, for unlock + * arch___clear_bit_unlock - Clear a bit in memory, for unlock * @nr: the bit to set * @addr: the address to start counting from * @@ -345,13 +345,13 @@ static inline void clear_bit_unlock( * non-atomic property here: it's a lot more instructions and we still have to * provide release semantics anyway. */ -static inline void __clear_bit_unlock( +static inline void arch___clear_bit_unlock( unsigned long nr, volatile unsigned long *addr) { - clear_bit_unlock(nr, addr); + arch_clear_bit_unlock(nr, addr); } -static inline bool xor_unlock_is_negative_byte(unsigned long mask, +static inline bool arch_xor_unlock_is_negative_byte(unsigned long mask, volatile unsigned long *addr) { unsigned long res; @@ -369,6 +369,9 @@ static inline bool xor_unlock_is_negative_byte(unsigned long mask, #undef __NOT #undef __AMO +#include +#include + #include #include #include From 732b177663e1757950d73e7763fa0de26272439f Mon Sep 17 00:00:00 2001 From: Haibo Xu Date: Mon, 29 Jul 2024 11:59:55 +0800 Subject: [PATCH 42/47] ACPI: NUMA: initialize all values of acpi_early_node_map to NUMA_NO_NODE Currently, only acpi_early_node_map[0] was initialized to NUMA_NO_NODE. To ensure all the values were properly initialized, switch to initialize all of them to NUMA_NO_NODE. Suggested-by: Andrew Jones Signed-off-by: Haibo Xu Reviewed-by: Anshuman Khandual (arm64 platform) Reviewed-by: Sunil V L Reviewed-by: Andrew Jones Link: https://lore.kernel.org/r/20240729035958.1957185-1-haibo1.xu@intel.com Signed-off-by: Palmer Dabbelt --- arch/riscv/kernel/acpi_numa.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/riscv/kernel/acpi_numa.c b/arch/riscv/kernel/acpi_numa.c index a93c773b60e6..130769e3a99c 100644 --- a/arch/riscv/kernel/acpi_numa.c +++ b/arch/riscv/kernel/acpi_numa.c @@ -28,7 +28,7 @@ #include -static int acpi_early_node_map[NR_CPUS] __initdata = { NUMA_NO_NODE }; +static int acpi_early_node_map[NR_CPUS] __initdata = { [0 ... NR_CPUS - 1] = NUMA_NO_NODE }; static int __init acpi_numa_get_nid(unsigned int cpu) { From ad380f6a0a5e82e794b45bb2eaec24ed51a56846 Mon Sep 17 00:00:00 2001 From: Palmer Dabbelt Date: Wed, 31 Jul 2024 09:22:00 -0700 Subject: [PATCH 43/47] RISC-V: Don't have MAX_PHYSMEM_BITS exceed phys_addr_t I recently ended up with a warning on some compilers along the lines of CC kernel/resource.o In file included from include/linux/ioport.h:16, from kernel/resource.c:15: kernel/resource.c: In function 'gfr_start': include/linux/minmax.h:49:37: error: conversion from 'long long unsigned int' to 'resource_size_t' {aka 'unsigned int'} changes value from '17179869183' to '4294967295' [-Werror=overflow] 49 | ({ type ux = (x); type uy = (y); __cmp(op, ux, uy); }) | ^ include/linux/minmax.h:52:9: note: in expansion of macro '__cmp_once_unique' 52 | __cmp_once_unique(op, type, x, y, __UNIQUE_ID(x_), __UNIQUE_ID(y_)) | ^~~~~~~~~~~~~~~~~ include/linux/minmax.h:161:27: note: in expansion of macro '__cmp_once' 161 | #define min_t(type, x, y) __cmp_once(min, type, x, y) | ^~~~~~~~~~ kernel/resource.c:1829:23: note: in expansion of macro 'min_t' 1829 | end = min_t(resource_size_t, base->end, | ^~~~~ kernel/resource.c: In function 'gfr_continue': include/linux/minmax.h:49:37: error: conversion from 'long long unsigned int' to 'resource_size_t' {aka 'unsigned int'} changes value from '17179869183' to '4294967295' [-Werror=overflow] 49 | ({ type ux = (x); type uy = (y); __cmp(op, ux, uy); }) | ^ include/linux/minmax.h:52:9: note: in expansion of macro '__cmp_once_unique' 52 | __cmp_once_unique(op, type, x, y, __UNIQUE_ID(x_), __UNIQUE_ID(y_)) | ^~~~~~~~~~~~~~~~~ include/linux/minmax.h:161:27: note: in expansion of macro '__cmp_once' 161 | #define min_t(type, x, y) __cmp_once(min, type, x, y) | ^~~~~~~~~~ kernel/resource.c:1847:24: note: in expansion of macro 'min_t' 1847 | addr <= min_t(resource_size_t, base->end, | ^~~~~ cc1: all warnings being treated as errors which looks like a real problem: our phys_addr_t is only 32 bits now, so having 34-bit masks is just going to result in overflows. Reviewed-by: Charlie Jenkins Reviewed-by: Alexandre Ghiti Link: https://lore.kernel.org/r/20240731162159.9235-2-palmer@rivosinc.com Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/sparsemem.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/riscv/include/asm/sparsemem.h b/arch/riscv/include/asm/sparsemem.h index 63acaecc3374..2f901a410586 100644 --- a/arch/riscv/include/asm/sparsemem.h +++ b/arch/riscv/include/asm/sparsemem.h @@ -7,7 +7,7 @@ #ifdef CONFIG_64BIT #define MAX_PHYSMEM_BITS 56 #else -#define MAX_PHYSMEM_BITS 34 +#define MAX_PHYSMEM_BITS 32 #endif /* CONFIG_64BIT */ #define SECTION_SIZE_BITS 27 #endif /* CONFIG_SPARSEMEM */ From 6d74d178fe6eaf61e384f3be6ba64150bddce8a6 Mon Sep 17 00:00:00 2001 From: Charlie Jenkins Date: Tue, 6 Aug 2024 15:01:23 -0700 Subject: [PATCH 44/47] tools: Add riscv barrier implementation Many of the other architectures use their custom barrier implementations. Use the barrier code from the kernel sources to optimize barriers in tools. Signed-off-by: Charlie Jenkins Reviewed-by: Andrea Parri Link: https://lore.kernel.org/r/20240806-optimize_ring_buffer_read_riscv-v2-1-ca7e193ae198@rivosinc.com Signed-off-by: Palmer Dabbelt --- tools/arch/riscv/include/asm/barrier.h | 39 ++++++++++++++++++++++++++ tools/arch/riscv/include/asm/fence.h | 13 +++++++++ tools/include/asm/barrier.h | 2 ++ 3 files changed, 54 insertions(+) create mode 100644 tools/arch/riscv/include/asm/barrier.h create mode 100644 tools/arch/riscv/include/asm/fence.h diff --git a/tools/arch/riscv/include/asm/barrier.h b/tools/arch/riscv/include/asm/barrier.h new file mode 100644 index 000000000000..6997f197086d --- /dev/null +++ b/tools/arch/riscv/include/asm/barrier.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copied from the kernel sources to tools/arch/riscv: + * + * Copyright (C) 2012 ARM Ltd. + * Copyright (C) 2013 Regents of the University of California + * Copyright (C) 2017 SiFive + */ + +#ifndef _TOOLS_LINUX_ASM_RISCV_BARRIER_H +#define _TOOLS_LINUX_ASM_RISCV_BARRIER_H + +#include +#include + +/* These barriers need to enforce ordering on both devices and memory. */ +#define mb() RISCV_FENCE(iorw, iorw) +#define rmb() RISCV_FENCE(ir, ir) +#define wmb() RISCV_FENCE(ow, ow) + +/* These barriers do not need to enforce ordering on devices, just memory. */ +#define smp_mb() RISCV_FENCE(rw, rw) +#define smp_rmb() RISCV_FENCE(r, r) +#define smp_wmb() RISCV_FENCE(w, w) + +#define smp_store_release(p, v) \ +do { \ + RISCV_FENCE(rw, w); \ + WRITE_ONCE(*p, v); \ +} while (0) + +#define smp_load_acquire(p) \ +({ \ + typeof(*p) ___p1 = READ_ONCE(*p); \ + RISCV_FENCE(r, rw); \ + ___p1; \ +}) + +#endif /* _TOOLS_LINUX_ASM_RISCV_BARRIER_H */ diff --git a/tools/arch/riscv/include/asm/fence.h b/tools/arch/riscv/include/asm/fence.h new file mode 100644 index 000000000000..37860e86771d --- /dev/null +++ b/tools/arch/riscv/include/asm/fence.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copied from the kernel sources to tools/arch/riscv: + */ + +#ifndef _ASM_RISCV_FENCE_H +#define _ASM_RISCV_FENCE_H + +#define RISCV_FENCE_ASM(p, s) "\tfence " #p "," #s "\n" +#define RISCV_FENCE(p, s) \ + ({ __asm__ __volatile__ (RISCV_FENCE_ASM(p, s) : : : "memory"); }) + +#endif /* _ASM_RISCV_FENCE_H */ diff --git a/tools/include/asm/barrier.h b/tools/include/asm/barrier.h index 8d378c57cb01..0c21678ac5e6 100644 --- a/tools/include/asm/barrier.h +++ b/tools/include/asm/barrier.h @@ -8,6 +8,8 @@ #include "../../arch/arm64/include/asm/barrier.h" #elif defined(__powerpc__) #include "../../arch/powerpc/include/asm/barrier.h" +#elif defined(__riscv) +#include "../../arch/riscv/include/asm/barrier.h" #elif defined(__s390__) #include "../../arch/s390/include/asm/barrier.h" #elif defined(__sh__) From aa5736dc7aa4d6f0e5e4e4147d9aef42bb82deab Mon Sep 17 00:00:00 2001 From: Charlie Jenkins Date: Tue, 6 Aug 2024 15:01:24 -0700 Subject: [PATCH 45/47] tools: Optimize ring buffer for riscv Now that the riscv tools tree supports optimized barriers, use them in the ring buffer. Signed-off-by: Charlie Jenkins Reviewed-by: Andrea Parri Link: https://lore.kernel.org/r/20240806-optimize_ring_buffer_read_riscv-v2-2-ca7e193ae198@rivosinc.com Signed-off-by: Palmer Dabbelt --- tools/include/linux/ring_buffer.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/include/linux/ring_buffer.h b/tools/include/linux/ring_buffer.h index 6c02617377c2..a74c397359c7 100644 --- a/tools/include/linux/ring_buffer.h +++ b/tools/include/linux/ring_buffer.h @@ -55,7 +55,7 @@ static inline u64 ring_buffer_read_head(struct perf_event_mmap_page *base) * READ_ONCE() + smp_mb() pair. */ #if defined(__x86_64__) || defined(__aarch64__) || defined(__powerpc64__) || \ - defined(__ia64__) || defined(__sparc__) && defined(__arch64__) + defined(__ia64__) || defined(__sparc__) && defined(__arch64__) || defined(__riscv) return smp_load_acquire(&base->data_head); #else u64 head = READ_ONCE(base->data_head); From f0c9363db2ddfb07723c00cb153c84c8179e92d4 Mon Sep 17 00:00:00 2001 From: Mayuresh Chitale Date: Mon, 12 Aug 2024 05:11:09 +0000 Subject: [PATCH 46/47] perf/riscv-sbi: Add platform specific firmware event handling The SBI v2.0 specification pointed to by the link below reserves the event code 0xffff for platform specific firmware events. Update the driver to be able to parse and program such events. The platform specific firmware events must now be specified in the perf command as below: perf stat -e rCxxx ... where bits[63:62] = 0x3 of the event config indicate a platform specific firmware event and xxx indicate the actual event code which is passed as the event data. Signed-off-by: Mayuresh Chitale Link: https://github.com/riscv-non-isa/riscv-sbi-doc/releases/download/v2.0/riscv-sbi.pdf Link: https://lore.kernel.org/r/20240812051109.6496-1-mchitale@ventanamicro.com Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/sbi.h | 1 + drivers/perf/riscv_pmu_sbi.c | 33 +++++++++++++++++++++++---------- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/arch/riscv/include/asm/sbi.h b/arch/riscv/include/asm/sbi.h index 7cffd4ffecd0..4ef8c44b15bb 100644 --- a/arch/riscv/include/asm/sbi.h +++ b/arch/riscv/include/asm/sbi.h @@ -158,6 +158,7 @@ struct riscv_pmu_snapshot_data { #define RISCV_PMU_RAW_EVENT_MASK GENMASK_ULL(47, 0) #define RISCV_PMU_RAW_EVENT_IDX 0x20000 +#define RISCV_PLAT_FW_EVENT 0xFFFF /** General pmu event codes specified in SBI PMU extension */ enum sbi_pmu_hw_generic_events_t { diff --git a/drivers/perf/riscv_pmu_sbi.c b/drivers/perf/riscv_pmu_sbi.c index 31a17a56eb3b..e77cbe299ce7 100644 --- a/drivers/perf/riscv_pmu_sbi.c +++ b/drivers/perf/riscv_pmu_sbi.c @@ -60,7 +60,7 @@ asm volatile(ALTERNATIVE( \ #define PERF_EVENT_FLAG_LEGACY BIT(SYSCTL_LEGACY) PMU_FORMAT_ATTR(event, "config:0-47"); -PMU_FORMAT_ATTR(firmware, "config:63"); +PMU_FORMAT_ATTR(firmware, "config:62-63"); static bool sbi_v2_available; static DEFINE_STATIC_KEY_FALSE(sbi_pmu_snapshot_available); @@ -507,7 +507,6 @@ static int pmu_sbi_event_map(struct perf_event *event, u64 *econfig) { u32 type = event->attr.type; u64 config = event->attr.config; - int bSoftware; u64 raw_config_val; int ret; @@ -528,18 +527,32 @@ static int pmu_sbi_event_map(struct perf_event *event, u64 *econfig) break; case PERF_TYPE_RAW: /* - * As per SBI specification, the upper 16 bits must be unused for - * a raw event. Use the MSB (63b) to distinguish between hardware - * raw event and firmware events. + * As per SBI specification, the upper 16 bits must be unused + * for a raw event. + * Bits 63:62 are used to distinguish between raw events + * 00 - Hardware raw event + * 10 - SBI firmware events + * 11 - Risc-V platform specific firmware event */ - bSoftware = config >> 63; raw_config_val = config & RISCV_PMU_RAW_EVENT_MASK; - if (bSoftware) { - ret = (raw_config_val & 0xFFFF) | - (SBI_PMU_EVENT_TYPE_FW << 16); - } else { + switch (config >> 62) { + case 0: ret = RISCV_PMU_RAW_EVENT_IDX; *econfig = raw_config_val; + break; + case 2: + ret = (raw_config_val & 0xFFFF) | + (SBI_PMU_EVENT_TYPE_FW << 16); + break; + case 3: + /* + * For Risc-V platform specific firmware events + * Event code - 0xFFFF + * Event data - raw event encoding + */ + ret = SBI_PMU_EVENT_TYPE_FW << 16 | RISCV_PLAT_FW_EVENT; + *econfig = raw_config_val; + break; } break; default: From b3f835cd7339919561866252a11831ead72e7073 Mon Sep 17 00:00:00 2001 From: Jinjie Ruan Date: Mon, 12 Aug 2024 14:20:17 +0800 Subject: [PATCH 47/47] crash: Fix riscv64 crash memory reserve dead loop On RISCV64 Qemu machine with 512MB memory, cmdline "crashkernel=500M,high" will cause system stall as below: Zone ranges: DMA32 [mem 0x0000000080000000-0x000000009fffffff] Normal empty Movable zone start for each node Early memory node ranges node 0: [mem 0x0000000080000000-0x000000008005ffff] node 0: [mem 0x0000000080060000-0x000000009fffffff] Initmem setup node 0 [mem 0x0000000080000000-0x000000009fffffff] (stall here) commit 5d99cadf1568 ("crash: fix x86_32 crash memory reserve dead loop bug") fix this on 32-bit architecture. However, the problem is not completely solved. If `CRASH_ADDR_LOW_MAX = CRASH_ADDR_HIGH_MAX` on 64-bit architecture, for example, when system memory is equal to CRASH_ADDR_LOW_MAX on RISCV64, the following infinite loop will also occur: -> reserve_crashkernel_generic() and high is true -> alloc at [CRASH_ADDR_LOW_MAX, CRASH_ADDR_HIGH_MAX] fail -> alloc at [0, CRASH_ADDR_LOW_MAX] fail and repeatedly (because CRASH_ADDR_LOW_MAX = CRASH_ADDR_HIGH_MAX). As Catalin suggested, do not remove the ",high" reservation fallback to ",low" logic which will change arm64's kdump behavior, but fix it by skipping the above situation similar to commit d2f32f23190b ("crash: fix x86_32 crash memory reserve dead loop"). After this patch, it print: cannot allocate crashkernel (size:0x1f400000) Signed-off-by: Jinjie Ruan Suggested-by: Catalin Marinas Reviewed-by: Catalin Marinas Acked-by: Baoquan He Link: https://lore.kernel.org/r/20240812062017.2674441-1-ruanjinjie@huawei.com Signed-off-by: Palmer Dabbelt --- kernel/crash_reserve.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kernel/crash_reserve.c b/kernel/crash_reserve.c index d3b4cd12bdd1..64d44a52c011 100644 --- a/kernel/crash_reserve.c +++ b/kernel/crash_reserve.c @@ -423,7 +423,8 @@ retry: if (high && search_end == CRASH_ADDR_HIGH_MAX) { search_end = CRASH_ADDR_LOW_MAX; search_base = 0; - goto retry; + if (search_end != CRASH_ADDR_HIGH_MAX) + goto retry; } pr_warn("cannot allocate crashkernel (size:0x%llx)\n", crash_size);