dummy/linux-stable-mirror
1
0
Fork 1
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/ synced 2025-04-19 20:58:31 +09:00
Code Issues Packages Projects Releases Wiki Activity

RISC-V Patches for the 6.15 Merge Window, Part 1

Browse Source 
* The sub-architecture selection Kconfig system has been cleaned up,
   the documentation has been improved, and various detections have been
   fixed.
 * The vector-related extensions dependencies are now validated when
   parsing from device tree and in the DT bindings.
 * Misaligned access probing can be overridden via a kernel command-line
   parameter, along with various fixes to misalign access handling.
 * Support for relocatable !MMU kernels builds.
 * Support for hpge pfnmaps, which should improve TLB utilization.
 * Support for runtime constants, which improves the d_hash()
   performance.
 * Support for bfloat16, Zicbom, Zaamo, Zalrsc, Zicntr, Zihpm.
 * Various fixes, including:
       - We were missing a secondary mmu notifier call when flushing the
 	tlb which is required for IOMMU.
       - Fix ftrace panics by saving the registers as expected by ftrace.
       - Fix a couple of stimecmp usage related to cpu hotplug.
       - purgatory_start is now aligned as per the STVEC requirements.
       - A fix for hugetlb when calculating the size of non-present PTEs.
 -----BEGIN PGP SIGNATURE-----
 
 iQJHBAABCAAxFiEEKzw3R0RoQ7JKlDp6LhMZ81+7GIkFAmfv/soTHHBhbG1lckBk
 YWJiZWx0LmNvbQAKCRAuExnzX7sYierZEACDwI9lJFCEbQPon3z8rAy1moTj0+AZ
 bMfZFqMphUTrJ0cMm2+Bc+XZgck12zHCyu1UljDcZVYMCHA9aOoj5C5NkBBVLCuL
 uLYrhIoQXtJaVIANiFl0SHAZmh2s2OoSgmUzrEZ8JGlHpKCF7EVX5bHEsOvzn9ir
 B2W992W6q3ISuKXHKsTpa7rmTtf7swGYg6zW3pX3l6HmY+EMEQOcQl0tAB383J/T
 lm0K4+YvLpRJdm2ARpNGWlcFXj9/UXUM5hplK3aBAHpPKQ5/83/4tMDsfRvhpEVC
 VJXNgK+H4XLD542aQ8d4ZROguyhwn9e2n6Dkv0OqfNk4lg5pUBcJUZftQ+rB7AWg
 VYB1KVpxhwcruheXJFz8S3EzjZTcS+JrcD80vvx8JmHdXkZwHTfYUgiFwe/TR7yr
 b518fEbXpVwDZiCbaAe3Cmpw0mlNnSVmU4hgNbiwt0fu9DGdPN9WQbyds68RKb7A
 TWwDmmD6kV2BTWl0mHPtu9VhX58CDG+0WYbHA7r82p2T50187766C92GYfN2UPpz
 lH0iMRDkmucclZ3fEoosJ+HsDntc4oe6Bhdzuj52Q7vBpDd/QB6t5cfrlDpEEdgU
 3qoWMN5mb5l1rbvrqENh5ZgmEpzV8K0R5F5quiXh/9wO0y1kepDslTqC2oXK/m0p
 DzsvvD6UnNMOUQ==
 =nCJo
 -----END PGP SIGNATURE-----

Merge tag 'riscv-for-linus-6.15-mw1' of git://git.kernel.org/pub/scm/linux/kernel/git/riscv/linux

Pull RISC-V updates from Palmer Dabbelt:

 - The sub-architecture selection Kconfig system has been cleaned up,
   the documentation has been improved, and various detections have been
   fixed

 - The vector-related extensions dependencies are now validated when
   parsing from device tree and in the DT bindings

 - Misaligned access probing can be overridden via a kernel command-line
   parameter, along with various fixes to misalign access handling

 - Support for relocatable !MMU kernels builds

 - Support for hpge pfnmaps, which should improve TLB utilization

 - Support for runtime constants, which improves the d_hash()
   performance

 - Support for bfloat16, Zicbom, Zaamo, Zalrsc, Zicntr, Zihpm

 - Various fixes, including:
      - We were missing a secondary mmu notifier call when flushing the
        tlb which is required for IOMMU
      - Fix ftrace panics by saving the registers as expected by ftrace
      - Fix a couple of stimecmp usage related to cpu hotplug
      - purgatory_start is now aligned as per the STVEC requirements
      - A fix for hugetlb when calculating the size of non-present PTEs

* tag 'riscv-for-linus-6.15-mw1' of git://git.kernel.org/pub/scm/linux/kernel/git/riscv/linux: (65 commits)
  riscv: Add norvc after .option arch in runtime const
  riscv: Make sure toolchain supports zba before using zba instructions
  riscv/purgatory: 4B align purgatory_start
  riscv/kexec_file: Handle R_RISCV_64 in purgatory relocator
  selftests: riscv: fix v_exec_initval_nolibc.c
  riscv: Fix hugetlb retrieval of number of ptes in case of !present pte
  riscv: print hartid on bringup
  riscv: Add norvc after .option arch in runtime const
  riscv: Remove CONFIG_PAGE_OFFSET
  riscv: Support CONFIG_RELOCATABLE on riscv32
  asm-generic: Always define Elf_Rel and Elf_Rela
  riscv: Support CONFIG_RELOCATABLE on NOMMU
  riscv: Allow NOMMU kernels to access all of RAM
  riscv: Remove duplicate CONFIG_PAGE_OFFSET definition
  RISC-V: errata: Use medany for relocatable builds
  dt-bindings: riscv: document vector crypto requirements
  dt-bindings: riscv: add vector sub-extension dependencies
  dt-bindings: riscv: d requires f
  RISC-V: add f & d extension validation checks
  RISC-V: add vector crypto extension validation checks
  ...
This commit is contained in:
Linus Torvalds 2025-04-04 09:49:17 -07:00
commit 4a1d8ababd
62 changed files with 1255 additions and 404 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
Documentation/admin-guide/kernel-parameters.txt
View File

@ -7535,6 +7535,22 @@
Note that genuine overcurrent events won't be
reported either.
unaligned_scalar_speed=
[RISCV]
Format: {slow | fast | unsupported}
Allow skipping scalar unaligned access speed tests. This
is useful for testing alternative code paths and to skip
the tests in environments where they run too slowly. All
CPUs must have the same scalar unaligned access speed.
unaligned_vector_speed=
[RISCV]
Format: {slow | fast | unsupported}
Allow skipping vector unaligned access speed tests. This
is useful for testing alternative code paths and to skip
the tests in environments where they run too slowly. All
CPUs must have the same vector unaligned access speed.
unknown_nmi_panic
[X86] Cause panic on unknown NMI.

32
Documentation/arch/riscv/hwprobe.rst
View File

@ -183,6 +183,9 @@ The following keys are defined:
defined in the Atomic Compare-and-Swap (CAS) instructions manual starting
from commit 5059e0ca641c ("update to ratified").
* :c:macro:`RISCV_HWPROBE_EXT_ZICNTR`: The Zicntr extension version 2.0
is supported as defined in the RISC-V ISA manual.
* :c:macro:`RISCV_HWPROBE_EXT_ZICOND`: The Zicond extension is supported as
defined in the RISC-V Integer Conditional (Zicond) operations extension
manual starting from commit 95cf1f9 ("Add changes requested by Ved
@ -192,6 +195,9 @@ The following keys are defined:
supported as defined in the RISC-V ISA manual starting from commit
d8ab5c78c207 ("Zihintpause is ratified").
* :c:macro:`RISCV_HWPROBE_EXT_ZIHPM`: The Zihpm extension version 2.0
is supported as defined in the RISC-V ISA manual.
* :c:macro:`RISCV_HWPROBE_EXT_ZVE32X`: The Vector sub-extension Zve32x is
supported, as defined by version 1.0 of the RISC-V Vector extension manual.
@ -239,9 +245,32 @@ The following keys are defined:
ratified in commit 98918c844281 ("Merge pull request #1217 from
riscv/zawrs") of riscv-isa-manual.
* :c:macro:`RISCV_HWPROBE_EXT_ZAAMO`: The Zaamo extension is supported as
defined in the in the RISC-V ISA manual starting from commit e87412e621f1
("integrate Zaamo and Zalrsc text (#1304)").
* :c:macro:`RISCV_HWPROBE_EXT_ZALRSC`: The Zalrsc extension is supported as
defined in the in the RISC-V ISA manual starting from commit e87412e621f1
("integrate Zaamo and Zalrsc text (#1304)").
* :c:macro:`RISCV_HWPROBE_EXT_SUPM`: The Supm extension is supported as
defined in version 1.0 of the RISC-V Pointer Masking extensions.
* :c:macro:`RISCV_HWPROBE_EXT_ZFBFMIN`: The Zfbfmin extension is supported as
defined in the RISC-V ISA manual starting from commit 4dc23d6229de
("Added Chapter title to BF16").
* :c:macro:`RISCV_HWPROBE_EXT_ZVFBFMIN`: The Zvfbfmin extension is supported as
defined in the RISC-V ISA manual starting from commit 4dc23d6229de
("Added Chapter title to BF16").
* :c:macro:`RISCV_HWPROBE_EXT_ZVFBFWMA`: The Zvfbfwma extension is supported as
defined in the RISC-V ISA manual starting from commit 4dc23d6229de
("Added Chapter title to BF16").
* :c:macro:`RISCV_HWPROBE_EXT_ZICBOM`: The Zicbom extension is supported, as
ratified in commit 3dd606f ("Create cmobase-v1.0.pdf") of riscv-CMOs.
* :c:macro:`RISCV_HWPROBE_KEY_CPUPERF_0`: Deprecated. Returns similar values to
:c:macro:`RISCV_HWPROBE_KEY_MISALIGNED_SCALAR_PERF`, but the key was
mistakenly classified as a bitmask rather than a value.
@ -303,3 +332,6 @@ The following keys are defined:
* :c:macro:`RISCV_HWPROBE_VENDOR_EXT_XTHEADVECTOR`: The xtheadvector vendor
extension is supported in the T-Head ISA extensions spec starting from
commit a18c801634 ("Add T-Head VECTOR vendor extension. ").
* :c:macro:`RISCV_HWPROBE_KEY_ZICBOM_BLOCK_SIZE`: An unsigned int which
represents the size of the Zicbom block in bytes.

149
Documentation/devicetree/bindings/riscv/extensions.yaml
View File

@ -224,6 +224,12 @@ properties:
as ratified at commit 4a69197e5617 ("Update to ratified state") of
riscv-svvptc.
- const: zaamo
description: |
The standard Zaamo extension for atomic memory operations as
ratified at commit e87412e621f1 ("integrate Zaamo and Zalrsc text
(#1304)") of the unprivileged ISA specification.
- const: zabha
description: |
The Zabha extension for Byte and Halfword Atomic Memory Operations
@ -236,6 +242,12 @@ properties:
is supported as ratified at commit 5059e0ca641c ("update to
ratified") of the riscv-zacas.
- const: zalrsc
description: |
The standard Zalrsc extension for load-reserved/store-conditional as
ratified at commit e87412e621f1 ("integrate Zaamo and Zalrsc text
(#1304)") of the unprivileged ISA specification.
- const: zawrs
description: |
The Zawrs extension for entering a low-power state or for trapping
@ -329,6 +341,12 @@ properties:
instructions, as ratified in commit 056b6ff ("Zfa is ratified") of
riscv-isa-manual.
- const: zfbfmin
description:
The standard Zfbfmin extension which provides minimal support for
16-bit half-precision brain floating-point instructions, as ratified
in commit 4dc23d62 ("Added Chapter title to BF16") of riscv-isa-manual.
- const: zfh
description:
The standard Zfh extension for 16-bit half-precision binary
@ -525,6 +543,18 @@ properties:
in commit 6f702a2 ("Vector extensions are now ratified") of
riscv-v-spec.
- const: zvfbfmin
description:
The standard Zvfbfmin extension for minimal support for vectored
16-bit half-precision brain floating-point instructions, as ratified
in commit 4dc23d62 ("Added Chapter title to BF16") of riscv-isa-manual.
- const: zvfbfwma
description:
The standard Zvfbfwma extension for vectored half-precision brain
floating-point widening multiply-accumulate instructions, as ratified
in commit 4dc23d62 ("Added Chapter title to BF16") of riscv-isa-manual.
- const: zvfh
description:
The standard Zvfh extension for vectored half-precision
@ -639,6 +669,12 @@ properties:
https://github.com/T-head-Semi/thead-extension-spec/blob/95358cb2cca9489361c61d335e03d3134b14133f/xtheadvector.adoc.
allOf:
- if:
contains:
const: d
then:
contains:
const: f
# Zcb depends on Zca
- if:
contains:
@ -673,6 +709,119 @@ properties:
then:
contains:
const: zca
# Zfbfmin depends on F
- if:
contains:
const: zfbfmin
then:
contains:
const: f
# Zvfbfmin depends on V or Zve32f
- if:
contains:
const: zvfbfmin
then:
oneOf:
- contains:
const: v
- contains:
const: zve32f
# Zvfbfwma depends on Zfbfmin and Zvfbfmin
- if:
contains:
const: zvfbfwma
then:
allOf:
- contains:
const: zfbfmin
- contains:
const: zvfbfmin
# Zacas depends on Zaamo
- if:
contains:
const: zacas
then:
contains:
const: zaamo
- if:
contains:
const: zve32x
then:
contains:
const: zicsr
- if:
contains:
const: zve32f
then:
allOf:
- contains:
const: f
- contains:
const: zve32x
- if:
contains:
const: zve64x
then:
contains:
const: zve32x
- if:
contains:
const: zve64f
then:
allOf:
- contains:
const: f
- contains:
const: zve32f
- contains:
const: zve64x
- if:
contains:
const: zve64d
then:
allOf:
- contains:
const: d
- contains:
const: zve64f
- if:
contains:
anyOf:
- const: zvbc
- const: zvkn
- const: zvknc
- const: zvkng
- const: zvknhb
- const: zvksc
then:
contains:
anyOf:
- const: v
- const: zve64x
- if:
contains:
anyOf:
- const: zvbb
- const: zvkb
- const: zvkg
- const: zvkned
- const: zvknha
- const: zvksed
- const: zvksh
- const: zvks
- const: zvkt
then:
contains:
anyOf:
- const: v
- const: zve32x
allOf:
# Zcf extension does not exist on rv64

1
arch/riscv/Kbuild
View File

@ -1,7 +1,6 @@
# SPDX-License-Identifier: GPL-2.0-only
obj-y += kernel/ mm/ net/
obj-$(CONFIG_BUILTIN_DTB) += boot/dts/
obj-$(CONFIG_CRYPTO) += crypto/
obj-y += errata/
obj-$(CONFIG_KVM) += kvm/

84
arch/riscv/Kconfig
View File

@ -65,6 +65,7 @@ config RISCV
select ARCH_SUPPORTS_ATOMIC_RMW
select ARCH_SUPPORTS_CFI_CLANG
select ARCH_SUPPORTS_DEBUG_PAGEALLOC if MMU
select ARCH_SUPPORTS_HUGE_PFNMAP if TRANSPARENT_HUGEPAGE
select ARCH_SUPPORTS_HUGETLBFS if MMU
# LLD >= 14: https://github.com/llvm/llvm-project/issues/50505
select ARCH_SUPPORTS_LTO_CLANG if LLD_VERSION >= 140000
@ -153,7 +154,7 @@ config RISCV
select HAVE_DYNAMIC_FTRACE_WITH_ARGS if HAVE_DYNAMIC_FTRACE
select HAVE_FTRACE_GRAPH_FUNC
select HAVE_FTRACE_MCOUNT_RECORD if !XIP_KERNEL
select HAVE_FUNCTION_GRAPH_TRACER
select HAVE_FUNCTION_GRAPH_TRACER if HAVE_DYNAMIC_FTRACE_WITH_ARGS
select HAVE_FUNCTION_GRAPH_FREGS
select HAVE_FUNCTION_TRACER if !XIP_KERNEL && !PREEMPTION
select HAVE_EBPF_JIT if MMU
@ -205,6 +206,7 @@ config RISCV
select PCI_DOMAINS_GENERIC if PCI
select PCI_ECAM if (ACPI && PCI)
select PCI_MSI if PCI
select RELOCATABLE if !MMU && !PHYS_RAM_BASE_FIXED
select RISCV_ALTERNATIVE if !XIP_KERNEL
select RISCV_APLIC
select RISCV_IMSIC
@ -292,13 +294,6 @@ config MMU
Select if you want MMU-based virtualised addressing space
support by paged memory management. If unsure, say 'Y'.
config PAGE_OFFSET
hex
default 0x80000000 if !MMU && RISCV_M_MODE
default 0x80200000 if !MMU
default 0xc0000000 if 32BIT
default 0xff60000000000000 if 64BIT
config KASAN_SHADOW_OFFSET
hex
depends on KASAN_GENERIC
@ -570,7 +565,8 @@ config RISCV_ISA_C
help
Adds "C" to the ISA subsets that the toolchain is allowed to emit
when building Linux, which results in compressed instructions in the
Linux binary.
Linux binary. This option produces a kernel that will not run on
systems that do not support compressed instructions.
If you don't know what to do here, say Y.
@ -591,8 +587,8 @@ config RISCV_ISA_SVNAPOT
depends on RISCV_ALTERNATIVE
default y
help
Allow kernel to detect the Svnapot ISA-extension dynamically at boot
time and enable its usage.
Enable support for the Svnapot ISA-extension when it is detected
at boot.
The Svnapot extension is used to mark contiguous PTEs as a range
of contiguous virtual-to-physical translations for a naturally
@ -610,9 +606,8 @@ config RISCV_ISA_SVPBMT
depends on RISCV_ALTERNATIVE
default y
help
Adds support to dynamically detect the presence of the Svpbmt
ISA-extension (Supervisor-mode: page-based memory types) and
enable its usage.
Add support for the Svpbmt ISA-extension (Supervisor-mode:
page-based memory types) in the kernel when it is detected at boot.
The memory type for a page contains a combination of attributes
that indicate the cacheability, idempotency, and ordering
@ -631,14 +626,15 @@ config TOOLCHAIN_HAS_V
depends on AS_HAS_OPTION_ARCH
config RISCV_ISA_V
bool "VECTOR extension support"
bool "Vector extension support"
depends on TOOLCHAIN_HAS_V
depends on FPU
select DYNAMIC_SIGFRAME
default y
help
Say N here if you want to disable all vector related procedure
in the kernel.
Add support for the Vector extension when it is detected at boot.
When this option is disabled, neither the kernel nor userspace may
use vector procedures.
If you don't know what to do here, say Y.
@ -737,6 +733,14 @@ config TOOLCHAIN_HAS_VECTOR_CRYPTO
def_bool $(as-instr, .option arch$(comma) +v$(comma) +zvkb)
depends on AS_HAS_OPTION_ARCH
config TOOLCHAIN_HAS_ZBA
bool
default y
depends on !64BIT || $(cc-option,-mabi=lp64 -march=rv64ima_zba)
depends on !32BIT || $(cc-option,-mabi=ilp32 -march=rv32ima_zba)
depends on LLD_VERSION >= 150000 || LD_VERSION >= 23900
depends on AS_HAS_OPTION_ARCH
config RISCV_ISA_ZBA
bool "Zba extension support for bit manipulation instructions"
default y
@ -751,12 +755,12 @@ config RISCV_ISA_ZBA
config RISCV_ISA_ZBB
bool "Zbb extension support for bit manipulation instructions"
depends on TOOLCHAIN_HAS_ZBB
depends on RISCV_ALTERNATIVE
default y
help
Adds support to dynamically detect the presence of the ZBB
extension (basic bit manipulation) and enable its usage.
Add support for enabling optimisations in the kernel when the
Zbb extension is detected at boot. Some optimisations may
additionally depend on toolchain support for Zbb.
The Zbb extension provides instructions to accelerate a number
of bit-specific operations (count bit population, sign extending,
@ -787,6 +791,28 @@ config RISCV_ISA_ZBC
If you don't know what to do here, say Y.
config TOOLCHAIN_HAS_ZBKB
bool
default y
depends on !64BIT || $(cc-option,-mabi=lp64 -march=rv64ima_zbkb)
depends on !32BIT || $(cc-option,-mabi=ilp32 -march=rv32ima_zbkb)
depends on LLD_VERSION >= 150000 || LD_VERSION >= 23900
depends on AS_HAS_OPTION_ARCH
config RISCV_ISA_ZBKB
bool "Zbkb extension support for bit manipulation instructions"
depends on TOOLCHAIN_HAS_ZBKB
depends on RISCV_ALTERNATIVE
default y
help
Adds support to dynamically detect the presence of the ZBKB
extension (bit manipulation for cryptography) and enable its usage.
The Zbkb extension provides instructions to accelerate a number
of common cryptography operations (pack, zip, etc).
If you don't know what to do here, say Y.
config RISCV_ISA_ZICBOM
bool "Zicbom extension support for non-coherent DMA operation"
depends on MMU
@ -795,9 +821,9 @@ config RISCV_ISA_ZICBOM
select RISCV_DMA_NONCOHERENT
select DMA_DIRECT_REMAP
help
Adds support to dynamically detect the presence of the ZICBOM
extension (Cache Block Management Operations) and enable its
usage.
Add support for the Zicbom extension (Cache Block Management
Operations) and enable its use in the kernel when it is detected
at boot.
The Zicbom extension can be used to handle for example
non-coherent DMA support on devices that need it.
@ -810,7 +836,7 @@ config RISCV_ISA_ZICBOZ
default y
help
Enable the use of the Zicboz extension (cbo.zero instruction)
when available.
in the kernel when it is detected at boot.
The Zicboz extension is used for faster zeroing of memory.
@ -848,8 +874,9 @@ config FPU
bool "FPU support"
default y
help
Say N here if you want to disable all floating-point related procedure
in the kernel.
Add support for floating point operations when an FPU is detected at
boot. When this option is disabled, neither the kernel nor userspace
may use the floating point unit.
If you don't know what to do here, say Y.
@ -1079,7 +1106,7 @@ config PARAVIRT_TIME_ACCOUNTING
config RELOCATABLE
bool "Build a relocatable kernel"
depends on MMU && 64BIT && !XIP_KERNEL
depends on !XIP_KERNEL
select MODULE_SECTIONS if MODULES
help
This builds a kernel as a Position Independent Executable (PIE),
@ -1273,13 +1300,14 @@ config RISCV_ISA_FALLBACK
config BUILTIN_DTB
bool "Built-in device tree"
depends on OF && NONPORTABLE
select GENERIC_BUILTIN_DTB
help
Build a device tree into the Linux image.
This option should be selected if no bootloader is being used.
If unsure, say N.
config BUILTIN_DTB_SOURCE
config BUILTIN_DTB_NAME
string "Built-in device tree source"
depends on BUILTIN_DTB
help

2
arch/riscv/Kconfig.socs
View File

@ -59,7 +59,6 @@ config ARCH_THEAD
config ARCH_VIRT
bool "QEMU Virt Machine"
select CLINT_TIMER if RISCV_M_MODE
select POWER_RESET
select POWER_RESET_SYSCON
select POWER_RESET_SYSCON_POWEROFF
@ -79,7 +78,6 @@ config ARCH_CANAAN
config SOC_CANAAN_K210
bool "Canaan Kendryte K210 SoC"
depends on !MMU && ARCH_CANAAN
select CLINT_TIMER if RISCV_M_MODE
select ARCH_HAS_RESET_CONTROLLER
select PINCTRL
select COMMON_CLK

1
arch/riscv/Makefile
View File

@ -98,7 +98,6 @@ KBUILD_AFLAGS += -march=$(riscv-march-y)
CC_FLAGS_FPU := -march=$(shell echo $(riscv-march-y) | sed -E 's/(rv32ima|rv64ima)([^v_]*)v?/\1\2/')
KBUILD_CFLAGS += -mno-save-restore
KBUILD_CFLAGS += -DCONFIG_PAGE_OFFSET=$(CONFIG_PAGE_OFFSET)
ifeq ($(CONFIG_CMODEL_MEDLOW),y)
KBUILD_CFLAGS += -mcmodel=medlow

2
arch/riscv/boot/dts/Makefile
View File

@ -8,5 +8,3 @@ subdir-y += sophgo
subdir-y += spacemit
subdir-y += starfive
subdir-y += thead
obj-$(CONFIG_BUILTIN_DTB) := $(addsuffix .dtb.o, $(CONFIG_BUILTIN_DTB_SOURCE))

2
arch/riscv/configs/defconfig
View File

@ -26,7 +26,6 @@ CONFIG_EXPERT=y
# CONFIG_SYSFS_SYSCALL is not set
CONFIG_PROFILING=y
CONFIG_ARCH_MICROCHIP=y
CONFIG_ARCH_RENESAS=y
CONFIG_ARCH_SIFIVE=y
CONFIG_ARCH_SOPHGO=y
CONFIG_ARCH_SPACEMIT=y
@ -202,7 +201,6 @@ CONFIG_USB=y
CONFIG_USB_OTG=y
CONFIG_USB_XHCI_HCD=y
CONFIG_USB_XHCI_PLATFORM=y
# CONFIG_USB_XHCI_RCAR is not set
CONFIG_USB_EHCI_HCD=y
CONFIG_USB_EHCI_HCD_PLATFORM=y
CONFIG_USB_OHCI_HCD=y

2
arch/riscv/configs/nommu_k210_defconfig
View File

@ -35,7 +35,7 @@ CONFIG_NR_CPUS=2
CONFIG_CMDLINE="earlycon console=ttySIF0"
CONFIG_CMDLINE_FORCE=y
CONFIG_BUILTIN_DTB=y
CONFIG_BUILTIN_DTB_SOURCE="canaan/k210_generic"
CONFIG_BUILTIN_DTB_NAME="canaan/k210_generic"
# CONFIG_SECCOMP is not set
# CONFIG_STACKPROTECTOR is not set
# CONFIG_GCC_PLUGINS is not set

2
arch/riscv/configs/nommu_k210_sdcard_defconfig
View File

@ -27,7 +27,7 @@ CONFIG_NR_CPUS=2
CONFIG_CMDLINE="earlycon console=ttySIF0 root=/dev/mmcblk0p1 rootwait ro"
CONFIG_CMDLINE_FORCE=y
CONFIG_BUILTIN_DTB=y
CONFIG_BUILTIN_DTB_SOURCE="canaan/k210_generic"
CONFIG_BUILTIN_DTB_NAME="canaan/k210_generic"
# CONFIG_SECCOMP is not set
# CONFIG_STACKPROTECTOR is not set
# CONFIG_GCC_PLUGINS is not set

6
arch/riscv/errata/Makefile
View File

@ -1,5 +1,9 @@
ifdef CONFIG_RELOCATABLE
KBUILD_CFLAGS += -fno-pie
# We can't use PIC/PIE when handling early-boot errata parsing, as the kernel
# doesn't have a GOT setup at that point. So instead just use medany: it's
# usually position-independent, so it should be good enough for the errata
# handling.
KBUILD_CFLAGS += -fno-pie -mcmodel=medany
endif
ifdef CONFIG_RISCV_ALTERNATIVE_EARLY

6
arch/riscv/include/asm/arch_hweight.h
View File

@ -19,7 +19,7 @@
static __always_inline unsigned int __arch_hweight32(unsigned int w)
{
#ifdef CONFIG_RISCV_ISA_ZBB
#if defined(CONFIG_RISCV_ISA_ZBB) && defined(CONFIG_TOOLCHAIN_HAS_ZBB)
asm goto(ALTERNATIVE("j %l[legacy]", "nop", 0,
RISCV_ISA_EXT_ZBB, 1)
: : : : legacy);
@ -50,7 +50,7 @@ static inline unsigned int __arch_hweight8(unsigned int w)
#if BITS_PER_LONG == 64
static __always_inline unsigned long __arch_hweight64(__u64 w)
{
# ifdef CONFIG_RISCV_ISA_ZBB
#if defined(CONFIG_RISCV_ISA_ZBB) && defined(CONFIG_TOOLCHAIN_HAS_ZBB)
asm goto(ALTERNATIVE("j %l[legacy]", "nop", 0,
RISCV_ISA_EXT_ZBB, 1)
: : : : legacy);
@ -64,7 +64,7 @@ static __always_inline unsigned long __arch_hweight64(__u64 w)
return w;
legacy:
# endif
#endif
return __sw_hweight64(w);
}
#else /* BITS_PER_LONG == 64 */

1
arch/riscv/include/asm/asm.h
View File

@ -27,6 +27,7 @@
#define REG_ASM __REG_SEL(.dword, .word)
#define SZREG __REG_SEL(8, 4)
#define LGREG __REG_SEL(3, 2)
#define SRLI __REG_SEL(srliw, srli)
#if __SIZEOF_POINTER__ == 8
#ifdef __ASSEMBLY__

4
arch/riscv/include/asm/bitops.h
View File

@ -15,7 +15,7 @@
#include <asm/barrier.h>
#include <asm/bitsperlong.h>
#if !defined(CONFIG_RISCV_ISA_ZBB) || defined(NO_ALTERNATIVE)
#if !(defined(CONFIG_RISCV_ISA_ZBB) && defined(CONFIG_TOOLCHAIN_HAS_ZBB)) || defined(NO_ALTERNATIVE)
#include <asm-generic/bitops/__ffs.h>
#include <asm-generic/bitops/__fls.h>
#include <asm-generic/bitops/ffs.h>
@ -175,7 +175,7 @@ legacy:
variable_fls(x_); \
})
#endif /* !defined(CONFIG_RISCV_ISA_ZBB) || defined(NO_ALTERNATIVE) */
#endif /* !(defined(CONFIG_RISCV_ISA_ZBB) && defined(CONFIG_TOOLCHAIN_HAS_ZBB)) || defined(NO_ALTERNATIVE) */
#include <asm-generic/bitops/ffz.h>
#include <asm-generic/bitops/fls64.h>

3
arch/riscv/include/asm/checksum.h
View File

@ -49,8 +49,7 @@ static inline __sum16 ip_fast_csum(const void *iph, unsigned int ihl)
* ZBB only saves three instructions on 32-bit and five on 64-bit so not
* worth checking if supported without Alternatives.
*/
if (IS_ENABLED(CONFIG_RISCV_ISA_ZBB) &&
IS_ENABLED(CONFIG_RISCV_ALTERNATIVE)) {
if (IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZBB)) {
unsigned long fold_temp;
asm goto(ALTERNATIVE("j %l[no_zbb]", "nop", 0,

38
arch/riscv/include/asm/cmpxchg.h
View File

@ -365,16 +365,48 @@ static __always_inline void __cmpwait(volatile void *ptr,
{
unsigned long tmp;
u32 *__ptr32b;
ulong __s, __val, __mask;
asm goto(ALTERNATIVE("j %l[no_zawrs]", "nop",
0, RISCV_ISA_EXT_ZAWRS, 1)
: : : : no_zawrs);
switch (size) {
case 1:
fallthrough;
__ptr32b = (u32 *)((ulong)(ptr) & ~0x3);
__s = ((ulong)(ptr) & 0x3) * BITS_PER_BYTE;
__val = val << __s;
__mask = 0xff << __s;
asm volatile(
" lr.w %0, %1\n"
" and %0, %0, %3\n"
" xor %0, %0, %2\n"
" bnez %0, 1f\n"
ZAWRS_WRS_NTO "\n"
"1:"
: "=&r" (tmp), "+A" (*(__ptr32b))
: "r" (__val), "r" (__mask)
: "memory");
break;
case 2:
/* RISC-V doesn't have lr instructions on byte and half-word. */
goto no_zawrs;
__ptr32b = (u32 *)((ulong)(ptr) & ~0x3);
__s = ((ulong)(ptr) & 0x2) * BITS_PER_BYTE;
__val = val << __s;
__mask = 0xffff << __s;
asm volatile(
" lr.w %0, %1\n"
" and %0, %0, %3\n"
" xor %0, %0, %2\n"
" bnez %0, 1f\n"
ZAWRS_WRS_NTO "\n"
"1:"
: "=&r" (tmp), "+A" (*(__ptr32b))
: "r" (__val), "r" (__mask)
: "memory");
break;
case 4:
asm volatile(
" lr.w %0, %1\n"

7
arch/riscv/include/asm/cpufeature.h
View File

@ -56,6 +56,9 @@ void __init riscv_user_isa_enable(void);
#define __RISCV_ISA_EXT_BUNDLE(_name, _bundled_exts) \
_RISCV_ISA_EXT_DATA(_name, RISCV_ISA_EXT_INVALID, _bundled_exts, \
ARRAY_SIZE(_bundled_exts), NULL)
#define __RISCV_ISA_EXT_BUNDLE_VALIDATE(_name, _bundled_exts, _validate) \
_RISCV_ISA_EXT_DATA(_name, RISCV_ISA_EXT_INVALID, _bundled_exts, \
ARRAY_SIZE(_bundled_exts), _validate)
/* Used to declare extensions that are a superset of other extensions (Zvbb for instance) */
#define __RISCV_ISA_EXT_SUPERSET(_name, _id, _sub_exts) \
@ -63,7 +66,7 @@ void __init riscv_user_isa_enable(void);
#define __RISCV_ISA_EXT_SUPERSET_VALIDATE(_name, _id, _sub_exts, _validate) \
_RISCV_ISA_EXT_DATA(_name, _id, _sub_exts, ARRAY_SIZE(_sub_exts), _validate)
bool check_unaligned_access_emulated_all_cpus(void);
bool __init check_unaligned_access_emulated_all_cpus(void);
#if defined(CONFIG_RISCV_SCALAR_MISALIGNED)
void check_unaligned_access_emulated(struct work_struct *work __always_unused);
void unaligned_emulation_finish(void);
@ -76,7 +79,7 @@ static inline bool unaligned_ctl_available(void)
}
#endif
bool check_vector_unaligned_access_emulated_all_cpus(void);
bool __init check_vector_unaligned_access_emulated_all_cpus(void);
#if defined(CONFIG_RISCV_VECTOR_MISALIGNED)
void check_vector_unaligned_access_emulated(struct work_struct *work __always_unused);
DECLARE_PER_CPU(long, vector_misaligned_access);

7
arch/riscv/include/asm/ftrace.h
View File

@ -79,7 +79,6 @@ struct dyn_arch_ftrace {
#define AUIPC_RA (0x00000097)
#define JALR_T0 (0x000282e7)
#define AUIPC_T0 (0x00000297)
#define NOP4 (0x00000013)
#define to_jalr_t0(offset) \
(((offset & JALR_OFFSET_MASK) << JALR_SHIFT) | JALR_T0)
@ -92,7 +91,7 @@ struct dyn_arch_ftrace {
#define make_call_t0(caller, callee, call) \
do { \
unsigned int offset = \
(unsigned long) callee - (unsigned long) caller; \
(unsigned long) (callee) - (unsigned long) (caller); \
call[0] = to_auipc_t0(offset); \
call[1] = to_jalr_t0(offset); \
} while (0)
@ -108,7 +107,7 @@ do { \
#define make_call_ra(caller, callee, call) \
do { \
unsigned int offset = \
(unsigned long) callee - (unsigned long) caller; \
(unsigned long) (callee) - (unsigned long) (caller); \
call[0] = to_auipc_ra(offset); \
call[1] = to_jalr_ra(offset); \
} while (0)
@ -207,7 +206,7 @@ ftrace_partial_regs(const struct ftrace_regs *fregs, struct pt_regs *regs)
{
struct __arch_ftrace_regs *afregs = arch_ftrace_regs(fregs);
memcpy(&regs->a0, afregs->args, sizeof(afregs->args));
memcpy(&regs->a_regs, afregs->args, sizeof(afregs->args));
regs->epc = afregs->epc;
regs->ra = afregs->ra;
regs->sp = afregs->sp;

5
arch/riscv/include/asm/hwcap.h
View File

@ -100,6 +100,11 @@
#define RISCV_ISA_EXT_ZICCRSE 91
#define RISCV_ISA_EXT_SVADE 92
#define RISCV_ISA_EXT_SVADU 93
#define RISCV_ISA_EXT_ZFBFMIN 94
#define RISCV_ISA_EXT_ZVFBFMIN 95
#define RISCV_ISA_EXT_ZVFBFWMA 96
#define RISCV_ISA_EXT_ZAAMO 97
#define RISCV_ISA_EXT_ZALRSC 98
#define RISCV_ISA_EXT_XLINUXENVCFG 127

2
arch/riscv/include/asm/hwprobe.h
View File

@ -8,7 +8,7 @@
#include <uapi/asm/hwprobe.h>
#define RISCV_HWPROBE_MAX_KEY 11
#define RISCV_HWPROBE_MAX_KEY 12
static inline bool riscv_hwprobe_key_is_valid(__s64 key)
{

3
arch/riscv/include/asm/insn-def.h
View File

@ -199,5 +199,8 @@
#define RISCV_PAUSE ".4byte 0x100000f"
#define ZAWRS_WRS_NTO ".4byte 0x00d00073"
#define ZAWRS_WRS_STO ".4byte 0x01d00073"
#define RISCV_NOP4 ".4byte 0x00000013"
#define RISCV_INSN_NOP4 _AC(0x00000013, U)
#endif /* __ASM_INSN_DEF_H */

27
arch/riscv/include/asm/page.h
View File

@ -24,21 +24,22 @@
* When not using MMU this corresponds to the first free page in
* physical memory (aligned on a page boundary).
*/
#ifdef CONFIG_64BIT
#ifdef CONFIG_MMU
#define PAGE_OFFSET kernel_map.page_offset
#else
#define PAGE_OFFSET _AC(CONFIG_PAGE_OFFSET, UL)
#endif
/*
* By default, CONFIG_PAGE_OFFSET value corresponds to SV57 address space so
* define the PAGE_OFFSET value for SV48 and SV39.
*/
#ifdef CONFIG_64BIT
#define PAGE_OFFSET_L5 _AC(0xff60000000000000, UL)
#define PAGE_OFFSET_L4 _AC(0xffffaf8000000000, UL)
#define PAGE_OFFSET_L3 _AC(0xffffffd600000000, UL)
#ifdef CONFIG_XIP_KERNEL
#define PAGE_OFFSET PAGE_OFFSET_L3
#else
#define PAGE_OFFSET _AC(CONFIG_PAGE_OFFSET, UL)
#define PAGE_OFFSET kernel_map.page_offset
#endif /* CONFIG_XIP_KERNEL */
#else
#define PAGE_OFFSET _AC(0xc0000000, UL)
#endif /* CONFIG_64BIT */
#else
#define PAGE_OFFSET ((unsigned long)phys_ram_base)
#endif /* CONFIG_MMU */
#ifndef __ASSEMBLY__
@ -95,14 +96,9 @@ typedef struct page *pgtable_t;
#define MIN_MEMBLOCK_ADDR 0
#endif
#ifdef CONFIG_MMU
#define ARCH_PFN_OFFSET (PFN_DOWN((unsigned long)phys_ram_base))
#else
#define ARCH_PFN_OFFSET (PAGE_OFFSET >> PAGE_SHIFT)
#endif /* CONFIG_MMU */
struct kernel_mapping {
unsigned long page_offset;
unsigned long virt_addr;
unsigned long virt_offset;
uintptr_t phys_addr;
@ -116,6 +112,7 @@ struct kernel_mapping {
uintptr_t xiprom;
uintptr_t xiprom_sz;
#else
unsigned long page_offset;
unsigned long va_kernel_pa_offset;
#endif
};

55
arch/riscv/include/asm/pgtable.h
View File

@ -12,7 +12,11 @@
#include <asm/pgtable-bits.h>
#ifndef CONFIG_MMU
#define KERNEL_LINK_ADDR PAGE_OFFSET
#ifdef CONFIG_RELOCATABLE
#define KERNEL_LINK_ADDR UL(0)
#else
#define KERNEL_LINK_ADDR _AC(CONFIG_PHYS_RAM_BASE, UL)
#endif
#define KERN_VIRT_SIZE (UL(-1))
#else
@ -341,6 +345,14 @@ static inline pte_t pfn_pte(unsigned long pfn, pgprot_t prot)
#define mk_pte(page, prot) pfn_pte(page_to_pfn(page), prot)
#define pte_pgprot pte_pgprot
static inline pgprot_t pte_pgprot(pte_t pte)
{
unsigned long pfn = pte_pfn(pte);
return __pgprot(pte_val(pfn_pte(pfn, __pgprot(0))) ^ pte_val(pte));
}
static inline int pte_present(pte_t pte)
{
return (pte_val(pte) & (_PAGE_PRESENT | _PAGE_PROT_NONE));
@ -674,6 +686,11 @@ static inline pmd_t pte_pmd(pte_t pte)
return __pmd(pte_val(pte));
}
static inline pud_t pte_pud(pte_t pte)
{
return __pud(pte_val(pte));
}
static inline pmd_t pmd_mkhuge(pmd_t pmd)
{
return pmd;
@ -699,6 +716,18 @@ static inline unsigned long pud_pfn(pud_t pud)
return ((__pud_to_phys(pud) & PUD_MASK) >> PAGE_SHIFT);
}
#define pmd_pgprot pmd_pgprot
static inline pgprot_t pmd_pgprot(pmd_t pmd)
{
return pte_pgprot(pmd_pte(pmd));
}
#define pud_pgprot pud_pgprot
static inline pgprot_t pud_pgprot(pud_t pud)
{
return pte_pgprot(pud_pte(pud));
}
static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
{
return pte_pmd(pte_modify(pmd_pte(pmd), newprot));
@ -768,6 +797,30 @@ static inline pmd_t pmd_mkdevmap(pmd_t pmd)
return pte_pmd(pte_mkdevmap(pmd_pte(pmd)));
}
#ifdef CONFIG_ARCH_SUPPORTS_PMD_PFNMAP
static inline bool pmd_special(pmd_t pmd)
{
return pte_special(pmd_pte(pmd));
}
static inline pmd_t pmd_mkspecial(pmd_t pmd)
{
return pte_pmd(pte_mkspecial(pmd_pte(pmd)));
}
#endif
#ifdef CONFIG_ARCH_SUPPORTS_PUD_PFNMAP
static inline bool pud_special(pud_t pud)
{
return pte_special(pud_pte(pud));
}
static inline pud_t pud_mkspecial(pud_t pud)
{
return pte_pud(pte_mkspecial(pud_pte(pud)));
}
#endif
static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr,
pmd_t *pmdp, pmd_t pmd)
{

18
arch/riscv/include/asm/ptrace.h
View File

@ -23,14 +23,16 @@ struct pt_regs {
unsigned long t2;
unsigned long s0;
unsigned long s1;
unsigned long a0;
unsigned long a1;
unsigned long a2;
unsigned long a3;
unsigned long a4;
unsigned long a5;
unsigned long a6;
unsigned long a7;
struct_group(a_regs,
unsigned long a0;
unsigned long a1;
unsigned long a2;
unsigned long a3;
unsigned long a4;
unsigned long a5;
unsigned long a6;
unsigned long a7;
);
unsigned long s2;
unsigned long s3;
unsigned long s4;

268
arch/riscv/include/asm/runtime-const.h Normal file
View File

@ -0,0 +1,268 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_RISCV_RUNTIME_CONST_H
#define _ASM_RISCV_RUNTIME_CONST_H
#include <asm/asm.h>
#include <asm/alternative.h>
#include <asm/cacheflush.h>
#include <asm/insn-def.h>
#include <linux/memory.h>
#include <asm/text-patching.h>
#include <linux/uaccess.h>
#ifdef CONFIG_32BIT
#define runtime_const_ptr(sym) \
({ \
typeof(sym) __ret; \
asm_inline(".option push\n\t" \
".option norvc\n\t" \
"1:\t" \
"lui %[__ret],0x89abd\n\t" \
"addi %[__ret],%[__ret],-0x211\n\t" \
".option pop\n\t" \
".pushsection runtime_ptr_" #sym ",\"a\"\n\t" \
".long 1b - .\n\t" \
".popsection" \
: [__ret] "=r" (__ret)); \
__ret; \
})
#else
/*
* Loading 64-bit constants into a register from immediates is a non-trivial
* task on riscv64. To get it somewhat performant, load 32 bits into two
* different registers and then combine the results.
*
* If the processor supports the Zbkb extension, we can combine the final
* "slli,slli,srli,add" into the single "pack" instruction. If the processor
* doesn't support Zbkb but does support the Zbb extension, we can
* combine the final "slli,srli,add" into one instruction "add.uw".
*/
#define RISCV_RUNTIME_CONST_64_PREAMBLE \
".option push\n\t" \
".option norvc\n\t" \
"1:\t" \
"lui %[__ret],0x89abd\n\t" \
"lui %[__tmp],0x1234\n\t" \
"addiw %[__ret],%[__ret],-0x211\n\t" \
"addiw %[__tmp],%[__tmp],0x567\n\t" \
#define RISCV_RUNTIME_CONST_64_BASE \
"slli %[__tmp],%[__tmp],32\n\t" \
"slli %[__ret],%[__ret],32\n\t" \
"srli %[__ret],%[__ret],32\n\t" \
"add %[__ret],%[__ret],%[__tmp]\n\t" \
#define RISCV_RUNTIME_CONST_64_ZBA \
".option push\n\t" \
".option arch,+zba\n\t" \
".option norvc\n\t" \
"slli %[__tmp],%[__tmp],32\n\t" \
"add.uw %[__ret],%[__ret],%[__tmp]\n\t" \
"nop\n\t" \
"nop\n\t" \
".option pop\n\t" \
#define RISCV_RUNTIME_CONST_64_ZBKB \
".option push\n\t" \
".option arch,+zbkb\n\t" \
".option norvc\n\t" \
"pack %[__ret],%[__ret],%[__tmp]\n\t" \
"nop\n\t" \
"nop\n\t" \
"nop\n\t" \
".option pop\n\t" \
#define RISCV_RUNTIME_CONST_64_POSTAMBLE(sym) \
".option pop\n\t" \
".pushsection runtime_ptr_" #sym ",\"a\"\n\t" \
".long 1b - .\n\t" \
".popsection" \
#if defined(CONFIG_RISCV_ISA_ZBA) && defined(CONFIG_TOOLCHAIN_HAS_ZBA) \
&& defined(CONFIG_RISCV_ISA_ZBKB)
#define runtime_const_ptr(sym) \
({ \
typeof(sym) __ret, __tmp; \
asm_inline(RISCV_RUNTIME_CONST_64_PREAMBLE \
ALTERNATIVE_2( \
RISCV_RUNTIME_CONST_64_BASE, \
RISCV_RUNTIME_CONST_64_ZBA, \
0, RISCV_ISA_EXT_ZBA, 1, \
RISCV_RUNTIME_CONST_64_ZBKB, \
0, RISCV_ISA_EXT_ZBKB, 1 \
) \
RISCV_RUNTIME_CONST_64_POSTAMBLE(sym) \
: [__ret] "=r" (__ret), [__tmp] "=r" (__tmp)); \
__ret; \
})
#elif defined(CONFIG_RISCV_ISA_ZBA) && defined(CONFIG_TOOLCHAIN_HAS_ZBA)
#define runtime_const_ptr(sym) \
({ \
typeof(sym) __ret, __tmp; \
asm_inline(RISCV_RUNTIME_CONST_64_PREAMBLE \
ALTERNATIVE( \
RISCV_RUNTIME_CONST_64_BASE, \
RISCV_RUNTIME_CONST_64_ZBA, \
0, RISCV_ISA_EXT_ZBA, 1 \
) \
RISCV_RUNTIME_CONST_64_POSTAMBLE(sym) \
: [__ret] "=r" (__ret), [__tmp] "=r" (__tmp)); \
__ret; \
})
#elif defined(CONFIG_RISCV_ISA_ZBKB)
#define runtime_const_ptr(sym) \
({ \
typeof(sym) __ret, __tmp; \
asm_inline(RISCV_RUNTIME_CONST_64_PREAMBLE \
ALTERNATIVE( \
RISCV_RUNTIME_CONST_64_BASE, \
RISCV_RUNTIME_CONST_64_ZBKB, \
0, RISCV_ISA_EXT_ZBKB, 1 \
) \
RISCV_RUNTIME_CONST_64_POSTAMBLE(sym) \
: [__ret] "=r" (__ret), [__tmp] "=r" (__tmp)); \
__ret; \
})
#else
#define runtime_const_ptr(sym) \
({ \
typeof(sym) __ret, __tmp; \
asm_inline(RISCV_RUNTIME_CONST_64_PREAMBLE \
RISCV_RUNTIME_CONST_64_BASE \
RISCV_RUNTIME_CONST_64_POSTAMBLE(sym) \
: [__ret] "=r" (__ret), [__tmp] "=r" (__tmp)); \
__ret; \
})
#endif
#endif
#define runtime_const_shift_right_32(val, sym) \
({ \
u32 __ret; \
asm_inline(".option push\n\t" \
".option norvc\n\t" \
"1:\t" \
SRLI " %[__ret],%[__val],12\n\t" \
".option pop\n\t" \
".pushsection runtime_shift_" #sym ",\"a\"\n\t" \
".long 1b - .\n\t" \
".popsection" \
: [__ret] "=r" (__ret) \
: [__val] "r" (val)); \
__ret; \
})
#define runtime_const_init(type, sym) do { \
extern s32 __start_runtime_##type##_##sym[]; \
extern s32 __stop_runtime_##type##_##sym[]; \
\
runtime_const_fixup(__runtime_fixup_##type, \
(unsigned long)(sym), \
__start_runtime_##type##_##sym, \
__stop_runtime_##type##_##sym); \
} while (0)
static inline void __runtime_fixup_caches(void *where, unsigned int insns)
{
/* On riscv there are currently only cache-wide flushes so va is ignored. */
__always_unused uintptr_t va = (uintptr_t)where;
flush_icache_range(va, va + 4 * insns);
}
/*
* The 32-bit immediate is stored in a lui+addi pairing.
* lui holds the upper 20 bits of the immediate in the first 20 bits of the instruction.
* addi holds the lower 12 bits of the immediate in the first 12 bits of the instruction.
*/
static inline void __runtime_fixup_32(__le16 *lui_parcel, __le16 *addi_parcel, unsigned int val)
{
unsigned int lower_immediate, upper_immediate;
u32 lui_insn, addi_insn, addi_insn_mask;
__le32 lui_res, addi_res;
/* Mask out upper 12 bit of addi */
addi_insn_mask = 0x000fffff;
lui_insn = (u32)le16_to_cpu(lui_parcel[0]) | (u32)le16_to_cpu(lui_parcel[1]) << 16;
addi_insn = (u32)le16_to_cpu(addi_parcel[0]) | (u32)le16_to_cpu(addi_parcel[1]) << 16;
lower_immediate = sign_extend32(val, 11);
upper_immediate = (val - lower_immediate);
if (upper_immediate & 0xfffff000) {
/* replace upper 20 bits of lui with upper immediate */
lui_insn &= 0x00000fff;
lui_insn |= upper_immediate & 0xfffff000;
} else {
/* replace lui with nop if immediate is small enough to fit in addi */
lui_insn = RISCV_INSN_NOP4;
/*
* lui is being skipped, so do a load instead of an add. A load
* is performed by adding with the x0 register. Setting rs to
* zero with the following mask will accomplish this goal.
*/
addi_insn_mask &= 0x07fff;
}
if (lower_immediate & 0x00000fff) {
/* replace upper 12 bits of addi with lower 12 bits of val */
addi_insn &= addi_insn_mask;
addi_insn |= (lower_immediate & 0x00000fff) << 20;
} else {
/* replace addi with nop if lower_immediate is empty */
addi_insn = RISCV_INSN_NOP4;
}
addi_res = cpu_to_le32(addi_insn);
lui_res = cpu_to_le32(lui_insn);
mutex_lock(&text_mutex);
patch_insn_write(addi_parcel, &addi_res, sizeof(addi_res));
patch_insn_write(lui_parcel, &lui_res, sizeof(lui_res));
mutex_unlock(&text_mutex);
}
static inline void __runtime_fixup_ptr(void *where, unsigned long val)
{
#ifdef CONFIG_32BIT
__runtime_fixup_32(where, where + 4, val);
__runtime_fixup_caches(where, 2);
#else
__runtime_fixup_32(where, where + 8, val);
__runtime_fixup_32(where + 4, where + 12, val >> 32);
__runtime_fixup_caches(where, 4);
#endif
}
/*
* Replace the least significant 5 bits of the srli/srliw immediate that is
* located at bits 20-24
*/
static inline void __runtime_fixup_shift(void *where, unsigned long val)
{
__le16 *parcel = where;
__le32 res;
u32 insn;
insn = (u32)le16_to_cpu(parcel[0]) | (u32)le16_to_cpu(parcel[1]) << 16;
insn &= 0xfe0fffff;
insn |= (val & 0b11111) << 20;
res = cpu_to_le32(insn);
mutex_lock(&text_mutex);
patch_text_nosync(where, &res, sizeof(insn));
mutex_unlock(&text_mutex);
}
static inline void runtime_const_fixup(void (*fn)(void *, unsigned long),
unsigned long val, s32 *start, s32 *end)
{
while (start < end) {
fn(*start + (void *)start, val);
start++;
}
}
#endif /* _ASM_RISCV_RUNTIME_CONST_H */

4
arch/riscv/include/asm/suspend.h
View File

@ -18,6 +18,10 @@ struct suspend_context {
unsigned long ie;
#ifdef CONFIG_MMU
unsigned long satp;
unsigned long stimecmp;
#if __riscv_xlen < 64
unsigned long stimecmph;
#endif
#endif
};

9
arch/riscv/include/uapi/asm/hwprobe.h
View File

@ -73,6 +73,14 @@ struct riscv_hwprobe {
#define RISCV_HWPROBE_EXT_ZCMOP (1ULL << 47)
#define RISCV_HWPROBE_EXT_ZAWRS (1ULL << 48)
#define RISCV_HWPROBE_EXT_SUPM (1ULL << 49)
#define RISCV_HWPROBE_EXT_ZICNTR (1ULL << 50)
#define RISCV_HWPROBE_EXT_ZIHPM (1ULL << 51)
#define RISCV_HWPROBE_EXT_ZFBFMIN (1ULL << 52)
#define RISCV_HWPROBE_EXT_ZVFBFMIN (1ULL << 53)
#define RISCV_HWPROBE_EXT_ZVFBFWMA (1ULL << 54)
#define RISCV_HWPROBE_EXT_ZICBOM (1ULL << 55)
#define RISCV_HWPROBE_EXT_ZAAMO (1ULL << 56)
#define RISCV_HWPROBE_EXT_ZALRSC (1ULL << 57)
#define RISCV_HWPROBE_KEY_CPUPERF_0 5
#define RISCV_HWPROBE_MISALIGNED_UNKNOWN (0 << 0)
#define RISCV_HWPROBE_MISALIGNED_EMULATED (1 << 0)
@ -95,6 +103,7 @@ struct riscv_hwprobe {
#define RISCV_HWPROBE_MISALIGNED_VECTOR_FAST 3
#define RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED 4
#define RISCV_HWPROBE_KEY_VENDOR_EXT_THEAD_0 11
#define RISCV_HWPROBE_KEY_ZICBOM_BLOCK_SIZE 12
/* Increase RISCV_HWPROBE_MAX_KEY when adding items. */
/* Flags */

2
arch/riscv/include/uapi/asm/kvm.h
View File

@ -182,6 +182,8 @@ enum KVM_RISCV_ISA_EXT_ID {
KVM_RISCV_ISA_EXT_SVVPTC,
KVM_RISCV_ISA_EXT_ZABHA,
KVM_RISCV_ISA_EXT_ZICCRSE,
KVM_RISCV_ISA_EXT_ZAAMO,
KVM_RISCV_ISA_EXT_ZALRSC,
KVM_RISCV_ISA_EXT_MAX,
};

1
arch/riscv/kernel/asm-offsets.c
View File

@ -36,7 +36,6 @@ void asm_offsets(void)
OFFSET(TASK_THREAD_S11, task_struct, thread.s[11]);
OFFSET(TASK_TI_CPU, task_struct, thread_info.cpu);
OFFSET(TASK_TI_FLAGS, task_struct, thread_info.flags);
OFFSET(TASK_TI_PREEMPT_COUNT, task_struct, thread_info.preempt_count);
OFFSET(TASK_TI_KERNEL_SP, task_struct, thread_info.kernel_sp);
OFFSET(TASK_TI_USER_SP, task_struct, thread_info.user_sp);

197
arch/riscv/kernel/cpufeature.c
View File

@ -32,6 +32,7 @@
#define NUM_ALPHA_EXTS ('z' - 'a' + 1)
static bool any_cpu_has_zicboz;
static bool any_cpu_has_zicbom;
unsigned long elf_hwcap __read_mostly;
@ -53,9 +54,7 @@ u32 thead_vlenb_of;
*/
unsigned long riscv_isa_extension_base(const unsigned long *isa_bitmap)
{
if (!isa_bitmap)
return riscv_isa[0];
return isa_bitmap[0];
return !isa_bitmap ? riscv_isa[0] : isa_bitmap[0];
}
EXPORT_SYMBOL_GPL(riscv_isa_extension_base);
@ -76,10 +75,19 @@ bool __riscv_isa_extension_available(const unsigned long *isa_bitmap, unsigned i
if (bit >= RISCV_ISA_EXT_MAX)
return false;
return test_bit(bit, bmap) ? true : false;
return test_bit(bit, bmap);
}
EXPORT_SYMBOL_GPL(__riscv_isa_extension_available);
static int riscv_ext_f_depends(const struct riscv_isa_ext_data *data,
const unsigned long *isa_bitmap)
{
if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_f))
return 0;
return -EPROBE_DEFER;
}
static int riscv_ext_zicbom_validate(const struct riscv_isa_ext_data *data,
const unsigned long *isa_bitmap)
{
@ -91,6 +99,8 @@ static int riscv_ext_zicbom_validate(const struct riscv_isa_ext_data *data,
pr_err("Zicbom disabled as cbom-block-size present, but is not a power-of-2\n");
return -EINVAL;
}
any_cpu_has_zicbom = true;
return 0;
}
@ -109,6 +119,82 @@ static int riscv_ext_zicboz_validate(const struct riscv_isa_ext_data *data,
return 0;
}
static int riscv_ext_f_validate(const struct riscv_isa_ext_data *data,
const unsigned long *isa_bitmap)
{
if (!IS_ENABLED(CONFIG_FPU))
return -EINVAL;
/*
* Due to extension ordering, d is checked before f, so no deferral
* is required.
*/
if (!__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_d)) {
pr_warn_once("This kernel does not support systems with F but not D\n");
return -EINVAL;
}
return 0;
}
static int riscv_ext_d_validate(const struct riscv_isa_ext_data *data,
const unsigned long *isa_bitmap)
{
if (!IS_ENABLED(CONFIG_FPU))
return -EINVAL;
return 0;
}
static int riscv_ext_vector_x_validate(const struct riscv_isa_ext_data *data,
const unsigned long *isa_bitmap)
{
if (!IS_ENABLED(CONFIG_RISCV_ISA_V))
return -EINVAL;
return 0;
}
static int riscv_ext_vector_float_validate(const struct riscv_isa_ext_data *data,
const unsigned long *isa_bitmap)
{
if (!IS_ENABLED(CONFIG_RISCV_ISA_V))
return -EINVAL;
if (!IS_ENABLED(CONFIG_FPU))
return -EINVAL;
/*
* The kernel doesn't support systems that don't implement both of
* F and D, so if any of the vector extensions that do floating point
* are to be usable, both floating point extensions need to be usable.
*
* Since this function validates vector only, and v/Zve* are probed
* after f/d, there's no need for a deferral here.
*/
if (!__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_d))
return -EINVAL;
return 0;
}
static int riscv_ext_vector_crypto_validate(const struct riscv_isa_ext_data *data,
const unsigned long *isa_bitmap)
{
if (!IS_ENABLED(CONFIG_RISCV_ISA_V))
return -EINVAL;
/*
* It isn't the kernel's job to check that the binding is correct, so
* it should be enough to check that any of the vector extensions are
* enabled, which in-turn means that vector is usable in this kernel
*/
if (!__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZVE32X))
return -EPROBE_DEFER;
return 0;
}
static int riscv_ext_zca_depends(const struct riscv_isa_ext_data *data,
const unsigned long *isa_bitmap)
{
@ -140,6 +226,28 @@ static int riscv_ext_zcf_validate(const struct riscv_isa_ext_data *data,
return -EPROBE_DEFER;
}
static int riscv_vector_f_validate(const struct riscv_isa_ext_data *data,
const unsigned long *isa_bitmap)
{
if (!IS_ENABLED(CONFIG_RISCV_ISA_V))
return -EINVAL;
if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZVE32F))
return 0;
return -EPROBE_DEFER;
}
static int riscv_ext_zvfbfwma_validate(const struct riscv_isa_ext_data *data,
const unsigned long *isa_bitmap)
{
if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZFBFMIN) &&
__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZVFBFMIN))
return 0;
return -EPROBE_DEFER;
}
static int riscv_ext_svadu_validate(const struct riscv_isa_ext_data *data,
const unsigned long *isa_bitmap)
{
@ -150,6 +258,11 @@ static int riscv_ext_svadu_validate(const struct riscv_isa_ext_data *data,
return 0;
}
static const unsigned int riscv_a_exts[] = {
RISCV_ISA_EXT_ZAAMO,
RISCV_ISA_EXT_ZALRSC,
};
static const unsigned int riscv_zk_bundled_exts[] = {
RISCV_ISA_EXT_ZBKB,
RISCV_ISA_EXT_ZBKC,
@ -321,17 +434,15 @@ static const unsigned int riscv_c_exts[] = {
const struct riscv_isa_ext_data riscv_isa_ext[] = {
__RISCV_ISA_EXT_DATA(i, RISCV_ISA_EXT_i),
__RISCV_ISA_EXT_DATA(m, RISCV_ISA_EXT_m),
__RISCV_ISA_EXT_DATA(a, RISCV_ISA_EXT_a),
__RISCV_ISA_EXT_DATA(f, RISCV_ISA_EXT_f),
__RISCV_ISA_EXT_DATA(d, RISCV_ISA_EXT_d),
__RISCV_ISA_EXT_SUPERSET(a, RISCV_ISA_EXT_a, riscv_a_exts),
__RISCV_ISA_EXT_DATA_VALIDATE(f, RISCV_ISA_EXT_f, riscv_ext_f_validate),
__RISCV_ISA_EXT_DATA_VALIDATE(d, RISCV_ISA_EXT_d, riscv_ext_d_validate),
__RISCV_ISA_EXT_DATA(q, RISCV_ISA_EXT_q),
__RISCV_ISA_EXT_SUPERSET(c, RISCV_ISA_EXT_c, riscv_c_exts),
__RISCV_ISA_EXT_SUPERSET(v, RISCV_ISA_EXT_v, riscv_v_exts),
__RISCV_ISA_EXT_SUPERSET_VALIDATE(v, RISCV_ISA_EXT_v, riscv_v_exts, riscv_ext_vector_float_validate),
__RISCV_ISA_EXT_DATA(h, RISCV_ISA_EXT_h),
__RISCV_ISA_EXT_SUPERSET_VALIDATE(zicbom, RISCV_ISA_EXT_ZICBOM, riscv_xlinuxenvcfg_exts,
riscv_ext_zicbom_validate),
__RISCV_ISA_EXT_SUPERSET_VALIDATE(zicboz, RISCV_ISA_EXT_ZICBOZ, riscv_xlinuxenvcfg_exts,
riscv_ext_zicboz_validate),
__RISCV_ISA_EXT_SUPERSET_VALIDATE(zicbom, RISCV_ISA_EXT_ZICBOM, riscv_xlinuxenvcfg_exts, riscv_ext_zicbom_validate),
__RISCV_ISA_EXT_SUPERSET_VALIDATE(zicboz, RISCV_ISA_EXT_ZICBOZ, riscv_xlinuxenvcfg_exts, riscv_ext_zicboz_validate),
__RISCV_ISA_EXT_DATA(ziccrse, RISCV_ISA_EXT_ZICCRSE),
__RISCV_ISA_EXT_DATA(zicntr, RISCV_ISA_EXT_ZICNTR),
__RISCV_ISA_EXT_DATA(zicond, RISCV_ISA_EXT_ZICOND),
@ -341,10 +452,13 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = {
__RISCV_ISA_EXT_DATA(zihintpause, RISCV_ISA_EXT_ZIHINTPAUSE),
__RISCV_ISA_EXT_DATA(zihpm, RISCV_ISA_EXT_ZIHPM),
__RISCV_ISA_EXT_DATA(zimop, RISCV_ISA_EXT_ZIMOP),
__RISCV_ISA_EXT_DATA(zaamo, RISCV_ISA_EXT_ZAAMO),
__RISCV_ISA_EXT_DATA(zabha, RISCV_ISA_EXT_ZABHA),
__RISCV_ISA_EXT_DATA(zacas, RISCV_ISA_EXT_ZACAS),
__RISCV_ISA_EXT_DATA(zalrsc, RISCV_ISA_EXT_ZALRSC),
__RISCV_ISA_EXT_DATA(zawrs, RISCV_ISA_EXT_ZAWRS),
__RISCV_ISA_EXT_DATA(zfa, RISCV_ISA_EXT_ZFA),
__RISCV_ISA_EXT_DATA_VALIDATE(zfbfmin, RISCV_ISA_EXT_ZFBFMIN, riscv_ext_f_depends),
__RISCV_ISA_EXT_DATA(zfh, RISCV_ISA_EXT_ZFH),
__RISCV_ISA_EXT_DATA(zfhmin, RISCV_ISA_EXT_ZFHMIN),
__RISCV_ISA_EXT_DATA(zca, RISCV_ISA_EXT_ZCA),
@ -370,29 +484,31 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = {
__RISCV_ISA_EXT_DATA(zksed, RISCV_ISA_EXT_ZKSED),
__RISCV_ISA_EXT_DATA(zksh, RISCV_ISA_EXT_ZKSH),
__RISCV_ISA_EXT_DATA(ztso, RISCV_ISA_EXT_ZTSO),
__RISCV_ISA_EXT_SUPERSET(zvbb, RISCV_ISA_EXT_ZVBB, riscv_zvbb_exts),
__RISCV_ISA_EXT_DATA(zvbc, RISCV_ISA_EXT_ZVBC),
__RISCV_ISA_EXT_SUPERSET(zve32f, RISCV_ISA_EXT_ZVE32F, riscv_zve32f_exts),
__RISCV_ISA_EXT_DATA(zve32x, RISCV_ISA_EXT_ZVE32X),
__RISCV_ISA_EXT_SUPERSET(zve64d, RISCV_ISA_EXT_ZVE64D, riscv_zve64d_exts),
__RISCV_ISA_EXT_SUPERSET(zve64f, RISCV_ISA_EXT_ZVE64F, riscv_zve64f_exts),
__RISCV_ISA_EXT_SUPERSET(zve64x, RISCV_ISA_EXT_ZVE64X, riscv_zve64x_exts),
__RISCV_ISA_EXT_SUPERSET_VALIDATE(zvbb, RISCV_ISA_EXT_ZVBB, riscv_zvbb_exts, riscv_ext_vector_crypto_validate),
__RISCV_ISA_EXT_DATA_VALIDATE(zvbc, RISCV_ISA_EXT_ZVBC, riscv_ext_vector_crypto_validate),
__RISCV_ISA_EXT_SUPERSET_VALIDATE(zve32f, RISCV_ISA_EXT_ZVE32F, riscv_zve32f_exts, riscv_ext_vector_float_validate),
__RISCV_ISA_EXT_DATA_VALIDATE(zve32x, RISCV_ISA_EXT_ZVE32X, riscv_ext_vector_x_validate),
__RISCV_ISA_EXT_SUPERSET_VALIDATE(zve64d, RISCV_ISA_EXT_ZVE64D, riscv_zve64d_exts, riscv_ext_vector_float_validate),
__RISCV_ISA_EXT_SUPERSET_VALIDATE(zve64f, RISCV_ISA_EXT_ZVE64F, riscv_zve64f_exts, riscv_ext_vector_float_validate),
__RISCV_ISA_EXT_SUPERSET_VALIDATE(zve64x, RISCV_ISA_EXT_ZVE64X, riscv_zve64x_exts, riscv_ext_vector_x_validate),
__RISCV_ISA_EXT_DATA_VALIDATE(zvfbfmin, RISCV_ISA_EXT_ZVFBFMIN, riscv_vector_f_validate),
__RISCV_ISA_EXT_DATA_VALIDATE(zvfbfwma, RISCV_ISA_EXT_ZVFBFWMA, riscv_ext_zvfbfwma_validate),
__RISCV_ISA_EXT_DATA(zvfh, RISCV_ISA_EXT_ZVFH),
__RISCV_ISA_EXT_DATA(zvfhmin, RISCV_ISA_EXT_ZVFHMIN),
__RISCV_ISA_EXT_DATA(zvkb, RISCV_ISA_EXT_ZVKB),
__RISCV_ISA_EXT_DATA(zvkg, RISCV_ISA_EXT_ZVKG),
__RISCV_ISA_EXT_BUNDLE(zvkn, riscv_zvkn_bundled_exts),
__RISCV_ISA_EXT_BUNDLE(zvknc, riscv_zvknc_bundled_exts),
__RISCV_ISA_EXT_DATA(zvkned, RISCV_ISA_EXT_ZVKNED),
__RISCV_ISA_EXT_BUNDLE(zvkng, riscv_zvkng_bundled_exts),
__RISCV_ISA_EXT_DATA(zvknha, RISCV_ISA_EXT_ZVKNHA),
__RISCV_ISA_EXT_DATA(zvknhb, RISCV_ISA_EXT_ZVKNHB),
__RISCV_ISA_EXT_BUNDLE(zvks, riscv_zvks_bundled_exts),
__RISCV_ISA_EXT_BUNDLE(zvksc, riscv_zvksc_bundled_exts),
__RISCV_ISA_EXT_DATA(zvksed, RISCV_ISA_EXT_ZVKSED),
__RISCV_ISA_EXT_DATA(zvksh, RISCV_ISA_EXT_ZVKSH),
__RISCV_ISA_EXT_BUNDLE(zvksg, riscv_zvksg_bundled_exts),
__RISCV_ISA_EXT_DATA(zvkt, RISCV_ISA_EXT_ZVKT),
__RISCV_ISA_EXT_DATA_VALIDATE(zvkb, RISCV_ISA_EXT_ZVKB, riscv_ext_vector_crypto_validate),
__RISCV_ISA_EXT_DATA_VALIDATE(zvkg, RISCV_ISA_EXT_ZVKG, riscv_ext_vector_crypto_validate),
__RISCV_ISA_EXT_BUNDLE_VALIDATE(zvkn, riscv_zvkn_bundled_exts, riscv_ext_vector_crypto_validate),
__RISCV_ISA_EXT_BUNDLE_VALIDATE(zvknc, riscv_zvknc_bundled_exts, riscv_ext_vector_crypto_validate),
__RISCV_ISA_EXT_DATA_VALIDATE(zvkned, RISCV_ISA_EXT_ZVKNED, riscv_ext_vector_crypto_validate),
__RISCV_ISA_EXT_BUNDLE_VALIDATE(zvkng, riscv_zvkng_bundled_exts, riscv_ext_vector_crypto_validate),
__RISCV_ISA_EXT_DATA_VALIDATE(zvknha, RISCV_ISA_EXT_ZVKNHA, riscv_ext_vector_crypto_validate),
__RISCV_ISA_EXT_DATA_VALIDATE(zvknhb, RISCV_ISA_EXT_ZVKNHB, riscv_ext_vector_crypto_validate),
__RISCV_ISA_EXT_BUNDLE_VALIDATE(zvks, riscv_zvks_bundled_exts, riscv_ext_vector_crypto_validate),
__RISCV_ISA_EXT_BUNDLE_VALIDATE(zvksc, riscv_zvksc_bundled_exts, riscv_ext_vector_crypto_validate),
__RISCV_ISA_EXT_DATA_VALIDATE(zvksed, RISCV_ISA_EXT_ZVKSED, riscv_ext_vector_crypto_validate),
__RISCV_ISA_EXT_DATA_VALIDATE(zvksh, RISCV_ISA_EXT_ZVKSH, riscv_ext_vector_crypto_validate),
__RISCV_ISA_EXT_BUNDLE_VALIDATE(zvksg, riscv_zvksg_bundled_exts, riscv_ext_vector_crypto_validate),
__RISCV_ISA_EXT_DATA_VALIDATE(zvkt, RISCV_ISA_EXT_ZVKT, riscv_ext_vector_crypto_validate),
__RISCV_ISA_EXT_DATA(smaia, RISCV_ISA_EXT_SMAIA),
__RISCV_ISA_EXT_DATA(smmpm, RISCV_ISA_EXT_SMMPM),
__RISCV_ISA_EXT_SUPERSET(smnpm, RISCV_ISA_EXT_SMNPM, riscv_xlinuxenvcfg_exts),
@ -960,16 +1076,6 @@ void __init riscv_fill_hwcap(void)
riscv_v_setup_vsize();
}
if (elf_hwcap & COMPAT_HWCAP_ISA_V) {
/*
* ISA string in device tree might have 'v' flag, but
* CONFIG_RISCV_ISA_V is disabled in kernel.
* Clear V flag in elf_hwcap if CONFIG_RISCV_ISA_V is disabled.
*/
if (!IS_ENABLED(CONFIG_RISCV_ISA_V))
elf_hwcap &= ~COMPAT_HWCAP_ISA_V;
}
memset(print_str, 0, sizeof(print_str));
for (i = 0, j = 0; i < NUM_ALPHA_EXTS; i++)
if (riscv_isa[0] & BIT_MASK(i))
@ -1001,6 +1107,11 @@ void __init riscv_user_isa_enable(void)
current->thread.envcfg |= ENVCFG_CBZE;
else if (any_cpu_has_zicboz)
pr_warn("Zicboz disabled as it is unavailable on some harts\n");
if (riscv_has_extension_unlikely(RISCV_ISA_EXT_ZICBOM))
current->thread.envcfg |= ENVCFG_CBCFE;
else if (any_cpu_has_zicbom)
pr_warn("Zicbom disabled as it is unavailable on some harts\n");
}
#ifdef CONFIG_RISCV_ALTERNATIVE

3
arch/riscv/kernel/elf_kexec.c
View File

@ -468,6 +468,9 @@ int arch_kexec_apply_relocations_add(struct purgatory_info *pi,
case R_RISCV_ALIGN:
case R_RISCV_RELAX:
break;
case R_RISCV_64:
*(u64 *)loc = val;
break;
default:
pr_err("Unknown rela relocation: %d\n", r_type);
return -ENOEXEC;

6
arch/riscv/kernel/ftrace.c
View File

@ -36,7 +36,7 @@ static int ftrace_check_current_call(unsigned long hook_pos,
unsigned int *expected)
{
unsigned int replaced[2];
unsigned int nops[2] = {NOP4, NOP4};
unsigned int nops[2] = {RISCV_INSN_NOP4, RISCV_INSN_NOP4};
/* we expect nops at the hook position */
if (!expected)
@ -68,7 +68,7 @@ static int __ftrace_modify_call(unsigned long hook_pos, unsigned long target,
bool enable, bool ra)
{
unsigned int call[2];
unsigned int nops[2] = {NOP4, NOP4};
unsigned int nops[2] = {RISCV_INSN_NOP4, RISCV_INSN_NOP4};
if (ra)
make_call_ra(hook_pos, target, call);
@ -97,7 +97,7 @@ int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec,
unsigned long addr)
{
unsigned int nops[2] = {NOP4, NOP4};
unsigned int nops[2] = {RISCV_INSN_NOP4, RISCV_INSN_NOP4};
if (patch_insn_write((void *)rec->ip, nops, MCOUNT_INSN_SIZE))
return -EPERM;

4
arch/riscv/kernel/jump_label.c
View File

@ -11,8 +11,8 @@
#include <asm/bug.h>
#include <asm/cacheflush.h>
#include <asm/text-patching.h>
#include <asm/insn-def.h>
#define RISCV_INSN_NOP 0x00000013U
#define RISCV_INSN_JAL 0x0000006fU
bool arch_jump_label_transform_queue(struct jump_entry *entry,
@ -33,7 +33,7 @@ bool arch_jump_label_transform_queue(struct jump_entry *entry,
(((u32)offset & GENMASK(10, 1)) << (21 - 1)) |
(((u32)offset & GENMASK(20, 20)) << (31 - 20));
} else {
insn = RISCV_INSN_NOP;
insn = RISCV_INSN_NOP4;
}
if (early_boot_irqs_disabled) {

24
arch/riscv/kernel/mcount.S
View File

@ -12,8 +12,6 @@
#include <asm/asm-offsets.h>
#include <asm/ftrace.h>
#define ABI_SIZE_ON_STACK 80
.text
.macro SAVE_ABI_STATE
@ -28,12 +26,12 @@
* register if a0 was not saved.
*/
.macro SAVE_RET_ABI_STATE
addi sp, sp, -ABI_SIZE_ON_STACK
REG_S ra, 1*SZREG(sp)
REG_S s0, 8*SZREG(sp)
REG_S a0, 10*SZREG(sp)
REG_S a1, 11*SZREG(sp)
addi s0, sp, ABI_SIZE_ON_STACK
addi sp, sp, -FREGS_SIZE_ON_STACK
REG_S ra, FREGS_RA(sp)
REG_S s0, FREGS_S0(sp)
REG_S a0, FREGS_A0(sp)
REG_S a1, FREGS_A1(sp)
addi s0, sp, FREGS_SIZE_ON_STACK
.endm
.macro RESTORE_ABI_STATE
@ -43,11 +41,11 @@
.endm
.macro RESTORE_RET_ABI_STATE
REG_L ra, 1*SZREG(sp)
REG_L s0, 8*SZREG(sp)
REG_L a0, 10*SZREG(sp)
REG_L a1, 11*SZREG(sp)
addi sp, sp, ABI_SIZE_ON_STACK
REG_L ra, FREGS_RA(sp)
REG_L s0, FREGS_S0(sp)
REG_L a0, FREGS_A0(sp)
REG_L a1, FREGS_A1(sp)
addi sp, sp, FREGS_SIZE_ON_STACK
.endm
SYM_TYPED_FUNC_START(ftrace_stub)

5
arch/riscv/kernel/setup.c
View File

@ -235,11 +235,6 @@ static void __init parse_dtb(void)
} else {
pr_err("No DTB passed to the kernel\n");
}
#ifdef CONFIG_CMDLINE_FORCE
strscpy(boot_command_line, CONFIG_CMDLINE, COMMAND_LINE_SIZE);
pr_info("Forcing kernel command line to: %s\n", boot_command_line);
#endif
}
#if defined(CONFIG_RISCV_COMBO_SPINLOCKS)

2
arch/riscv/kernel/smp.c
View File

@ -48,6 +48,8 @@ EXPORT_SYMBOL_GPL(__cpuid_to_hartid_map);
void __init smp_setup_processor_id(void)
{
cpuid_to_hartid_map(0) = boot_cpu_hartid;
pr_info("Booting Linux on hartid %lu\n", boot_cpu_hartid);
}
static DEFINE_PER_CPU_READ_MOSTLY(int, ipi_dummy_dev);

4
arch/riscv/kernel/smpboot.c
View File

@ -231,6 +231,10 @@ asmlinkage __visible void smp_callin(void)
riscv_ipi_enable();
numa_add_cpu(curr_cpuid);
pr_debug("CPU%u: Booted secondary hartid %lu\n", curr_cpuid,
cpuid_to_hartid_map(curr_cpuid));
set_cpu_online(curr_cpuid, true);
/*

2
arch/riscv/kernel/stacktrace.c
View File

@ -74,7 +74,7 @@ void notrace walk_stackframe(struct task_struct *task, struct pt_regs *regs,
&frame->ra);
if (pc >= (unsigned long)handle_exception &&
pc < (unsigned long)&ret_from_exception_end) {
if (unlikely(!__kernel_text_address(pc) || !fn(arg, pc)))
if (unlikely(!fn(arg, pc)))
break;
pc = ((struct pt_regs *)sp)->epc;

14
arch/riscv/kernel/suspend.c
View File

@ -30,6 +30,13 @@ void suspend_save_csrs(struct suspend_context *context)
*/
#ifdef CONFIG_MMU
if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SSTC)) {
context->stimecmp = csr_read(CSR_STIMECMP);
#if __riscv_xlen < 64
context->stimecmph = csr_read(CSR_STIMECMPH);
#endif
}
context->satp = csr_read(CSR_SATP);
#endif
}
@ -43,6 +50,13 @@ void suspend_restore_csrs(struct suspend_context *context)
csr_write(CSR_IE, context->ie);
#ifdef CONFIG_MMU
if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SSTC)) {
csr_write(CSR_STIMECMP, context->stimecmp);
#if __riscv_xlen < 64
csr_write(CSR_STIMECMPH, context->stimecmph);
#endif
}
csr_write(CSR_SATP, context->satp);
#endif
}

15
arch/riscv/kernel/sys_hwprobe.c
View File

@ -95,7 +95,9 @@ static void hwprobe_isa_ext0(struct riscv_hwprobe *pair,
* regardless of the kernel's configuration, as no other checks, besides
* presence in the hart_isa bitmap, are made.
*/
EXT_KEY(ZAAMO);
EXT_KEY(ZACAS);
EXT_KEY(ZALRSC);
EXT_KEY(ZAWRS);
EXT_KEY(ZBA);
EXT_KEY(ZBB);
@ -107,10 +109,13 @@ static void hwprobe_isa_ext0(struct riscv_hwprobe *pair,
EXT_KEY(ZCA);
EXT_KEY(ZCB);
EXT_KEY(ZCMOP);
EXT_KEY(ZICBOM);
EXT_KEY(ZICBOZ);
EXT_KEY(ZICNTR);
EXT_KEY(ZICOND);
EXT_KEY(ZIHINTNTL);
EXT_KEY(ZIHINTPAUSE);
EXT_KEY(ZIHPM);
EXT_KEY(ZIMOP);
EXT_KEY(ZKND);
EXT_KEY(ZKNE);
@ -132,6 +137,8 @@ static void hwprobe_isa_ext0(struct riscv_hwprobe *pair,
EXT_KEY(ZVE64D);
EXT_KEY(ZVE64F);
EXT_KEY(ZVE64X);
EXT_KEY(ZVFBFMIN);
EXT_KEY(ZVFBFWMA);
EXT_KEY(ZVFH);
EXT_KEY(ZVFHMIN);
EXT_KEY(ZVKB);
@ -148,6 +155,7 @@ static void hwprobe_isa_ext0(struct riscv_hwprobe *pair,
EXT_KEY(ZCD);
EXT_KEY(ZCF);
EXT_KEY(ZFA);
EXT_KEY(ZFBFMIN);
EXT_KEY(ZFH);
EXT_KEY(ZFHMIN);
}
@ -161,7 +169,7 @@ static void hwprobe_isa_ext0(struct riscv_hwprobe *pair,
pair->value &= ~missing;
}
static bool hwprobe_ext0_has(const struct cpumask *cpus, unsigned long ext)
static bool hwprobe_ext0_has(const struct cpumask *cpus, u64 ext)
{
struct riscv_hwprobe pair;
@ -279,6 +287,11 @@ static void hwprobe_one_pair(struct riscv_hwprobe *pair,
if (hwprobe_ext0_has(cpus, RISCV_HWPROBE_EXT_ZICBOZ))
pair->value = riscv_cboz_block_size;
break;
case RISCV_HWPROBE_KEY_ZICBOM_BLOCK_SIZE:
pair->value = 0;
if (hwprobe_ext0_has(cpus, RISCV_HWPROBE_EXT_ZICBOM))
pair->value = riscv_cbom_block_size;
break;
case RISCV_HWPROBE_KEY_HIGHEST_VIRT_ADDRESS:
pair->value = user_max_virt_addr();
break;

14
arch/riscv/kernel/traps_misaligned.c
View File

@ -605,16 +605,10 @@ void check_vector_unaligned_access_emulated(struct work_struct *work __always_un
kernel_vector_end();
}
bool check_vector_unaligned_access_emulated_all_cpus(void)
bool __init check_vector_unaligned_access_emulated_all_cpus(void)
{
int cpu;
if (!has_vector()) {
for_each_online_cpu(cpu)
per_cpu(vector_misaligned_access, cpu) = RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED;
return false;
}
schedule_on_each_cpu(check_vector_unaligned_access_emulated);
for_each_online_cpu(cpu)
@ -625,7 +619,7 @@ bool check_vector_unaligned_access_emulated_all_cpus(void)
return true;
}
#else
bool check_vector_unaligned_access_emulated_all_cpus(void)
bool __init check_vector_unaligned_access_emulated_all_cpus(void)
{
return false;
}
@ -659,7 +653,7 @@ void check_unaligned_access_emulated(struct work_struct *work __always_unused)
}
}
bool check_unaligned_access_emulated_all_cpus(void)
bool __init check_unaligned_access_emulated_all_cpus(void)
{
int cpu;
@ -684,7 +678,7 @@ bool unaligned_ctl_available(void)
return unaligned_ctl;
}
#else
bool check_unaligned_access_emulated_all_cpus(void)
bool __init check_unaligned_access_emulated_all_cpus(void)
{
return false;
}

254
arch/riscv/kernel/unaligned_access_speed.c
View File

@ -24,8 +24,12 @@
DEFINE_PER_CPU(long, misaligned_access_speed) = RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN;
DEFINE_PER_CPU(long, vector_misaligned_access) = RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED;
#ifdef CONFIG_RISCV_PROBE_UNALIGNED_ACCESS
static long unaligned_scalar_speed_param = RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN;
static long unaligned_vector_speed_param = RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN;
static cpumask_t fast_misaligned_access;
#ifdef CONFIG_RISCV_PROBE_UNALIGNED_ACCESS
static int check_unaligned_access(void *param)
{
int cpu = smp_processor_id();
@ -121,7 +125,7 @@ static int check_unaligned_access(void *param)
return 0;
}
static void check_unaligned_access_nonboot_cpu(void *param)
static void __init check_unaligned_access_nonboot_cpu(void *param)
{
unsigned int cpu = smp_processor_id();
struct page **pages = param;
@ -130,6 +134,50 @@ static void check_unaligned_access_nonboot_cpu(void *param)
check_unaligned_access(pages[cpu]);
}
/* Measure unaligned access speed on all CPUs present at boot in parallel. */
static void __init check_unaligned_access_speed_all_cpus(void)
{
unsigned int cpu;
unsigned int cpu_count = num_possible_cpus();
struct page **bufs = kcalloc(cpu_count, sizeof(*bufs), GFP_KERNEL);
if (!bufs) {
pr_warn("Allocation failure, not measuring misaligned performance\n");
return;
}
/*
* Allocate separate buffers for each CPU so there's no fighting over
* cache lines.
*/
for_each_cpu(cpu, cpu_online_mask) {
bufs[cpu] = alloc_pages(GFP_KERNEL, MISALIGNED_BUFFER_ORDER);
if (!bufs[cpu]) {
pr_warn("Allocation failure, not measuring misaligned performance\n");
goto out;
}
}
/* Check everybody except 0, who stays behind to tend jiffies. */
on_each_cpu(check_unaligned_access_nonboot_cpu, bufs, 1);
/* Check core 0. */
smp_call_on_cpu(0, check_unaligned_access, bufs[0], true);
out:
for_each_cpu(cpu, cpu_online_mask) {
if (bufs[cpu])
__free_pages(bufs[cpu], MISALIGNED_BUFFER_ORDER);
}
kfree(bufs);
}
#else /* CONFIG_RISCV_PROBE_UNALIGNED_ACCESS */
static void __init check_unaligned_access_speed_all_cpus(void)
{
}
#endif
DEFINE_STATIC_KEY_FALSE(fast_unaligned_access_speed_key);
static void modify_unaligned_access_branches(cpumask_t *mask, int weight)
@ -175,7 +223,7 @@ static void set_unaligned_access_static_branches(void)
modify_unaligned_access_branches(&fast_and_online, num_online_cpus());
}
static int lock_and_set_unaligned_access_static_branch(void)
static int __init lock_and_set_unaligned_access_static_branch(void)
{
cpus_read_lock();
set_unaligned_access_static_branches();
@ -188,21 +236,29 @@ arch_initcall_sync(lock_and_set_unaligned_access_static_branch);
static int riscv_online_cpu(unsigned int cpu)
{
static struct page *buf;
/* We are already set since the last check */
if (per_cpu(misaligned_access_speed, cpu) != RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN)
if (per_cpu(misaligned_access_speed, cpu) != RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN) {
goto exit;
} else if (unaligned_scalar_speed_param != RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN) {
per_cpu(misaligned_access_speed, cpu) = unaligned_scalar_speed_param;
goto exit;
check_unaligned_access_emulated(NULL);
buf = alloc_pages(GFP_KERNEL, MISALIGNED_BUFFER_ORDER);
if (!buf) {
pr_warn("Allocation failure, not measuring misaligned performance\n");
return -ENOMEM;
}
check_unaligned_access(buf);
__free_pages(buf, MISALIGNED_BUFFER_ORDER);
#ifdef CONFIG_RISCV_PROBE_UNALIGNED_ACCESS
{
static struct page *buf;
check_unaligned_access_emulated(NULL);
buf = alloc_pages(GFP_KERNEL, MISALIGNED_BUFFER_ORDER);
if (!buf) {
pr_warn("Allocation failure, not measuring misaligned performance\n");
return -ENOMEM;
}
check_unaligned_access(buf);
__free_pages(buf, MISALIGNED_BUFFER_ORDER);
}
#endif
exit:
set_unaligned_access_static_branches();
@ -217,59 +273,6 @@ static int riscv_offline_cpu(unsigned int cpu)
return 0;
}
/* Measure unaligned access speed on all CPUs present at boot in parallel. */
static int check_unaligned_access_speed_all_cpus(void)
{
unsigned int cpu;
unsigned int cpu_count = num_possible_cpus();
struct page **bufs = kcalloc(cpu_count, sizeof(*bufs), GFP_KERNEL);
if (!bufs) {
pr_warn("Allocation failure, not measuring misaligned performance\n");
return 0;
}
/*
* Allocate separate buffers for each CPU so there's no fighting over
* cache lines.
*/
for_each_cpu(cpu, cpu_online_mask) {
bufs[cpu] = alloc_pages(GFP_KERNEL, MISALIGNED_BUFFER_ORDER);
if (!bufs[cpu]) {
pr_warn("Allocation failure, not measuring misaligned performance\n");
goto out;
}
}
/* Check everybody except 0, who stays behind to tend jiffies. */
on_each_cpu(check_unaligned_access_nonboot_cpu, bufs, 1);
/* Check core 0. */
smp_call_on_cpu(0, check_unaligned_access, bufs[0], true);
/*
* Setup hotplug callbacks for any new CPUs that come online or go
* offline.
*/
cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "riscv:online",
riscv_online_cpu, riscv_offline_cpu);
out:
for_each_cpu(cpu, cpu_online_mask) {
if (bufs[cpu])
__free_pages(bufs[cpu], MISALIGNED_BUFFER_ORDER);
}
kfree(bufs);
return 0;
}
#else /* CONFIG_RISCV_PROBE_UNALIGNED_ACCESS */
static int check_unaligned_access_speed_all_cpus(void)
{
return 0;
}
#endif
#ifdef CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS
static void check_vector_unaligned_access(struct work_struct *work __always_unused)
{
@ -349,7 +352,7 @@ static void check_vector_unaligned_access(struct work_struct *work __always_unus
pr_warn("cpu%d: rdtime lacks granularity needed to measure unaligned vector access speed\n",
cpu);
return;
goto free;
}
if (word_cycles < byte_cycles)
@ -363,59 +366,114 @@ static void check_vector_unaligned_access(struct work_struct *work __always_unus
(speed == RISCV_HWPROBE_MISALIGNED_VECTOR_FAST) ? "fast" : "slow");
per_cpu(vector_misaligned_access, cpu) = speed;
free:
__free_pages(page, MISALIGNED_BUFFER_ORDER);
}
/* Measure unaligned access speed on all CPUs present at boot in parallel. */
static int __init vec_check_unaligned_access_speed_all_cpus(void *unused __always_unused)
{
schedule_on_each_cpu(check_vector_unaligned_access);
return 0;
}
#else /* CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS */
static int __init vec_check_unaligned_access_speed_all_cpus(void *unused __always_unused)
{
return 0;
}
#endif
static int riscv_online_cpu_vec(unsigned int cpu)
{
if (!has_vector())
if (unaligned_vector_speed_param != RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN) {
per_cpu(vector_misaligned_access, cpu) = unaligned_vector_speed_param;
return 0;
}
if (per_cpu(vector_misaligned_access, cpu) != RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED)
#ifdef CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS
if (per_cpu(vector_misaligned_access, cpu) != RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN)
return 0;
check_vector_unaligned_access_emulated(NULL);
check_vector_unaligned_access(NULL);
#endif
return 0;
}
/* Measure unaligned access speed on all CPUs present at boot in parallel. */
static int vec_check_unaligned_access_speed_all_cpus(void *unused __always_unused)
static const char * const speed_str[] __initconst = { NULL, NULL, "slow", "fast", "unsupported" };
static int __init set_unaligned_scalar_speed_param(char *str)
{
schedule_on_each_cpu(check_vector_unaligned_access);
if (!strcmp(str, speed_str[RISCV_HWPROBE_MISALIGNED_SCALAR_SLOW]))
unaligned_scalar_speed_param = RISCV_HWPROBE_MISALIGNED_SCALAR_SLOW;
else if (!strcmp(str, speed_str[RISCV_HWPROBE_MISALIGNED_SCALAR_FAST]))
unaligned_scalar_speed_param = RISCV_HWPROBE_MISALIGNED_SCALAR_FAST;
else if (!strcmp(str, speed_str[RISCV_HWPROBE_MISALIGNED_SCALAR_UNSUPPORTED]))
unaligned_scalar_speed_param = RISCV_HWPROBE_MISALIGNED_SCALAR_UNSUPPORTED;
else
return -EINVAL;
return 1;
}
__setup("unaligned_scalar_speed=", set_unaligned_scalar_speed_param);
static int __init set_unaligned_vector_speed_param(char *str)
{
if (!strcmp(str, speed_str[RISCV_HWPROBE_MISALIGNED_VECTOR_SLOW]))
unaligned_vector_speed_param = RISCV_HWPROBE_MISALIGNED_VECTOR_SLOW;
else if (!strcmp(str, speed_str[RISCV_HWPROBE_MISALIGNED_VECTOR_FAST]))
unaligned_vector_speed_param = RISCV_HWPROBE_MISALIGNED_VECTOR_FAST;
else if (!strcmp(str, speed_str[RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED]))
unaligned_vector_speed_param = RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED;
else
return -EINVAL;
return 1;
}
__setup("unaligned_vector_speed=", set_unaligned_vector_speed_param);
static int __init check_unaligned_access_all_cpus(void)
{
int cpu;
if (unaligned_scalar_speed_param == RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN &&
!check_unaligned_access_emulated_all_cpus()) {
check_unaligned_access_speed_all_cpus();
} else {
pr_info("scalar unaligned access speed set to '%s' by command line\n",
speed_str[unaligned_scalar_speed_param]);
for_each_online_cpu(cpu)
per_cpu(misaligned_access_speed, cpu) = unaligned_scalar_speed_param;
}
if (!has_vector())
unaligned_vector_speed_param = RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED;
if (unaligned_vector_speed_param == RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN &&
!check_vector_unaligned_access_emulated_all_cpus() &&
IS_ENABLED(CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS)) {
kthread_run(vec_check_unaligned_access_speed_all_cpus,
NULL, "vec_check_unaligned_access_speed_all_cpus");
} else {
pr_info("vector unaligned access speed set to '%s' by command line\n",
speed_str[unaligned_vector_speed_param]);
for_each_online_cpu(cpu)
per_cpu(vector_misaligned_access, cpu) = unaligned_vector_speed_param;
}
/*
* Setup hotplug callbacks for any new CPUs that come online or go
* offline.
*/
cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "riscv:online",
riscv_online_cpu, riscv_offline_cpu);
cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "riscv:online",
riscv_online_cpu_vec, NULL);
return 0;
}
#else /* CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS */
static int vec_check_unaligned_access_speed_all_cpus(void *unused __always_unused)
{
return 0;
}
#endif
static int check_unaligned_access_all_cpus(void)
{
bool all_cpus_emulated, all_cpus_vec_unsupported;
all_cpus_emulated = check_unaligned_access_emulated_all_cpus();
all_cpus_vec_unsupported = check_vector_unaligned_access_emulated_all_cpus();
if (!all_cpus_vec_unsupported &&
IS_ENABLED(CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS)) {
kthread_run(vec_check_unaligned_access_speed_all_cpus,
NULL, "vec_check_unaligned_access_speed_all_cpus");
}
if (!all_cpus_emulated)
return check_unaligned_access_speed_all_cpus();
return 0;
}
arch_initcall(check_unaligned_access_all_cpus);

2
arch/riscv/kernel/vec-copy-unaligned.S
View File

@ -11,7 +11,7 @@
#define WORD_SEW CONCATENATE(e, WORD_EEW)
#define VEC_L CONCATENATE(vle, WORD_EEW).v
#define VEC_S CONCATENATE(vle, WORD_EEW).v
#define VEC_S CONCATENATE(vse, WORD_EEW).v
/* void __riscv_copy_vec_words_unaligned(void *, const void *, size_t) */
/* Performs a memcpy without aligning buffers, using word loads and stores. */

2
arch/riscv/kernel/vendor_extensions.c
View File

@ -61,6 +61,6 @@ bool __riscv_isa_vendor_extension_available(int cpu, unsigned long vendor, unsig
if (bit >= RISCV_ISA_VENDOR_EXT_MAX)
return false;
return test_bit(bit, bmap->isa) ? true : false;
return test_bit(bit, bmap->isa);
}
EXPORT_SYMBOL_GPL(__riscv_isa_vendor_extension_available);

3
arch/riscv/kernel/vmlinux.lds.S
View File

@ -97,6 +97,9 @@ SECTIONS
{
EXIT_DATA
}
RUNTIME_CONST_VARIABLES
PERCPU_SECTION(L1_CACHE_BYTES)
.rel.dyn : {

4
arch/riscv/kvm/vcpu_onereg.c
View File

@ -47,8 +47,10 @@ static const unsigned long kvm_isa_ext_arr[] = {
KVM_ISA_EXT_ARR(SVNAPOT),
KVM_ISA_EXT_ARR(SVPBMT),
KVM_ISA_EXT_ARR(SVVPTC),
KVM_ISA_EXT_ARR(ZAAMO),
KVM_ISA_EXT_ARR(ZABHA),
KVM_ISA_EXT_ARR(ZACAS),
KVM_ISA_EXT_ARR(ZALRSC),
KVM_ISA_EXT_ARR(ZAWRS),
KVM_ISA_EXT_ARR(ZBA),
KVM_ISA_EXT_ARR(ZBB),
@ -149,8 +151,10 @@ static bool kvm_riscv_vcpu_isa_disable_allowed(unsigned long ext)
case KVM_RISCV_ISA_EXT_SVINVAL:
case KVM_RISCV_ISA_EXT_SVNAPOT:
case KVM_RISCV_ISA_EXT_SVVPTC:
case KVM_RISCV_ISA_EXT_ZAAMO:
case KVM_RISCV_ISA_EXT_ZABHA:
case KVM_RISCV_ISA_EXT_ZACAS:
case KVM_RISCV_ISA_EXT_ZALRSC:
case KVM_RISCV_ISA_EXT_ZAWRS:
case KVM_RISCV_ISA_EXT_ZBA:
case KVM_RISCV_ISA_EXT_ZBB:

21
arch/riscv/lib/csum.c
View File

@ -40,12 +40,7 @@ __sum16 csum_ipv6_magic(const struct in6_addr *saddr,
uproto = (__force unsigned int)htonl(proto);
sum += uproto;
/*
* Zbb support saves 4 instructions, so not worth checking without
* alternatives if supported
*/
if (IS_ENABLED(CONFIG_RISCV_ISA_ZBB) &&
IS_ENABLED(CONFIG_RISCV_ALTERNATIVE)) {
if (IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZBB)) {
unsigned long fold_temp;
/*
@ -157,12 +152,7 @@ do_csum_with_alignment(const unsigned char *buff, int len)
csum = do_csum_common(ptr, end, data);
#ifdef CC_HAS_ASM_GOTO_TIED_OUTPUT
/*
* Zbb support saves 6 instructions, so not worth checking without
* alternatives if supported
*/
if (IS_ENABLED(CONFIG_RISCV_ISA_ZBB) &&
IS_ENABLED(CONFIG_RISCV_ALTERNATIVE)) {
if (IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZBB)) {
unsigned long fold_temp;
/*
@ -244,12 +234,7 @@ do_csum_no_alignment(const unsigned char *buff, int len)
end = (const unsigned long *)(buff + len);
csum = do_csum_common(ptr, end, data);
/*
* Zbb support saves 6 instructions, so not worth checking without
* alternatives if supported
*/
if (IS_ENABLED(CONFIG_RISCV_ISA_ZBB) &&
IS_ENABLED(CONFIG_RISCV_ALTERNATIVE)) {
if (IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZBB)) {
unsigned long fold_temp;
/*

5
arch/riscv/lib/strcmp.S
View File

@ -8,7 +8,8 @@
/* int strcmp(const char *cs, const char *ct) */
SYM_FUNC_START(strcmp)
ALTERNATIVE("nop", "j strcmp_zbb", 0, RISCV_ISA_EXT_ZBB, CONFIG_RISCV_ISA_ZBB)
__ALTERNATIVE_CFG("nop", "j strcmp_zbb", 0, RISCV_ISA_EXT_ZBB,
IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZBB))
/*
* Returns
@ -43,7 +44,7 @@ SYM_FUNC_START(strcmp)
* The code was published as part of the bitmanip manual
* in Appendix A.
*/
#ifdef CONFIG_RISCV_ISA_ZBB
#if defined(CONFIG_RISCV_ISA_ZBB) && defined(CONFIG_TOOLCHAIN_HAS_ZBB)
strcmp_zbb:
.option push

5
arch/riscv/lib/strlen.S
View File

@ -8,7 +8,8 @@
/* int strlen(const char *s) */
SYM_FUNC_START(strlen)
ALTERNATIVE("nop", "j strlen_zbb", 0, RISCV_ISA_EXT_ZBB, CONFIG_RISCV_ISA_ZBB)
__ALTERNATIVE_CFG("nop", "j strlen_zbb", 0, RISCV_ISA_EXT_ZBB,
IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZBB))
/*
* Returns
@ -33,7 +34,7 @@ SYM_FUNC_START(strlen)
/*
* Variant of strlen using the ZBB extension if available
*/
#ifdef CONFIG_RISCV_ISA_ZBB
#if defined(CONFIG_RISCV_ISA_ZBB) && defined(CONFIG_TOOLCHAIN_HAS_ZBB)
strlen_zbb:
#ifdef CONFIG_CPU_BIG_ENDIAN

5
arch/riscv/lib/strncmp.S
View File

@ -8,7 +8,8 @@
/* int strncmp(const char *cs, const char *ct, size_t count) */
SYM_FUNC_START(strncmp)
ALTERNATIVE("nop", "j strncmp_zbb", 0, RISCV_ISA_EXT_ZBB, CONFIG_RISCV_ISA_ZBB)
__ALTERNATIVE_CFG("nop", "j strncmp_zbb", 0, RISCV_ISA_EXT_ZBB,
IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZBB))
/*
* Returns
@ -46,7 +47,7 @@ SYM_FUNC_START(strncmp)
/*
* Variant of strncmp using the ZBB extension if available
*/
#ifdef CONFIG_RISCV_ISA_ZBB
#if defined(CONFIG_RISCV_ISA_ZBB) && defined(CONFIG_TOOLCHAIN_HAS_ZBB)
strncmp_zbb:
.option push

2
arch/riscv/mm/context.c
View File

@ -158,7 +158,7 @@ static void set_mm_asid(struct mm_struct *mm, unsigned int cpu)
*
* - We get a zero back from the cmpxchg and end up waiting on the
* lock. Taking the lock synchronises with the rollover and so
* we are forced to see the updated verion.
* we are forced to see the updated version.
*
* - We get a valid context back from the cmpxchg then we continue
* using old ASID because __flush_context() would have marked ASID

74
arch/riscv/mm/hugetlbpage.c
View File

@ -148,22 +148,25 @@ unsigned long hugetlb_mask_last_page(struct hstate *h)
static pte_t get_clear_contig(struct mm_struct *mm,
unsigned long addr,
pte_t *ptep,
unsigned long pte_num)
unsigned long ncontig)
{
pte_t orig_pte = ptep_get(ptep);
unsigned long i;
pte_t pte, tmp_pte;
bool present;
for (i = 0; i < pte_num; i++, addr += PAGE_SIZE, ptep++) {
pte_t pte = ptep_get_and_clear(mm, addr, ptep);
if (pte_dirty(pte))
orig_pte = pte_mkdirty(orig_pte);
if (pte_young(pte))
orig_pte = pte_mkyoung(orig_pte);
pte = ptep_get_and_clear(mm, addr, ptep);
present = pte_present(pte);
while (--ncontig) {
ptep++;
addr += PAGE_SIZE;
tmp_pte = ptep_get_and_clear(mm, addr, ptep);
if (present) {
if (pte_dirty(tmp_pte))
pte = pte_mkdirty(pte);
if (pte_young(tmp_pte))
pte = pte_mkyoung(pte);
}
}
return orig_pte;
return pte;
}
static pte_t get_clear_contig_flush(struct mm_struct *mm,
@ -212,6 +215,26 @@ static void clear_flush(struct mm_struct *mm,
flush_tlb_range(&vma, saddr, addr);
}
static int num_contig_ptes_from_size(unsigned long sz, size_t *pgsize)
{
unsigned long hugepage_shift;
if (sz >= PGDIR_SIZE)
hugepage_shift = PGDIR_SHIFT;
else if (sz >= P4D_SIZE)
hugepage_shift = P4D_SHIFT;
else if (sz >= PUD_SIZE)
hugepage_shift = PUD_SHIFT;
else if (sz >= PMD_SIZE)
hugepage_shift = PMD_SHIFT;
else
hugepage_shift = PAGE_SHIFT;
*pgsize = 1 << hugepage_shift;
return sz >> hugepage_shift;
}
/*
* When dealing with NAPOT mappings, the privileged specification indicates that
* "if an update needs to be made, the OS generally should first mark all of the
@ -226,22 +249,10 @@ void set_huge_pte_at(struct mm_struct *mm,
pte_t pte,
unsigned long sz)
{
unsigned long hugepage_shift, pgsize;
size_t pgsize;
int i, pte_num;
if (sz >= PGDIR_SIZE)
hugepage_shift = PGDIR_SHIFT;
else if (sz >= P4D_SIZE)
hugepage_shift = P4D_SHIFT;
else if (sz >= PUD_SIZE)
hugepage_shift = PUD_SHIFT;
else if (sz >= PMD_SIZE)
hugepage_shift = PMD_SHIFT;
else
hugepage_shift = PAGE_SHIFT;
pte_num = sz >> hugepage_shift;
pgsize = 1 << hugepage_shift;
pte_num = num_contig_ptes_from_size(sz, &pgsize);
if (!pte_present(pte)) {
for (i = 0; i < pte_num; i++, ptep++, addr += pgsize)
@ -295,13 +306,14 @@ pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
unsigned long addr,
pte_t *ptep, unsigned long sz)
{
size_t pgsize;
pte_t orig_pte = ptep_get(ptep);
int pte_num;
if (!pte_napot(orig_pte))
return ptep_get_and_clear(mm, addr, ptep);
pte_num = napot_pte_num(napot_cont_order(orig_pte));
pte_num = num_contig_ptes_from_size(sz, &pgsize);
return get_clear_contig(mm, addr, ptep, pte_num);
}
@ -351,6 +363,7 @@ void huge_pte_clear(struct mm_struct *mm,
pte_t *ptep,
unsigned long sz)
{
size_t pgsize;
pte_t pte = ptep_get(ptep);
int i, pte_num;
@ -359,8 +372,9 @@ void huge_pte_clear(struct mm_struct *mm,
return;
}
pte_num = napot_pte_num(napot_cont_order(pte));
for (i = 0; i < pte_num; i++, addr += PAGE_SIZE, ptep++)
pte_num = num_contig_ptes_from_size(sz, &pgsize);
for (i = 0; i < pte_num; i++, addr += pgsize, ptep++)
pte_clear(mm, addr, ptep);
}

97
arch/riscv/mm/init.c
View File

@ -20,15 +20,13 @@
#include <linux/dma-map-ops.h>
#include <linux/crash_dump.h>
#include <linux/hugetlb.h>
#ifdef CONFIG_RELOCATABLE
#include <linux/elf.h>
#endif
#include <linux/kfence.h>
#include <linux/execmem.h>
#include <asm/fixmap.h>
#include <asm/io.h>
#include <asm/kasan.h>
#include <asm/module.h>
#include <asm/numa.h>
#include <asm/pgtable.h>
#include <asm/sections.h>
@ -320,6 +318,44 @@ static void __init setup_bootmem(void)
hugetlb_cma_reserve(PUD_SHIFT - PAGE_SHIFT);
}
#ifdef CONFIG_RELOCATABLE
extern unsigned long __rela_dyn_start, __rela_dyn_end;
static void __init relocate_kernel(void)
{
Elf_Rela *rela = (Elf_Rela *)&__rela_dyn_start;
/*
* This holds the offset between the linked virtual address and the
* relocated virtual address.
*/
uintptr_t reloc_offset = kernel_map.virt_addr - KERNEL_LINK_ADDR;
/*
* This holds the offset between kernel linked virtual address and
* physical address.
*/
uintptr_t va_kernel_link_pa_offset = KERNEL_LINK_ADDR - kernel_map.phys_addr;
for ( ; rela < (Elf_Rela *)&__rela_dyn_end; rela++) {
Elf_Addr addr = (rela->r_offset - va_kernel_link_pa_offset);
Elf_Addr relocated_addr = rela->r_addend;
if (rela->r_info != R_RISCV_RELATIVE)
continue;
/*
* Make sure to not relocate vdso symbols like rt_sigreturn
* which are linked from the address 0 in vmlinux since
* vdso symbol addresses are actually used as an offset from
* mm->context.vdso in VDSO_OFFSET macro.
*/
if (relocated_addr >= KERNEL_LINK_ADDR)
relocated_addr += reloc_offset;
*(Elf_Addr *)addr = relocated_addr;
}
}
#endif /* CONFIG_RELOCATABLE */
#ifdef CONFIG_MMU
struct pt_alloc_ops pt_ops __meminitdata;
@ -820,6 +856,8 @@ static __init void set_satp_mode(uintptr_t dtb_pa)
uintptr_t set_satp_mode_pmd = ((unsigned long)set_satp_mode) & PMD_MASK;
u64 satp_mode_cmdline = __pi_set_satp_mode_from_cmdline(dtb_pa);
kernel_map.page_offset = PAGE_OFFSET_L5;
if (satp_mode_cmdline == SATP_MODE_57) {
disable_pgtable_l5();
} else if (satp_mode_cmdline == SATP_MODE_48) {
@ -890,44 +928,6 @@ retry:
#error "setup_vm() is called from head.S before relocate so it should not use absolute addressing."
#endif
#ifdef CONFIG_RELOCATABLE
extern unsigned long __rela_dyn_start, __rela_dyn_end;
static void __init relocate_kernel(void)
{
Elf64_Rela *rela = (Elf64_Rela *)&__rela_dyn_start;
/*
* This holds the offset between the linked virtual address and the
* relocated virtual address.
*/
uintptr_t reloc_offset = kernel_map.virt_addr - KERNEL_LINK_ADDR;
/*
* This holds the offset between kernel linked virtual address and
* physical address.
*/
uintptr_t va_kernel_link_pa_offset = KERNEL_LINK_ADDR - kernel_map.phys_addr;
for ( ; rela < (Elf64_Rela *)&__rela_dyn_end; rela++) {
Elf64_Addr addr = (rela->r_offset - va_kernel_link_pa_offset);
Elf64_Addr relocated_addr = rela->r_addend;
if (rela->r_info != R_RISCV_RELATIVE)
continue;
/*
* Make sure to not relocate vdso symbols like rt_sigreturn
* which are linked from the address 0 in vmlinux since
* vdso symbol addresses are actually used as an offset from
* mm->context.vdso in VDSO_OFFSET macro.
*/
if (relocated_addr >= KERNEL_LINK_ADDR)
relocated_addr += reloc_offset;
*(Elf64_Addr *)addr = relocated_addr;
}
}
#endif /* CONFIG_RELOCATABLE */
#ifdef CONFIG_XIP_KERNEL
static void __init create_kernel_page_table(pgd_t *pgdir,
__always_unused bool early)
@ -1105,11 +1105,6 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)
kernel_map.virt_addr = KERNEL_LINK_ADDR + kernel_map.virt_offset;
#ifdef CONFIG_XIP_KERNEL
#ifdef CONFIG_64BIT
kernel_map.page_offset = PAGE_OFFSET_L3;
#else
kernel_map.page_offset = _AC(CONFIG_PAGE_OFFSET, UL);
#endif
kernel_map.xiprom = (uintptr_t)CONFIG_XIP_PHYS_ADDR;
kernel_map.xiprom_sz = (uintptr_t)(&_exiprom) - (uintptr_t)(&_xiprom);
@ -1124,7 +1119,6 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)
kernel_map.va_kernel_xip_data_pa_offset = kernel_map.virt_addr - kernel_map.phys_addr
+ (uintptr_t)&_sdata - (uintptr_t)&_start;
#else
kernel_map.page_offset = _AC(CONFIG_PAGE_OFFSET, UL);
kernel_map.phys_addr = (uintptr_t)(&_start);
kernel_map.size = (uintptr_t)(&_end) - kernel_map.phys_addr;
kernel_map.va_kernel_pa_offset = kernel_map.virt_addr - kernel_map.phys_addr;
@ -1171,7 +1165,8 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)
* makes the kernel cross over a PUD_SIZE boundary, raise a bug
* since a part of the kernel would not get mapped.
*/
BUG_ON(PUD_SIZE - (kernel_map.virt_addr & (PUD_SIZE - 1)) < kernel_map.size);
if (IS_ENABLED(CONFIG_64BIT))
BUG_ON(PUD_SIZE - (kernel_map.virt_addr & (PUD_SIZE - 1)) < kernel_map.size);
relocate_kernel();
#endif
@ -1375,6 +1370,12 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)
{
dtb_early_va = (void *)dtb_pa;
dtb_early_pa = dtb_pa;
#ifdef CONFIG_RELOCATABLE
kernel_map.virt_addr = (uintptr_t)_start;
kernel_map.phys_addr = (uintptr_t)_start;
relocate_kernel();
#endif
}
static inline void setup_vm_final(void)

2
arch/riscv/mm/physaddr.c
View File

@ -12,7 +12,7 @@ phys_addr_t __virt_to_phys(unsigned long x)
* Boundary checking aginst the kernel linear mapping space.
*/
WARN(!is_linear_mapping(x) && !is_kernel_mapping(x),
"virt_to_phys used for non-linear address: %pK (%pS)\n",
"virt_to_phys used for non-linear address: %p (%pS)\n",
(void *)x, (void *)x);
return __va_to_pa_nodebug(x);

35
arch/riscv/mm/tlbflush.c
View File

@ -4,6 +4,7 @@
#include <linux/smp.h>
#include <linux/sched.h>
#include <linux/hugetlb.h>
#include <linux/mmu_notifier.h>
#include <asm/sbi.h>
#include <asm/mmu_context.h>
@ -78,10 +79,17 @@ static void __ipi_flush_tlb_range_asid(void *info)
local_flush_tlb_range_asid(d->start, d->size, d->stride, d->asid);
}
static void __flush_tlb_range(const struct cpumask *cmask, unsigned long asid,
static inline unsigned long get_mm_asid(struct mm_struct *mm)
{
return mm ? cntx2asid(atomic_long_read(&mm->context.id)) : FLUSH_TLB_NO_ASID;
}
static void __flush_tlb_range(struct mm_struct *mm,
const struct cpumask *cmask,
unsigned long start, unsigned long size,
unsigned long stride)
{
unsigned long asid = get_mm_asid(mm);
unsigned int cpu;
if (cpumask_empty(cmask))
@ -105,30 +113,26 @@ static void __flush_tlb_range(const struct cpumask *cmask, unsigned long asid,
}
put_cpu();
}
static inline unsigned long get_mm_asid(struct mm_struct *mm)
{
return cntx2asid(atomic_long_read(&mm->context.id));
if (mm)
mmu_notifier_arch_invalidate_secondary_tlbs(mm, start, start + size);
}
void flush_tlb_mm(struct mm_struct *mm)
{
__flush_tlb_range(mm_cpumask(mm), get_mm_asid(mm),
0, FLUSH_TLB_MAX_SIZE, PAGE_SIZE);
__flush_tlb_range(mm, mm_cpumask(mm), 0, FLUSH_TLB_MAX_SIZE, PAGE_SIZE);
}
void flush_tlb_mm_range(struct mm_struct *mm,
unsigned long start, unsigned long end,
unsigned int page_size)
{
__flush_tlb_range(mm_cpumask(mm), get_mm_asid(mm),
start, end - start, page_size);
__flush_tlb_range(mm, mm_cpumask(mm), start, end - start, page_size);
}
void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr)
{
__flush_tlb_range(mm_cpumask(vma->vm_mm), get_mm_asid(vma->vm_mm),
__flush_tlb_range(vma->vm_mm, mm_cpumask(vma->vm_mm),
addr, PAGE_SIZE, PAGE_SIZE);
}
@ -161,13 +165,13 @@ void flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
}
}
__flush_tlb_range(mm_cpumask(vma->vm_mm), get_mm_asid(vma->vm_mm),
__flush_tlb_range(vma->vm_mm, mm_cpumask(vma->vm_mm),
start, end - start, stride_size);
}
void flush_tlb_kernel_range(unsigned long start, unsigned long end)
{
__flush_tlb_range(cpu_online_mask, FLUSH_TLB_NO_ASID,
__flush_tlb_range(NULL, cpu_online_mask,
start, end - start, PAGE_SIZE);
}
@ -175,7 +179,7 @@ void flush_tlb_kernel_range(unsigned long start, unsigned long end)
void flush_pmd_tlb_range(struct vm_area_struct *vma, unsigned long start,
unsigned long end)
{
__flush_tlb_range(mm_cpumask(vma->vm_mm), get_mm_asid(vma->vm_mm),
__flush_tlb_range(vma->vm_mm, mm_cpumask(vma->vm_mm),
start, end - start, PMD_SIZE);
}
#endif
@ -189,6 +193,7 @@ void arch_tlbbatch_add_pending(struct arch_tlbflush_unmap_batch *batch,
struct mm_struct *mm, unsigned long start, unsigned long end)
{
cpumask_or(&batch->cpumask, &batch->cpumask, mm_cpumask(mm));
mmu_notifier_arch_invalidate_secondary_tlbs(mm, start, end);
}
void arch_flush_tlb_batched_pending(struct mm_struct *mm)
@ -198,7 +203,7 @@ void arch_flush_tlb_batched_pending(struct mm_struct *mm)
void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch)
{
__flush_tlb_range(&batch->cpumask, FLUSH_TLB_NO_ASID, 0,
FLUSH_TLB_MAX_SIZE, PAGE_SIZE);
__flush_tlb_range(NULL, &batch->cpumask,
0, FLUSH_TLB_MAX_SIZE, PAGE_SIZE);
cpumask_clear(&batch->cpumask);
}

1
arch/riscv/purgatory/entry.S
View File

@ -12,6 +12,7 @@
.text
.align 2
SYM_CODE_START(purgatory_start)
lla sp, .Lstack

6
drivers/clocksource/timer-riscv.c
View File

@ -126,7 +126,13 @@ static int riscv_timer_starting_cpu(unsigned int cpu)
static int riscv_timer_dying_cpu(unsigned int cpu)
{
/*
* Stop the timer when the cpu is going to be offline otherwise
* the timer interrupt may be pending while performing power-down.
*/
riscv_clock_event_stop();
disable_percpu_irq(riscv_clock_event_irq);
return 0;
}

8
include/asm-generic/module.h
View File

@ -19,12 +19,8 @@ struct mod_arch_specific
#define Elf_Dyn Elf64_Dyn
#define Elf_Ehdr Elf64_Ehdr
#define Elf_Addr Elf64_Addr
#ifdef CONFIG_MODULES_USE_ELF_REL
#define Elf_Rel Elf64_Rel
#endif
#ifdef CONFIG_MODULES_USE_ELF_RELA
#define Elf_Rela Elf64_Rela
#endif
#define ELF_R_TYPE(X) ELF64_R_TYPE(X)
#define ELF_R_SYM(X) ELF64_R_SYM(X)
@ -36,12 +32,8 @@ struct mod_arch_specific
#define Elf_Dyn Elf32_Dyn
#define Elf_Ehdr Elf32_Ehdr
#define Elf_Addr Elf32_Addr
#ifdef CONFIG_MODULES_USE_ELF_REL
#define Elf_Rel Elf32_Rel
#endif
#ifdef CONFIG_MODULES_USE_ELF_RELA
#define Elf_Rela Elf32_Rela
#endif
#define ELF_R_TYPE(X) ELF32_R_TYPE(X)
#define ELF_R_SYM(X) ELF32_R_SYM(X)
#endif

8
tools/testing/selftests/kvm/riscv/get-reg-list.c
View File

@ -53,8 +53,10 @@ bool filter_reg(__u64 reg)
case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_SVNAPOT:
case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_SVPBMT:
case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_SVVPTC:
case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZAAMO:
case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZABHA:
case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZACAS:
case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZALRSC:
case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZAWRS:
case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZBA:
case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZBB:
@ -434,8 +436,10 @@ static const char *isa_ext_single_id_to_str(__u64 reg_off)
KVM_ISA_EXT_ARR(SVNAPOT),
KVM_ISA_EXT_ARR(SVPBMT),
KVM_ISA_EXT_ARR(SVVPTC),
KVM_ISA_EXT_ARR(ZAAMO),
KVM_ISA_EXT_ARR(ZABHA),
KVM_ISA_EXT_ARR(ZACAS),
KVM_ISA_EXT_ARR(ZALRSC),
KVM_ISA_EXT_ARR(ZAWRS),
KVM_ISA_EXT_ARR(ZBA),
KVM_ISA_EXT_ARR(ZBB),
@ -974,8 +978,10 @@ KVM_ISA_EXT_SIMPLE_CONFIG(svinval, SVINVAL);
KVM_ISA_EXT_SIMPLE_CONFIG(svnapot, SVNAPOT);
KVM_ISA_EXT_SIMPLE_CONFIG(svpbmt, SVPBMT);
KVM_ISA_EXT_SIMPLE_CONFIG(svvptc, SVVPTC);
KVM_ISA_EXT_SIMPLE_CONFIG(zaamo, ZAAMO);
KVM_ISA_EXT_SIMPLE_CONFIG(zabha, ZABHA);
KVM_ISA_EXT_SIMPLE_CONFIG(zacas, ZACAS);
KVM_ISA_EXT_SIMPLE_CONFIG(zalrsc, ZALRSC);
KVM_ISA_EXT_SIMPLE_CONFIG(zawrs, ZAWRS);
KVM_ISA_EXT_SIMPLE_CONFIG(zba, ZBA);
KVM_ISA_EXT_SIMPLE_CONFIG(zbb, ZBB);
@ -1045,8 +1051,10 @@ struct vcpu_reg_list *vcpu_configs[] = {
&config_svnapot,
&config_svpbmt,
&config_svvptc,
&config_zaamo,
&config_zabha,
&config_zacas,
&config_zalrsc,
&config_zawrs,
&config_zba,
&config_zbb,

66
tools/testing/selftests/riscv/hwprobe/cbo.c
View File

@ -50,6 +50,14 @@ static void cbo_clean(char *base) { cbo_insn(base, 1); }
static void cbo_flush(char *base) { cbo_insn(base, 2); }
static void cbo_zero(char *base) { cbo_insn(base, 4); }
static void test_no_cbo_inval(void *arg)
{
ksft_print_msg("Testing cbo.inval instruction remain privileged\n");
illegal_insn = false;
cbo_inval(&mem[0]);
ksft_test_result(illegal_insn, "No cbo.inval\n");
}
static void test_no_zicbom(void *arg)
{
ksft_print_msg("Testing Zicbom instructions remain privileged\n");
@ -61,10 +69,6 @@ static void test_no_zicbom(void *arg)
illegal_insn = false;
cbo_flush(&mem[0]);
ksft_test_result(illegal_insn, "No cbo.flush\n");
illegal_insn = false;
cbo_inval(&mem[0]);
ksft_test_result(illegal_insn, "No cbo.inval\n");
}
static void test_no_zicboz(void *arg)
@ -81,6 +85,30 @@ static bool is_power_of_2(__u64 n)
return n != 0 && (n & (n - 1)) == 0;
}
static void test_zicbom(void *arg)
{
struct riscv_hwprobe pair = {
.key = RISCV_HWPROBE_KEY_ZICBOM_BLOCK_SIZE,
};
cpu_set_t *cpus = (cpu_set_t *)arg;
__u64 block_size;
long rc;
rc = riscv_hwprobe(&pair, 1, sizeof(cpu_set_t), (unsigned long *)cpus, 0);
block_size = pair.value;
ksft_test_result(rc == 0 && pair.key == RISCV_HWPROBE_KEY_ZICBOM_BLOCK_SIZE &&
is_power_of_2(block_size), "Zicbom block size\n");
ksft_print_msg("Zicbom block size: %llu\n", block_size);
illegal_insn = false;
cbo_clean(&mem[block_size]);
ksft_test_result(!illegal_insn, "cbo.clean\n");
illegal_insn = false;
cbo_flush(&mem[block_size]);
ksft_test_result(!illegal_insn, "cbo.flush\n");
}
static void test_zicboz(void *arg)
{
struct riscv_hwprobe pair = {
@ -129,7 +157,7 @@ static void test_zicboz(void *arg)
ksft_test_result_pass("cbo.zero check\n");
}
static void check_no_zicboz_cpus(cpu_set_t *cpus)
static void check_no_zicbo_cpus(cpu_set_t *cpus, __u64 cbo)
{
struct riscv_hwprobe pair = {
.key = RISCV_HWPROBE_KEY_IMA_EXT_0,
@ -137,6 +165,7 @@ static void check_no_zicboz_cpus(cpu_set_t *cpus)
cpu_set_t one_cpu;
int i = 0, c = 0;
long rc;
char *cbostr;
while (i++ < CPU_COUNT(cpus)) {
while (!CPU_ISSET(c, cpus))
@ -148,10 +177,13 @@ static void check_no_zicboz_cpus(cpu_set_t *cpus)
rc = riscv_hwprobe(&pair, 1, sizeof(cpu_set_t), (unsigned long *)&one_cpu, 0);
assert(rc == 0 && pair.key == RISCV_HWPROBE_KEY_IMA_EXT_0);
if (pair.value & RISCV_HWPROBE_EXT_ZICBOZ)
ksft_exit_fail_msg("Zicboz is only present on a subset of harts.\n"
"Use taskset to select a set of harts where Zicboz\n"
"presence (present or not) is consistent for each hart\n");
cbostr = cbo == RISCV_HWPROBE_EXT_ZICBOZ ? "Zicboz" : "Zicbom";
if (pair.value & cbo)
ksft_exit_fail_msg("%s is only present on a subset of harts.\n"
"Use taskset to select a set of harts where %s\n"
"presence (present or not) is consistent for each hart\n",
cbostr, cbostr);
++c;
}
}
@ -159,7 +191,9 @@ static void check_no_zicboz_cpus(cpu_set_t *cpus)
enum {
TEST_ZICBOZ,
TEST_NO_ZICBOZ,
TEST_ZICBOM,
TEST_NO_ZICBOM,
TEST_NO_CBO_INVAL,
};
static struct test_info {
@ -169,7 +203,9 @@ static struct test_info {
} tests[] = {
[TEST_ZICBOZ] = { .nr_tests = 3, test_zicboz },
[TEST_NO_ZICBOZ] = { .nr_tests = 1, test_no_zicboz },
[TEST_NO_ZICBOM] = { .nr_tests = 3, test_no_zicbom },
[TEST_ZICBOM] = { .nr_tests = 3, test_zicbom },
[TEST_NO_ZICBOM] = { .nr_tests = 2, test_no_zicbom },
[TEST_NO_CBO_INVAL] = { .nr_tests = 1, test_no_cbo_inval },
};
int main(int argc, char **argv)
@ -189,6 +225,7 @@ int main(int argc, char **argv)
assert(rc == 0);
tests[TEST_NO_ZICBOZ].enabled = true;
tests[TEST_NO_ZICBOM].enabled = true;
tests[TEST_NO_CBO_INVAL].enabled = true;
}
rc = sched_getaffinity(0, sizeof(cpu_set_t), &cpus);
@ -206,7 +243,14 @@ int main(int argc, char **argv)
tests[TEST_ZICBOZ].enabled = true;
tests[TEST_NO_ZICBOZ].enabled = false;
} else {
check_no_zicboz_cpus(&cpus);
check_no_zicbo_cpus(&cpus, RISCV_HWPROBE_EXT_ZICBOZ);
}
if (pair.value & RISCV_HWPROBE_EXT_ZICBOM) {
tests[TEST_ZICBOM].enabled = true;
tests[TEST_NO_ZICBOM].enabled = false;
} else {
check_no_zicbo_cpus(&cpus, RISCV_HWPROBE_EXT_ZICBOM);
}
for (i = 0; i < ARRAY_SIZE(tests); ++i)

10
tools/testing/selftests/riscv/vector/v_exec_initval_nolibc.c
View File

@ -6,7 +6,7 @@
* the values. To further ensure consistency, this file is compiled without
* libc and without auto-vectorization.
*
* To be "clean" all values must be either all ones or all zeroes.
* To be "clean" all values must be all zeroes.
*/
#define __stringify_1(x...) #x
@ -14,9 +14,8 @@
int main(int argc, char **argv)
{
char prev_value = 0, value;
char value = 0;
unsigned long vl;
int first = 1;
if (argc > 2 && strcmp(argv[2], "x"))
asm volatile (
@ -44,14 +43,11 @@ int main(int argc, char **argv)
"vsrl.vi " __stringify(register) ", " __stringify(register) ", 8\n\t" \
".option pop\n\t" \
: "=r" (value)); \
if (first) { \
first = 0; \
} else if (value != prev_value || !(value == 0x00 || value == 0xff)) { \
if (value != 0x00) { \
printf("Register " __stringify(register) \
" values not clean! value: %u\n", value); \
exit(-1); \
} \
prev_value = value; \
} \
})