mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/
synced 2025-04-19 20:58:31 +09:00
KVM/arm64: First batch of fixes for 6.15
- Rework heuristics for resolving the fault IPA (HPFAR_EL2 v. re-walk
stage-1 page tables) to align with the architecture. This avoids
possibly taking an SEA at EL2 on the page table walk or using an
architecturally UNKNOWN fault IPA.
- Use acquire/release semantics in the KVM FF-A proxy to avoid reading
a stale value for the FF-A version.
- Fix KVM guest driver to match PV CPUID hypercall ABI.
- Use Inner Shareable Normal Write-Back mappings at stage-1 in KVM
selftests, which is the only memory type for which atomic
instructions are architecturally guaranteed to work.
-----BEGIN PGP SIGNATURE-----
iI0EABYIADUWIQSNXHjWXuzMZutrKNKivnWIJHzdFgUCZ/RO9hccb2xpdmVyLnVw
dG9uQGxpbnV4LmRldgAKCRCivnWIJHzdFmRuAP0YajO4qHJe1vHtCkamuPnEY0Kp
E+t2TwPafPbrPdQ1PgEAq6lHuSdUnid1r/uhRKIT+ywW8tE97eNwQAa1LFma0Ac=
=d4G5
-----END PGP SIGNATURE-----
Merge tag 'kvmarm-fixes-6.15-1' of https://git.kernel.org/pub/scm/linux/kernel/git/kvmarm/kvmarm into HEAD
KVM/arm64: First batch of fixes for 6.15
- Rework heuristics for resolving the fault IPA (HPFAR_EL2 v. re-walk
stage-1 page tables) to align with the architecture. This avoids
possibly taking an SEA at EL2 on the page table walk or using an
architecturally UNKNOWN fault IPA.
- Use acquire/release semantics in the KVM FF-A proxy to avoid reading
a stale value for the FF-A version.
- Fix KVM guest driver to match PV CPUID hypercall ABI.
- Use Inner Shareable Normal Write-Back mappings at stage-1 in KVM
selftests, which is the only memory type for which atomic
instructions are architecturally guaranteed to work.
This commit is contained in:
Paolo Bonzini
commit
c478032df0
@ -121,6 +121,15 @@
|
||||
#define ESR_ELx_FSC_SEA_TTW(n) (0x14 + (n))
|
||||
#define ESR_ELx_FSC_SECC (0x18)
|
||||
#define ESR_ELx_FSC_SECC_TTW(n) (0x1c + (n))
|
||||
#define ESR_ELx_FSC_ADDRSZ (0x00)
|
||||
|
||||
/*
|
||||
* Annoyingly, the negative levels for Address size faults aren't laid out
|
||||
* contiguously (or in the desired order)
|
||||
*/
|
||||
#define ESR_ELx_FSC_ADDRSZ_nL(n) ((n) == -1 ? 0x25 : 0x2C)
|
||||
#define ESR_ELx_FSC_ADDRSZ_L(n) ((n) < 0 ? ESR_ELx_FSC_ADDRSZ_nL(n) : \
|
||||
(ESR_ELx_FSC_ADDRSZ + (n)))
|
||||
|
||||
/* Status codes for individual page table levels */
|
||||
#define ESR_ELx_FSC_ACCESS_L(n) (ESR_ELx_FSC_ACCESS + (n))
|
||||
@ -161,8 +170,6 @@
|
||||
#define ESR_ELx_Xs_MASK (GENMASK_ULL(4, 0))
|
||||
|
||||
/* ISS field definitions for exceptions taken in to Hyp */
|
||||
#define ESR_ELx_FSC_ADDRSZ (0x00)
|
||||
#define ESR_ELx_FSC_ADDRSZ_L(n) (ESR_ELx_FSC_ADDRSZ + (n))
|
||||
#define ESR_ELx_CV (UL(1) << 24)
|
||||
#define ESR_ELx_COND_SHIFT (20)
|
||||
#define ESR_ELx_COND_MASK (UL(0xF) << ESR_ELx_COND_SHIFT)
|
||||
@ -464,6 +471,39 @@ static inline bool esr_fsc_is_access_flag_fault(unsigned long esr)
|
||||
(esr == ESR_ELx_FSC_ACCESS_L(0));
|
||||
}
|
||||
|
||||
static inline bool esr_fsc_is_addr_sz_fault(unsigned long esr)
|
||||
{
|
||||
esr &= ESR_ELx_FSC;
|
||||
|
||||
return (esr == ESR_ELx_FSC_ADDRSZ_L(3)) ||
|
||||
(esr == ESR_ELx_FSC_ADDRSZ_L(2)) ||
|
||||
(esr == ESR_ELx_FSC_ADDRSZ_L(1)) ||
|
||||
(esr == ESR_ELx_FSC_ADDRSZ_L(0)) ||
|
||||
(esr == ESR_ELx_FSC_ADDRSZ_L(-1));
|
||||
}
|
||||
|
||||
static inline bool esr_fsc_is_sea_ttw(unsigned long esr)
|
||||
{
|
||||
esr = esr & ESR_ELx_FSC;
|
||||
|
||||
return (esr == ESR_ELx_FSC_SEA_TTW(3)) ||
|
||||
(esr == ESR_ELx_FSC_SEA_TTW(2)) ||
|
||||
(esr == ESR_ELx_FSC_SEA_TTW(1)) ||
|
||||
(esr == ESR_ELx_FSC_SEA_TTW(0)) ||
|
||||
(esr == ESR_ELx_FSC_SEA_TTW(-1));
|
||||
}
|
||||
|
||||
static inline bool esr_fsc_is_secc_ttw(unsigned long esr)
|
||||
{
|
||||
esr = esr & ESR_ELx_FSC;
|
||||
|
||||
return (esr == ESR_ELx_FSC_SECC_TTW(3)) ||
|
||||
(esr == ESR_ELx_FSC_SECC_TTW(2)) ||
|
||||
(esr == ESR_ELx_FSC_SECC_TTW(1)) ||
|
||||
(esr == ESR_ELx_FSC_SECC_TTW(0)) ||
|
||||
(esr == ESR_ELx_FSC_SECC_TTW(-1));
|
||||
}
|
||||
|
||||
/* Indicate whether ESR.EC==0x1A is for an ERETAx instruction */
|
||||
static inline bool esr_iss_is_eretax(unsigned long esr)
|
||||
{
|
||||
|
||||
@ -305,7 +305,12 @@ static __always_inline unsigned long kvm_vcpu_get_hfar(const struct kvm_vcpu *vc
|
||||
|
||||
static __always_inline phys_addr_t kvm_vcpu_get_fault_ipa(const struct kvm_vcpu *vcpu)
|
||||
{
|
||||
return ((phys_addr_t)vcpu->arch.fault.hpfar_el2 & HPFAR_MASK) << 8;
|
||||
u64 hpfar = vcpu->arch.fault.hpfar_el2;
|
||||
|
||||
if (unlikely(!(hpfar & HPFAR_EL2_NS)))
|
||||
return INVALID_GPA;
|
||||
|
||||
return FIELD_GET(HPFAR_EL2_FIPA, hpfar) << 12;
|
||||
}
|
||||
|
||||
static inline u64 kvm_vcpu_get_disr(const struct kvm_vcpu *vcpu)
|
||||
|
||||
@ -14,7 +14,7 @@
|
||||
* Was this synchronous external abort a RAS notification?
|
||||
* Returns '0' for errors handled by some RAS subsystem, or -ENOENT.
|
||||
*/
|
||||
static inline int kvm_handle_guest_sea(phys_addr_t addr, u64 esr)
|
||||
static inline int kvm_handle_guest_sea(void)
|
||||
{
|
||||
/* apei_claim_sea(NULL) expects to mask interrupts itself */
|
||||
lockdep_assert_irqs_enabled();
|
||||
|
||||
@ -12,6 +12,16 @@
|
||||
#include <asm/kvm_hyp.h>
|
||||
#include <asm/kvm_mmu.h>
|
||||
|
||||
static inline bool __fault_safe_to_translate(u64 esr)
|
||||
{
|
||||
u64 fsc = esr & ESR_ELx_FSC;
|
||||
|
||||
if (esr_fsc_is_sea_ttw(esr) || esr_fsc_is_secc_ttw(esr))
|
||||
return false;
|
||||
|
||||
return !(fsc == ESR_ELx_FSC_EXTABT && (esr & ESR_ELx_FnV));
|
||||
}
|
||||
|
||||
static inline bool __translate_far_to_hpfar(u64 far, u64 *hpfar)
|
||||
{
|
||||
int ret;
|
||||
@ -44,34 +54,50 @@ static inline bool __translate_far_to_hpfar(u64 far, u64 *hpfar)
|
||||
return true;
|
||||
}
|
||||
|
||||
/*
|
||||
* Checks for the conditions when HPFAR_EL2 is written, per ARM ARM R_FKLWR.
|
||||
*/
|
||||
static inline bool __hpfar_valid(u64 esr)
|
||||
{
|
||||
/*
|
||||
* CPUs affected by ARM erratum #834220 may incorrectly report a
|
||||
* stage-2 translation fault when a stage-1 permission fault occurs.
|
||||
*
|
||||
* Re-walk the page tables to determine if a stage-1 fault actually
|
||||
* occurred.
|
||||
*/
|
||||
if (cpus_have_final_cap(ARM64_WORKAROUND_834220) &&
|
||||
esr_fsc_is_translation_fault(esr))
|
||||
return false;
|
||||
|
||||
if (esr_fsc_is_translation_fault(esr) || esr_fsc_is_access_flag_fault(esr))
|
||||
return true;
|
||||
|
||||
if ((esr & ESR_ELx_S1PTW) && esr_fsc_is_permission_fault(esr))
|
||||
return true;
|
||||
|
||||
return esr_fsc_is_addr_sz_fault(esr);
|
||||
}
|
||||
|
||||
static inline bool __get_fault_info(u64 esr, struct kvm_vcpu_fault_info *fault)
|
||||
{
|
||||
u64 hpfar, far;
|
||||
u64 hpfar;
|
||||
|
||||
far = read_sysreg_el2(SYS_FAR);
|
||||
fault->far_el2 = read_sysreg_el2(SYS_FAR);
|
||||
fault->hpfar_el2 = 0;
|
||||
|
||||
if (__hpfar_valid(esr))
|
||||
hpfar = read_sysreg(hpfar_el2);
|
||||
else if (unlikely(!__fault_safe_to_translate(esr)))
|
||||
return true;
|
||||
else if (!__translate_far_to_hpfar(fault->far_el2, &hpfar))
|
||||
return false;
|
||||
|
||||
/*
|
||||
* The HPFAR can be invalid if the stage 2 fault did not
|
||||
* happen during a stage 1 page table walk (the ESR_EL2.S1PTW
|
||||
* bit is clear) and one of the two following cases are true:
|
||||
* 1. The fault was due to a permission fault
|
||||
* 2. The processor carries errata 834220
|
||||
*
|
||||
* Therefore, for all non S1PTW faults where we either have a
|
||||
* permission fault or the errata workaround is enabled, we
|
||||
* resolve the IPA using the AT instruction.
|
||||
* Hijack HPFAR_EL2.NS (RES0 in Non-secure) to indicate a valid
|
||||
* HPFAR value.
|
||||
*/
|
||||
if (!(esr & ESR_ELx_S1PTW) &&
|
||||
(cpus_have_final_cap(ARM64_WORKAROUND_834220) ||
|
||||
esr_fsc_is_permission_fault(esr))) {
|
||||
if (!__translate_far_to_hpfar(far, &hpfar))
|
||||
return false;
|
||||
} else {
|
||||
hpfar = read_sysreg(hpfar_el2);
|
||||
}
|
||||
|
||||
fault->far_el2 = far;
|
||||
fault->hpfar_el2 = hpfar;
|
||||
fault->hpfar_el2 = hpfar | HPFAR_EL2_NS;
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
@ -730,10 +730,10 @@ static void do_ffa_version(struct arm_smccc_res *res,
|
||||
hyp_ffa_version = ffa_req_version;
|
||||
}
|
||||
|
||||
if (hyp_ffa_post_init())
|
||||
if (hyp_ffa_post_init()) {
|
||||
res->a0 = FFA_RET_NOT_SUPPORTED;
|
||||
else {
|
||||
has_version_negotiated = true;
|
||||
} else {
|
||||
smp_store_release(&has_version_negotiated, true);
|
||||
res->a0 = hyp_ffa_version;
|
||||
}
|
||||
unlock:
|
||||
@ -809,7 +809,8 @@ bool kvm_host_ffa_handler(struct kvm_cpu_context *host_ctxt, u32 func_id)
|
||||
if (!is_ffa_call(func_id))
|
||||
return false;
|
||||
|
||||
if (!has_version_negotiated && func_id != FFA_VERSION) {
|
||||
if (func_id != FFA_VERSION &&
|
||||
!smp_load_acquire(&has_version_negotiated)) {
|
||||
ffa_to_smccc_error(&res, FFA_RET_INVALID_PARAMETERS);
|
||||
goto out_handled;
|
||||
}
|
||||
|
||||
@ -578,7 +578,14 @@ void handle_host_mem_abort(struct kvm_cpu_context *host_ctxt)
|
||||
return;
|
||||
}
|
||||
|
||||
addr = (fault.hpfar_el2 & HPFAR_MASK) << 8;
|
||||
|
||||
/*
|
||||
* Yikes, we couldn't resolve the fault IPA. This should reinject an
|
||||
* abort into the host when we figure out how to do that.
|
||||
*/
|
||||
BUG_ON(!(fault.hpfar_el2 & HPFAR_EL2_NS));
|
||||
addr = FIELD_GET(HPFAR_EL2_FIPA, fault.hpfar_el2) << 12;
|
||||
|
||||
ret = host_stage2_idmap(addr);
|
||||
BUG_ON(ret && ret != -EAGAIN);
|
||||
}
|
||||
|
||||
@ -1794,9 +1794,28 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu)
|
||||
gfn_t gfn;
|
||||
int ret, idx;
|
||||
|
||||
/* Synchronous External Abort? */
|
||||
if (kvm_vcpu_abt_issea(vcpu)) {
|
||||
/*
|
||||
* For RAS the host kernel may handle this abort.
|
||||
* There is no need to pass the error into the guest.
|
||||
*/
|
||||
if (kvm_handle_guest_sea())
|
||||
kvm_inject_vabt(vcpu);
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
esr = kvm_vcpu_get_esr(vcpu);
|
||||
|
||||
/*
|
||||
* The fault IPA should be reliable at this point as we're not dealing
|
||||
* with an SEA.
|
||||
*/
|
||||
ipa = fault_ipa = kvm_vcpu_get_fault_ipa(vcpu);
|
||||
if (KVM_BUG_ON(ipa == INVALID_GPA, vcpu->kvm))
|
||||
return -EFAULT;
|
||||
|
||||
is_iabt = kvm_vcpu_trap_is_iabt(vcpu);
|
||||
|
||||
if (esr_fsc_is_translation_fault(esr)) {
|
||||
@ -1818,18 +1837,6 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu)
|
||||
}
|
||||
}
|
||||
|
||||
/* Synchronous External Abort? */
|
||||
if (kvm_vcpu_abt_issea(vcpu)) {
|
||||
/*
|
||||
* For RAS the host kernel may handle this abort.
|
||||
* There is no need to pass the error into the guest.
|
||||
*/
|
||||
if (kvm_handle_guest_sea(fault_ipa, kvm_vcpu_get_esr(vcpu)))
|
||||
kvm_inject_vabt(vcpu);
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
trace_kvm_guest_fault(*vcpu_pc(vcpu), kvm_vcpu_get_esr(vcpu),
|
||||
kvm_vcpu_get_hfar(vcpu), fault_ipa);
|
||||
|
||||
|
||||
@ -3433,3 +3433,10 @@ Field 5 F
|
||||
Field 4 P
|
||||
Field 3:0 Align
|
||||
EndSysreg
|
||||
|
||||
Sysreg HPFAR_EL2 3 4 6 0 4
|
||||
Field 63 NS
|
||||
Res0 62:48
|
||||
Field 47:4 FIPA
|
||||
Res0 3:0
|
||||
EndSysreg
|
||||
|
||||
@ -95,7 +95,7 @@ void __init kvm_arm_target_impl_cpu_init(void)
|
||||
|
||||
for (i = 0; i < max_cpus; i++) {
|
||||
arm_smccc_1_1_invoke(ARM_SMCCC_VENDOR_HYP_KVM_DISCOVER_IMPL_CPUS_FUNC_ID,
|
||||
i, &res);
|
||||
i, 0, 0, &res);
|
||||
if (res.a0 != SMCCC_RET_SUCCESS) {
|
||||
pr_warn("Discovering target implementation CPUs failed\n");
|
||||
goto mem_free;
|
||||
@ -103,7 +103,7 @@ void __init kvm_arm_target_impl_cpu_init(void)
|
||||
target[i].midr = res.a1;
|
||||
target[i].revidr = res.a2;
|
||||
target[i].aidr = res.a3;
|
||||
};
|
||||
}
|
||||
|
||||
if (!cpu_errata_set_target_impl(max_cpus, target)) {
|
||||
pr_warn("Failed to set target implementation CPUs\n");
|
||||
|
||||
@ -199,7 +199,7 @@ static bool guest_set_ha(void)
|
||||
if (hadbs == 0)
|
||||
return false;
|
||||
|
||||
tcr = read_sysreg(tcr_el1) | TCR_EL1_HA;
|
||||
tcr = read_sysreg(tcr_el1) | TCR_HA;
|
||||
write_sysreg(tcr, tcr_el1);
|
||||
isb();
|
||||
|
||||
|
||||
@ -62,6 +62,67 @@
|
||||
MAIR_ATTRIDX(MAIR_ATTR_NORMAL, MT_NORMAL) | \
|
||||
MAIR_ATTRIDX(MAIR_ATTR_NORMAL_WT, MT_NORMAL_WT))
|
||||
|
||||
/* TCR_EL1 specific flags */
|
||||
#define TCR_T0SZ_OFFSET 0
|
||||
#define TCR_T0SZ(x) ((UL(64) - (x)) << TCR_T0SZ_OFFSET)
|
||||
|
||||
#define TCR_IRGN0_SHIFT 8
|
||||
#define TCR_IRGN0_MASK (UL(3) << TCR_IRGN0_SHIFT)
|
||||
#define TCR_IRGN0_NC (UL(0) << TCR_IRGN0_SHIFT)
|
||||
#define TCR_IRGN0_WBWA (UL(1) << TCR_IRGN0_SHIFT)
|
||||
#define TCR_IRGN0_WT (UL(2) << TCR_IRGN0_SHIFT)
|
||||
#define TCR_IRGN0_WBnWA (UL(3) << TCR_IRGN0_SHIFT)
|
||||
|
||||
#define TCR_ORGN0_SHIFT 10
|
||||
#define TCR_ORGN0_MASK (UL(3) << TCR_ORGN0_SHIFT)
|
||||
#define TCR_ORGN0_NC (UL(0) << TCR_ORGN0_SHIFT)
|
||||
#define TCR_ORGN0_WBWA (UL(1) << TCR_ORGN0_SHIFT)
|
||||
#define TCR_ORGN0_WT (UL(2) << TCR_ORGN0_SHIFT)
|
||||
#define TCR_ORGN0_WBnWA (UL(3) << TCR_ORGN0_SHIFT)
|
||||
|
||||
#define TCR_SH0_SHIFT 12
|
||||
#define TCR_SH0_MASK (UL(3) << TCR_SH0_SHIFT)
|
||||
#define TCR_SH0_INNER (UL(3) << TCR_SH0_SHIFT)
|
||||
|
||||
#define TCR_TG0_SHIFT 14
|
||||
#define TCR_TG0_MASK (UL(3) << TCR_TG0_SHIFT)
|
||||
#define TCR_TG0_4K (UL(0) << TCR_TG0_SHIFT)
|
||||
#define TCR_TG0_64K (UL(1) << TCR_TG0_SHIFT)
|
||||
#define TCR_TG0_16K (UL(2) << TCR_TG0_SHIFT)
|
||||
|
||||
#define TCR_IPS_SHIFT 32
|
||||
#define TCR_IPS_MASK (UL(7) << TCR_IPS_SHIFT)
|
||||
#define TCR_IPS_52_BITS (UL(6) << TCR_IPS_SHIFT)
|
||||
#define TCR_IPS_48_BITS (UL(5) << TCR_IPS_SHIFT)
|
||||
#define TCR_IPS_40_BITS (UL(2) << TCR_IPS_SHIFT)
|
||||
#define TCR_IPS_36_BITS (UL(1) << TCR_IPS_SHIFT)
|
||||
|
||||
#define TCR_HA (UL(1) << 39)
|
||||
#define TCR_DS (UL(1) << 59)
|
||||
|
||||
/*
|
||||
* AttrIndx[2:0] encoding (mapping attributes defined in the MAIR* registers).
|
||||
*/
|
||||
#define PTE_ATTRINDX(t) ((t) << 2)
|
||||
#define PTE_ATTRINDX_MASK GENMASK(4, 2)
|
||||
#define PTE_ATTRINDX_SHIFT 2
|
||||
|
||||
#define PTE_VALID BIT(0)
|
||||
#define PGD_TYPE_TABLE BIT(1)
|
||||
#define PUD_TYPE_TABLE BIT(1)
|
||||
#define PMD_TYPE_TABLE BIT(1)
|
||||
#define PTE_TYPE_PAGE BIT(1)
|
||||
|
||||
#define PTE_SHARED (UL(3) << 8) /* SH[1:0], inner shareable */
|
||||
#define PTE_AF BIT(10)
|
||||
|
||||
#define PTE_ADDR_MASK(page_shift) GENMASK(47, (page_shift))
|
||||
#define PTE_ADDR_51_48 GENMASK(15, 12)
|
||||
#define PTE_ADDR_51_48_SHIFT 12
|
||||
#define PTE_ADDR_MASK_LPA2(page_shift) GENMASK(49, (page_shift))
|
||||
#define PTE_ADDR_51_50_LPA2 GENMASK(9, 8)
|
||||
#define PTE_ADDR_51_50_LPA2_SHIFT 8
|
||||
|
||||
void aarch64_vcpu_setup(struct kvm_vcpu *vcpu, struct kvm_vcpu_init *init);
|
||||
struct kvm_vcpu *aarch64_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id,
|
||||
struct kvm_vcpu_init *init, void *guest_code);
|
||||
@ -102,12 +163,6 @@ enum {
|
||||
(v) == VECTOR_SYNC_LOWER_64 || \
|
||||
(v) == VECTOR_SYNC_LOWER_32)
|
||||
|
||||
/* Access flag */
|
||||
#define PTE_AF (1ULL << 10)
|
||||
|
||||
/* Access flag update enable/disable */
|
||||
#define TCR_EL1_HA (1ULL << 39)
|
||||
|
||||
void aarch64_get_supported_page_sizes(uint32_t ipa, uint32_t *ipa4k,
|
||||
uint32_t *ipa16k, uint32_t *ipa64k);
|
||||
|
||||
|
||||
@ -72,13 +72,13 @@ static uint64_t addr_pte(struct kvm_vm *vm, uint64_t pa, uint64_t attrs)
|
||||
uint64_t pte;
|
||||
|
||||
if (use_lpa2_pte_format(vm)) {
|
||||
pte = pa & GENMASK(49, vm->page_shift);
|
||||
pte |= FIELD_GET(GENMASK(51, 50), pa) << 8;
|
||||
attrs &= ~GENMASK(9, 8);
|
||||
pte = pa & PTE_ADDR_MASK_LPA2(vm->page_shift);
|
||||
pte |= FIELD_GET(GENMASK(51, 50), pa) << PTE_ADDR_51_50_LPA2_SHIFT;
|
||||
attrs &= ~PTE_ADDR_51_50_LPA2;
|
||||
} else {
|
||||
pte = pa & GENMASK(47, vm->page_shift);
|
||||
pte = pa & PTE_ADDR_MASK(vm->page_shift);
|
||||
if (vm->page_shift == 16)
|
||||
pte |= FIELD_GET(GENMASK(51, 48), pa) << 12;
|
||||
pte |= FIELD_GET(GENMASK(51, 48), pa) << PTE_ADDR_51_48_SHIFT;
|
||||
}
|
||||
pte |= attrs;
|
||||
|
||||
@ -90,12 +90,12 @@ static uint64_t pte_addr(struct kvm_vm *vm, uint64_t pte)
|
||||
uint64_t pa;
|
||||
|
||||
if (use_lpa2_pte_format(vm)) {
|
||||
pa = pte & GENMASK(49, vm->page_shift);
|
||||
pa |= FIELD_GET(GENMASK(9, 8), pte) << 50;
|
||||
pa = pte & PTE_ADDR_MASK_LPA2(vm->page_shift);
|
||||
pa |= FIELD_GET(PTE_ADDR_51_50_LPA2, pte) << 50;
|
||||
} else {
|
||||
pa = pte & GENMASK(47, vm->page_shift);
|
||||
pa = pte & PTE_ADDR_MASK(vm->page_shift);
|
||||
if (vm->page_shift == 16)
|
||||
pa |= FIELD_GET(GENMASK(15, 12), pte) << 48;
|
||||
pa |= FIELD_GET(PTE_ADDR_51_48, pte) << 48;
|
||||
}
|
||||
|
||||
return pa;
|
||||
@ -128,7 +128,8 @@ void virt_arch_pgd_alloc(struct kvm_vm *vm)
|
||||
static void _virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr,
|
||||
uint64_t flags)
|
||||
{
|
||||
uint8_t attr_idx = flags & 7;
|
||||
uint8_t attr_idx = flags & (PTE_ATTRINDX_MASK >> PTE_ATTRINDX_SHIFT);
|
||||
uint64_t pg_attr;
|
||||
uint64_t *ptep;
|
||||
|
||||
TEST_ASSERT((vaddr % vm->page_size) == 0,
|
||||
@ -147,18 +148,21 @@ static void _virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr,
|
||||
|
||||
ptep = addr_gpa2hva(vm, vm->pgd) + pgd_index(vm, vaddr) * 8;
|
||||
if (!*ptep)
|
||||
*ptep = addr_pte(vm, vm_alloc_page_table(vm), 3);
|
||||
*ptep = addr_pte(vm, vm_alloc_page_table(vm),
|
||||
PGD_TYPE_TABLE | PTE_VALID);
|
||||
|
||||
switch (vm->pgtable_levels) {
|
||||
case 4:
|
||||
ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + pud_index(vm, vaddr) * 8;
|
||||
if (!*ptep)
|
||||
*ptep = addr_pte(vm, vm_alloc_page_table(vm), 3);
|
||||
*ptep = addr_pte(vm, vm_alloc_page_table(vm),
|
||||
PUD_TYPE_TABLE | PTE_VALID);
|
||||
/* fall through */
|
||||
case 3:
|
||||
ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + pmd_index(vm, vaddr) * 8;
|
||||
if (!*ptep)
|
||||
*ptep = addr_pte(vm, vm_alloc_page_table(vm), 3);
|
||||
*ptep = addr_pte(vm, vm_alloc_page_table(vm),
|
||||
PMD_TYPE_TABLE | PTE_VALID);
|
||||
/* fall through */
|
||||
case 2:
|
||||
ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + pte_index(vm, vaddr) * 8;
|
||||
@ -167,7 +171,11 @@ static void _virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr,
|
||||
TEST_FAIL("Page table levels must be 2, 3, or 4");
|
||||
}
|
||||
|
||||
*ptep = addr_pte(vm, paddr, (attr_idx << 2) | (1 << 10) | 3); /* AF */
|
||||
pg_attr = PTE_AF | PTE_ATTRINDX(attr_idx) | PTE_TYPE_PAGE | PTE_VALID;
|
||||
if (!use_lpa2_pte_format(vm))
|
||||
pg_attr |= PTE_SHARED;
|
||||
|
||||
*ptep = addr_pte(vm, paddr, pg_attr);
|
||||
}
|
||||
|
||||
void virt_arch_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr)
|
||||
@ -293,20 +301,20 @@ void aarch64_vcpu_setup(struct kvm_vcpu *vcpu, struct kvm_vcpu_init *init)
|
||||
case VM_MODE_P48V48_64K:
|
||||
case VM_MODE_P40V48_64K:
|
||||
case VM_MODE_P36V48_64K:
|
||||
tcr_el1 |= 1ul << 14; /* TG0 = 64KB */
|
||||
tcr_el1 |= TCR_TG0_64K;
|
||||
break;
|
||||
case VM_MODE_P52V48_16K:
|
||||
case VM_MODE_P48V48_16K:
|
||||
case VM_MODE_P40V48_16K:
|
||||
case VM_MODE_P36V48_16K:
|
||||
case VM_MODE_P36V47_16K:
|
||||
tcr_el1 |= 2ul << 14; /* TG0 = 16KB */
|
||||
tcr_el1 |= TCR_TG0_16K;
|
||||
break;
|
||||
case VM_MODE_P52V48_4K:
|
||||
case VM_MODE_P48V48_4K:
|
||||
case VM_MODE_P40V48_4K:
|
||||
case VM_MODE_P36V48_4K:
|
||||
tcr_el1 |= 0ul << 14; /* TG0 = 4KB */
|
||||
tcr_el1 |= TCR_TG0_4K;
|
||||
break;
|
||||
default:
|
||||
TEST_FAIL("Unknown guest mode, mode: 0x%x", vm->mode);
|
||||
@ -319,35 +327,35 @@ void aarch64_vcpu_setup(struct kvm_vcpu *vcpu, struct kvm_vcpu_init *init)
|
||||
case VM_MODE_P52V48_4K:
|
||||
case VM_MODE_P52V48_16K:
|
||||
case VM_MODE_P52V48_64K:
|
||||
tcr_el1 |= 6ul << 32; /* IPS = 52 bits */
|
||||
tcr_el1 |= TCR_IPS_52_BITS;
|
||||
ttbr0_el1 |= FIELD_GET(GENMASK(51, 48), vm->pgd) << 2;
|
||||
break;
|
||||
case VM_MODE_P48V48_4K:
|
||||
case VM_MODE_P48V48_16K:
|
||||
case VM_MODE_P48V48_64K:
|
||||
tcr_el1 |= 5ul << 32; /* IPS = 48 bits */
|
||||
tcr_el1 |= TCR_IPS_48_BITS;
|
||||
break;
|
||||
case VM_MODE_P40V48_4K:
|
||||
case VM_MODE_P40V48_16K:
|
||||
case VM_MODE_P40V48_64K:
|
||||
tcr_el1 |= 2ul << 32; /* IPS = 40 bits */
|
||||
tcr_el1 |= TCR_IPS_40_BITS;
|
||||
break;
|
||||
case VM_MODE_P36V48_4K:
|
||||
case VM_MODE_P36V48_16K:
|
||||
case VM_MODE_P36V48_64K:
|
||||
case VM_MODE_P36V47_16K:
|
||||
tcr_el1 |= 1ul << 32; /* IPS = 36 bits */
|
||||
tcr_el1 |= TCR_IPS_36_BITS;
|
||||
break;
|
||||
default:
|
||||
TEST_FAIL("Unknown guest mode, mode: 0x%x", vm->mode);
|
||||
}
|
||||
|
||||
sctlr_el1 |= (1 << 0) | (1 << 2) | (1 << 12) /* M | C | I */;
|
||||
/* TCR_EL1 |= IRGN0:WBWA | ORGN0:WBWA | SH0:Inner-Shareable */;
|
||||
tcr_el1 |= (1 << 8) | (1 << 10) | (3 << 12);
|
||||
tcr_el1 |= (64 - vm->va_bits) /* T0SZ */;
|
||||
sctlr_el1 |= SCTLR_ELx_M | SCTLR_ELx_C | SCTLR_ELx_I;
|
||||
|
||||
tcr_el1 |= TCR_IRGN0_WBWA | TCR_ORGN0_WBWA | TCR_SH0_INNER;
|
||||
tcr_el1 |= TCR_T0SZ(vm->va_bits);
|
||||
if (use_lpa2_pte_format(vm))
|
||||
tcr_el1 |= (1ul << 59) /* DS */;
|
||||
tcr_el1 |= TCR_DS;
|
||||
|
||||
vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_SCTLR_EL1), sctlr_el1);
|
||||
vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TCR_EL1), tcr_el1);
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user