310 lines
7.5 KiB
ArmAsm
310 lines
7.5 KiB
ArmAsm
/* SPDX-License-Identifier: GPL-2.0 */
|
|
/*
|
|
* Asm versions of Xen pv-ops, suitable for direct use.
|
|
*
|
|
* We only bother with direct forms (ie, vcpu in percpu data) of the
|
|
* operations here; the indirect forms are better handled in C.
|
|
*/
|
|
|
|
#include <asm/errno.h>
|
|
#include <asm/asm-offsets.h>
|
|
#include <asm/percpu.h>
|
|
#include <asm/processor-flags.h>
|
|
#include <asm/segment.h>
|
|
#include <asm/thread_info.h>
|
|
#include <asm/asm.h>
|
|
#include <asm/frame.h>
|
|
#include <asm/unwind_hints.h>
|
|
|
|
#include <xen/interface/xen.h>
|
|
|
|
#include <linux/init.h>
|
|
#include <linux/linkage.h>
|
|
#include <../entry/calling.h>
|
|
|
|
.pushsection .noinstr.text, "ax"
|
|
/*
|
|
* Disabling events is simply a matter of making the event mask
|
|
* non-zero.
|
|
*/
|
|
SYM_FUNC_START(xen_irq_disable_direct)
|
|
movb $1, PER_CPU_VAR(xen_vcpu_info + XEN_vcpu_info_mask)
|
|
RET
|
|
SYM_FUNC_END(xen_irq_disable_direct)
|
|
|
|
/*
|
|
* Force an event check by making a hypercall, but preserve regs
|
|
* before making the call.
|
|
*/
|
|
SYM_FUNC_START(check_events)
|
|
FRAME_BEGIN
|
|
push %rax
|
|
push %rcx
|
|
push %rdx
|
|
push %rsi
|
|
push %rdi
|
|
push %r8
|
|
push %r9
|
|
push %r10
|
|
push %r11
|
|
call xen_force_evtchn_callback
|
|
pop %r11
|
|
pop %r10
|
|
pop %r9
|
|
pop %r8
|
|
pop %rdi
|
|
pop %rsi
|
|
pop %rdx
|
|
pop %rcx
|
|
pop %rax
|
|
FRAME_END
|
|
RET
|
|
SYM_FUNC_END(check_events)
|
|
|
|
/*
|
|
* Enable events. This clears the event mask and tests the pending
|
|
* event status with one and operation. If there are pending events,
|
|
* then enter the hypervisor to get them handled.
|
|
*/
|
|
SYM_FUNC_START(xen_irq_enable_direct)
|
|
FRAME_BEGIN
|
|
/* Unmask events */
|
|
movb $0, PER_CPU_VAR(xen_vcpu_info + XEN_vcpu_info_mask)
|
|
|
|
/*
|
|
* Preempt here doesn't matter because that will deal with any
|
|
* pending interrupts. The pending check may end up being run
|
|
* on the wrong CPU, but that doesn't hurt.
|
|
*/
|
|
|
|
/* Test for pending */
|
|
testb $0xff, PER_CPU_VAR(xen_vcpu_info + XEN_vcpu_info_pending)
|
|
jz 1f
|
|
|
|
call check_events
|
|
1:
|
|
FRAME_END
|
|
RET
|
|
SYM_FUNC_END(xen_irq_enable_direct)
|
|
|
|
/*
|
|
* (xen_)save_fl is used to get the current interrupt enable status.
|
|
* Callers expect the status to be in X86_EFLAGS_IF, and other bits
|
|
* may be set in the return value. We take advantage of this by
|
|
* making sure that X86_EFLAGS_IF has the right value (and other bits
|
|
* in that byte are 0), but other bits in the return value are
|
|
* undefined. We need to toggle the state of the bit, because Xen and
|
|
* x86 use opposite senses (mask vs enable).
|
|
*/
|
|
SYM_FUNC_START(xen_save_fl_direct)
|
|
testb $0xff, PER_CPU_VAR(xen_vcpu_info + XEN_vcpu_info_mask)
|
|
setz %ah
|
|
addb %ah, %ah
|
|
RET
|
|
SYM_FUNC_END(xen_save_fl_direct)
|
|
|
|
SYM_FUNC_START(xen_read_cr2)
|
|
FRAME_BEGIN
|
|
_ASM_MOV PER_CPU_VAR(xen_vcpu), %_ASM_AX
|
|
_ASM_MOV XEN_vcpu_info_arch_cr2(%_ASM_AX), %_ASM_AX
|
|
FRAME_END
|
|
RET
|
|
SYM_FUNC_END(xen_read_cr2);
|
|
|
|
SYM_FUNC_START(xen_read_cr2_direct)
|
|
FRAME_BEGIN
|
|
_ASM_MOV PER_CPU_VAR(xen_vcpu_info + XEN_vcpu_info_arch_cr2), %_ASM_AX
|
|
FRAME_END
|
|
RET
|
|
SYM_FUNC_END(xen_read_cr2_direct);
|
|
.popsection
|
|
|
|
.macro xen_pv_trap name
|
|
SYM_CODE_START(xen_\name)
|
|
UNWIND_HINT_ENTRY
|
|
ENDBR
|
|
pop %rcx
|
|
pop %r11
|
|
jmp \name
|
|
SYM_CODE_END(xen_\name)
|
|
_ASM_NOKPROBE(xen_\name)
|
|
.endm
|
|
|
|
xen_pv_trap asm_exc_divide_error
|
|
xen_pv_trap asm_xenpv_exc_debug
|
|
xen_pv_trap asm_exc_int3
|
|
xen_pv_trap asm_xenpv_exc_nmi
|
|
xen_pv_trap asm_exc_overflow
|
|
xen_pv_trap asm_exc_bounds
|
|
xen_pv_trap asm_exc_invalid_op
|
|
xen_pv_trap asm_exc_device_not_available
|
|
xen_pv_trap asm_xenpv_exc_double_fault
|
|
xen_pv_trap asm_exc_coproc_segment_overrun
|
|
xen_pv_trap asm_exc_invalid_tss
|
|
xen_pv_trap asm_exc_segment_not_present
|
|
xen_pv_trap asm_exc_stack_segment
|
|
xen_pv_trap asm_exc_general_protection
|
|
xen_pv_trap asm_exc_page_fault
|
|
xen_pv_trap asm_exc_spurious_interrupt_bug
|
|
xen_pv_trap asm_exc_coprocessor_error
|
|
xen_pv_trap asm_exc_alignment_check
|
|
#ifdef CONFIG_X86_CET
|
|
xen_pv_trap asm_exc_control_protection
|
|
#endif
|
|
#ifdef CONFIG_X86_MCE
|
|
xen_pv_trap asm_xenpv_exc_machine_check
|
|
#endif /* CONFIG_X86_MCE */
|
|
xen_pv_trap asm_exc_simd_coprocessor_error
|
|
#ifdef CONFIG_IA32_EMULATION
|
|
xen_pv_trap asm_int80_emulation
|
|
#endif
|
|
xen_pv_trap asm_exc_xen_unknown_trap
|
|
xen_pv_trap asm_exc_xen_hypervisor_callback
|
|
|
|
__INIT
|
|
SYM_CODE_START(xen_early_idt_handler_array)
|
|
i = 0
|
|
.rept NUM_EXCEPTION_VECTORS
|
|
UNWIND_HINT_UNDEFINED
|
|
ENDBR
|
|
pop %rcx
|
|
pop %r11
|
|
jmp early_idt_handler_array + i*EARLY_IDT_HANDLER_SIZE
|
|
i = i + 1
|
|
.fill xen_early_idt_handler_array + i*XEN_EARLY_IDT_HANDLER_SIZE - ., 1, 0xcc
|
|
.endr
|
|
SYM_CODE_END(xen_early_idt_handler_array)
|
|
__FINIT
|
|
|
|
hypercall_iret = hypercall_page + __HYPERVISOR_iret * 32
|
|
/*
|
|
* Xen64 iret frame:
|
|
*
|
|
* ss
|
|
* rsp
|
|
* rflags
|
|
* cs
|
|
* rip <-- standard iret frame
|
|
*
|
|
* flags
|
|
*
|
|
* rcx }
|
|
* r11 }<-- pushed by hypercall page
|
|
* rsp->rax }
|
|
*/
|
|
SYM_CODE_START(xen_iret)
|
|
UNWIND_HINT_UNDEFINED
|
|
ANNOTATE_NOENDBR
|
|
pushq $0
|
|
jmp hypercall_iret
|
|
SYM_CODE_END(xen_iret)
|
|
|
|
/*
|
|
* XEN pv doesn't use trampoline stack, PER_CPU_VAR(cpu_tss_rw + TSS_sp0) is
|
|
* also the kernel stack. Reusing swapgs_restore_regs_and_return_to_usermode()
|
|
* in XEN pv would cause %rsp to move up to the top of the kernel stack and
|
|
* leave the IRET frame below %rsp, which is dangerous to be corrupted if #NMI
|
|
* interrupts. And swapgs_restore_regs_and_return_to_usermode() pushing the IRET
|
|
* frame at the same address is useless.
|
|
*/
|
|
SYM_CODE_START(xenpv_restore_regs_and_return_to_usermode)
|
|
UNWIND_HINT_REGS
|
|
POP_REGS
|
|
|
|
/* stackleak_erase() can work safely on the kernel stack. */
|
|
STACKLEAK_ERASE_NOCLOBBER
|
|
|
|
addq $8, %rsp /* skip regs->orig_ax */
|
|
jmp xen_iret
|
|
SYM_CODE_END(xenpv_restore_regs_and_return_to_usermode)
|
|
|
|
/*
|
|
* Xen handles syscall callbacks much like ordinary exceptions, which
|
|
* means we have:
|
|
* - kernel gs
|
|
* - kernel rsp
|
|
* - an iret-like stack frame on the stack (including rcx and r11):
|
|
* ss
|
|
* rsp
|
|
* rflags
|
|
* cs
|
|
* rip
|
|
* r11
|
|
* rsp->rcx
|
|
*/
|
|
|
|
/* Normal 64-bit system call target */
|
|
SYM_CODE_START(xen_entry_SYSCALL_64)
|
|
UNWIND_HINT_ENTRY
|
|
ENDBR
|
|
popq %rcx
|
|
popq %r11
|
|
|
|
/*
|
|
* Neither Xen nor the kernel really knows what the old SS and
|
|
* CS were. The kernel expects __USER_DS and __USER_CS, so
|
|
* report those values even though Xen will guess its own values.
|
|
*/
|
|
movq $__USER_DS, 4*8(%rsp)
|
|
movq $__USER_CS, 1*8(%rsp)
|
|
|
|
jmp entry_SYSCALL_64_after_hwframe
|
|
SYM_CODE_END(xen_entry_SYSCALL_64)
|
|
|
|
#ifdef CONFIG_IA32_EMULATION
|
|
|
|
/* 32-bit compat syscall target */
|
|
SYM_CODE_START(xen_entry_SYSCALL_compat)
|
|
UNWIND_HINT_ENTRY
|
|
ENDBR
|
|
popq %rcx
|
|
popq %r11
|
|
|
|
/*
|
|
* Neither Xen nor the kernel really knows what the old SS and
|
|
* CS were. The kernel expects __USER_DS and __USER32_CS, so
|
|
* report those values even though Xen will guess its own values.
|
|
*/
|
|
movq $__USER_DS, 4*8(%rsp)
|
|
movq $__USER32_CS, 1*8(%rsp)
|
|
|
|
jmp entry_SYSCALL_compat_after_hwframe
|
|
SYM_CODE_END(xen_entry_SYSCALL_compat)
|
|
|
|
/* 32-bit compat sysenter target */
|
|
SYM_CODE_START(xen_entry_SYSENTER_compat)
|
|
UNWIND_HINT_ENTRY
|
|
ENDBR
|
|
/*
|
|
* NB: Xen is polite and clears TF from EFLAGS for us. This means
|
|
* that we don't need to guard against single step exceptions here.
|
|
*/
|
|
popq %rcx
|
|
popq %r11
|
|
|
|
/*
|
|
* Neither Xen nor the kernel really knows what the old SS and
|
|
* CS were. The kernel expects __USER_DS and __USER32_CS, so
|
|
* report those values even though Xen will guess its own values.
|
|
*/
|
|
movq $__USER_DS, 4*8(%rsp)
|
|
movq $__USER32_CS, 1*8(%rsp)
|
|
|
|
jmp entry_SYSENTER_compat_after_hwframe
|
|
SYM_CODE_END(xen_entry_SYSENTER_compat)
|
|
|
|
#else /* !CONFIG_IA32_EMULATION */
|
|
|
|
SYM_CODE_START(xen_entry_SYSCALL_compat)
|
|
SYM_CODE_START(xen_entry_SYSENTER_compat)
|
|
UNWIND_HINT_ENTRY
|
|
ENDBR
|
|
lea 16(%rsp), %rsp /* strip %rcx, %r11 */
|
|
mov $-ENOSYS, %rax
|
|
pushq $0
|
|
jmp hypercall_iret
|
|
SYM_CODE_END(xen_entry_SYSENTER_compat)
|
|
SYM_CODE_END(xen_entry_SYSCALL_compat)
|
|
|
|
#endif /* CONFIG_IA32_EMULATION */
|