xen: make direct versions of irq_enable/disable/save/restore to common code
Now that x86-64 has directly accessible percpu variables, it can also
implement the direct versions of these operations, which operate on a
vcpu_info structure directly embedded in the percpu area.
In fact, the 64-bit versions are more or less identical, and so can be
shared. The only two differences are:
1. xen_restore_fl_direct takes its argument in eax on 32-bit, and rdi on 64-bit.
Unfortunately it isn't possible to directly refer to the 2nd lsb of rdi directly
(as you can with %ah), so the code isn't quite as dense.
2. check_events needs to variants to save different registers.
Signed-off-by: Jeremy Fitzhardinge <jeremy.fitzhardinge@citrix.com>
Signed-off-by: H. Peter Anvin <hpa@linux.intel.com>
This commit is contained in:
Jeremy Fitzhardinge
H. Peter Anvin
parent
383414322b
commit
5393744b71
@ -6,7 +6,8 @@ CFLAGS_REMOVE_irq.o = -pg
|
|||||||
endif
|
endif
|
||||||
|
|
||||||
obj-y := enlighten.o setup.o multicalls.o mmu.o irq.o \
|
obj-y := enlighten.o setup.o multicalls.o mmu.o irq.o \
|
||||||
time.o xen-asm_$(BITS).o grant-table.o suspend.o
|
time.o xen-asm.o xen-asm_$(BITS).o \
|
||||||
|
grant-table.o suspend.o
|
||||||
|
|
||||||
obj-$(CONFIG_SMP) += smp.o spinlock.o
|
obj-$(CONFIG_SMP) += smp.o spinlock.o
|
||||||
obj-$(CONFIG_XEN_DEBUG_FS) += debugfs.o
|
obj-$(CONFIG_XEN_DEBUG_FS) += debugfs.o
|
||||||
140
arch/x86/xen/xen-asm.S
Normal file
140
arch/x86/xen/xen-asm.S
Normal file
@ -0,0 +1,140 @@
|
|||||||
|
/*
|
||||||
|
Asm versions of Xen pv-ops, suitable for either direct use or inlining.
|
||||||
|
The inline versions are the same as the direct-use versions, with the
|
||||||
|
pre- and post-amble chopped off.
|
||||||
|
|
||||||
|
This code is encoded for size rather than absolute efficiency,
|
||||||
|
with a view to being able to inline as much as possible.
|
||||||
|
|
||||||
|
We only bother with direct forms (ie, vcpu in percpu data) of
|
||||||
|
the operations here; the indirect forms are better handled in
|
||||||
|
C, since they're generally too large to inline anyway.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#include <asm/asm-offsets.h>
|
||||||
|
#include <asm/percpu.h>
|
||||||
|
#include <asm/processor-flags.h>
|
||||||
|
|
||||||
|
#include "xen-asm.h"
|
||||||
|
|
||||||
|
/*
|
||||||
|
Enable events. This clears the event mask and tests the pending
|
||||||
|
event status with one and operation. If there are pending
|
||||||
|
events, then enter the hypervisor to get them handled.
|
||||||
|
*/
|
||||||
|
ENTRY(xen_irq_enable_direct)
|
||||||
|
/* Unmask events */
|
||||||
|
movb $0, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
|
||||||
|
|
||||||
|
/* Preempt here doesn't matter because that will deal with
|
||||||
|
any pending interrupts. The pending check may end up being
|
||||||
|
run on the wrong CPU, but that doesn't hurt. */
|
||||||
|
|
||||||
|
/* Test for pending */
|
||||||
|
testb $0xff, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_pending
|
||||||
|
jz 1f
|
||||||
|
|
||||||
|
2: call check_events
|
||||||
|
1:
|
||||||
|
ENDPATCH(xen_irq_enable_direct)
|
||||||
|
ret
|
||||||
|
ENDPROC(xen_irq_enable_direct)
|
||||||
|
RELOC(xen_irq_enable_direct, 2b+1)
|
||||||
|
|
||||||
|
|
||||||
|
/*
|
||||||
|
Disabling events is simply a matter of making the event mask
|
||||||
|
non-zero.
|
||||||
|
*/
|
||||||
|
ENTRY(xen_irq_disable_direct)
|
||||||
|
movb $1, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
|
||||||
|
ENDPATCH(xen_irq_disable_direct)
|
||||||
|
ret
|
||||||
|
ENDPROC(xen_irq_disable_direct)
|
||||||
|
RELOC(xen_irq_disable_direct, 0)
|
||||||
|
|
||||||
|
/*
|
||||||
|
(xen_)save_fl is used to get the current interrupt enable status.
|
||||||
|
Callers expect the status to be in X86_EFLAGS_IF, and other bits
|
||||||
|
may be set in the return value. We take advantage of this by
|
||||||
|
making sure that X86_EFLAGS_IF has the right value (and other bits
|
||||||
|
in that byte are 0), but other bits in the return value are
|
||||||
|
undefined. We need to toggle the state of the bit, because
|
||||||
|
Xen and x86 use opposite senses (mask vs enable).
|
||||||
|
*/
|
||||||
|
ENTRY(xen_save_fl_direct)
|
||||||
|
testb $0xff, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
|
||||||
|
setz %ah
|
||||||
|
addb %ah,%ah
|
||||||
|
ENDPATCH(xen_save_fl_direct)
|
||||||
|
ret
|
||||||
|
ENDPROC(xen_save_fl_direct)
|
||||||
|
RELOC(xen_save_fl_direct, 0)
|
||||||
|
|
||||||
|
|
||||||
|
/*
|
||||||
|
In principle the caller should be passing us a value return
|
||||||
|
from xen_save_fl_direct, but for robustness sake we test only
|
||||||
|
the X86_EFLAGS_IF flag rather than the whole byte. After
|
||||||
|
setting the interrupt mask state, it checks for unmasked
|
||||||
|
pending events and enters the hypervisor to get them delivered
|
||||||
|
if so.
|
||||||
|
*/
|
||||||
|
ENTRY(xen_restore_fl_direct)
|
||||||
|
#ifdef CONFIG_X86_64
|
||||||
|
testw $X86_EFLAGS_IF, %di
|
||||||
|
#else
|
||||||
|
testb $X86_EFLAGS_IF>>8, %ah
|
||||||
|
#endif
|
||||||
|
setz PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
|
||||||
|
/* Preempt here doesn't matter because that will deal with
|
||||||
|
any pending interrupts. The pending check may end up being
|
||||||
|
run on the wrong CPU, but that doesn't hurt. */
|
||||||
|
|
||||||
|
/* check for unmasked and pending */
|
||||||
|
cmpw $0x0001, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_pending
|
||||||
|
jz 1f
|
||||||
|
2: call check_events
|
||||||
|
1:
|
||||||
|
ENDPATCH(xen_restore_fl_direct)
|
||||||
|
ret
|
||||||
|
ENDPROC(xen_restore_fl_direct)
|
||||||
|
RELOC(xen_restore_fl_direct, 2b+1)
|
||||||
|
|
||||||
|
|
||||||
|
/*
|
||||||
|
Force an event check by making a hypercall,
|
||||||
|
but preserve regs before making the call.
|
||||||
|
*/
|
||||||
|
check_events:
|
||||||
|
#ifdef CONFIG_X86_32
|
||||||
|
push %eax
|
||||||
|
push %ecx
|
||||||
|
push %edx
|
||||||
|
call xen_force_evtchn_callback
|
||||||
|
pop %edx
|
||||||
|
pop %ecx
|
||||||
|
pop %eax
|
||||||
|
#else
|
||||||
|
push %rax
|
||||||
|
push %rcx
|
||||||
|
push %rdx
|
||||||
|
push %rsi
|
||||||
|
push %rdi
|
||||||
|
push %r8
|
||||||
|
push %r9
|
||||||
|
push %r10
|
||||||
|
push %r11
|
||||||
|
call xen_force_evtchn_callback
|
||||||
|
pop %r11
|
||||||
|
pop %r10
|
||||||
|
pop %r9
|
||||||
|
pop %r8
|
||||||
|
pop %rdi
|
||||||
|
pop %rsi
|
||||||
|
pop %rdx
|
||||||
|
pop %rcx
|
||||||
|
pop %rax
|
||||||
|
#endif
|
||||||
|
ret
|
||||||
|
|
||||||
12
arch/x86/xen/xen-asm.h
Normal file
12
arch/x86/xen/xen-asm.h
Normal file
@ -0,0 +1,12 @@
|
|||||||
|
#ifndef _XEN_XEN_ASM_H
|
||||||
|
#define _XEN_XEN_ASM_H
|
||||||
|
|
||||||
|
#include <linux/linkage.h>
|
||||||
|
|
||||||
|
#define RELOC(x, v) .globl x##_reloc; x##_reloc=v
|
||||||
|
#define ENDPATCH(x) .globl x##_end; x##_end=.
|
||||||
|
|
||||||
|
/* Pseudo-flag used for virtual NMI, which we don't implement yet */
|
||||||
|
#define XEN_EFLAGS_NMI 0x80000000
|
||||||
|
|
||||||
|
#endif
|
||||||
@ -11,101 +11,28 @@
|
|||||||
generally too large to inline anyway.
|
generally too large to inline anyway.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
#include <linux/linkage.h>
|
//#include <asm/asm-offsets.h>
|
||||||
|
|
||||||
#include <asm/asm-offsets.h>
|
|
||||||
#include <asm/thread_info.h>
|
#include <asm/thread_info.h>
|
||||||
#include <asm/percpu.h>
|
|
||||||
#include <asm/processor-flags.h>
|
#include <asm/processor-flags.h>
|
||||||
#include <asm/segment.h>
|
#include <asm/segment.h>
|
||||||
|
|
||||||
#include <xen/interface/xen.h>
|
#include <xen/interface/xen.h>
|
||||||
|
|
||||||
#define RELOC(x, v) .globl x##_reloc; x##_reloc=v
|
#include "xen-asm.h"
|
||||||
#define ENDPATCH(x) .globl x##_end; x##_end=.
|
|
||||||
|
|
||||||
/* Pseudo-flag used for virtual NMI, which we don't implement yet */
|
|
||||||
#define XEN_EFLAGS_NMI 0x80000000
|
|
||||||
|
|
||||||
/*
|
/*
|
||||||
Enable events. This clears the event mask and tests the pending
|
Force an event check by making a hypercall,
|
||||||
event status with one and operation. If there are pending
|
but preserve regs before making the call.
|
||||||
events, then enter the hypervisor to get them handled.
|
|
||||||
*/
|
*/
|
||||||
ENTRY(xen_irq_enable_direct)
|
check_events:
|
||||||
/* Unmask events */
|
push %eax
|
||||||
movb $0, PER_CPU_VAR(xen_vcpu_info)+XEN_vcpu_info_mask
|
push %ecx
|
||||||
|
push %edx
|
||||||
/* Preempt here doesn't matter because that will deal with
|
call xen_force_evtchn_callback
|
||||||
any pending interrupts. The pending check may end up being
|
pop %edx
|
||||||
run on the wrong CPU, but that doesn't hurt. */
|
pop %ecx
|
||||||
|
pop %eax
|
||||||
/* Test for pending */
|
|
||||||
testb $0xff, PER_CPU_VAR(xen_vcpu_info)+XEN_vcpu_info_pending
|
|
||||||
jz 1f
|
|
||||||
|
|
||||||
2: call check_events
|
|
||||||
1:
|
|
||||||
ENDPATCH(xen_irq_enable_direct)
|
|
||||||
ret
|
ret
|
||||||
ENDPROC(xen_irq_enable_direct)
|
|
||||||
RELOC(xen_irq_enable_direct, 2b+1)
|
|
||||||
|
|
||||||
|
|
||||||
/*
|
|
||||||
Disabling events is simply a matter of making the event mask
|
|
||||||
non-zero.
|
|
||||||
*/
|
|
||||||
ENTRY(xen_irq_disable_direct)
|
|
||||||
movb $1, PER_CPU_VAR(xen_vcpu_info)+XEN_vcpu_info_mask
|
|
||||||
ENDPATCH(xen_irq_disable_direct)
|
|
||||||
ret
|
|
||||||
ENDPROC(xen_irq_disable_direct)
|
|
||||||
RELOC(xen_irq_disable_direct, 0)
|
|
||||||
|
|
||||||
/*
|
|
||||||
(xen_)save_fl is used to get the current interrupt enable status.
|
|
||||||
Callers expect the status to be in X86_EFLAGS_IF, and other bits
|
|
||||||
may be set in the return value. We take advantage of this by
|
|
||||||
making sure that X86_EFLAGS_IF has the right value (and other bits
|
|
||||||
in that byte are 0), but other bits in the return value are
|
|
||||||
undefined. We need to toggle the state of the bit, because
|
|
||||||
Xen and x86 use opposite senses (mask vs enable).
|
|
||||||
*/
|
|
||||||
ENTRY(xen_save_fl_direct)
|
|
||||||
testb $0xff, PER_CPU_VAR(xen_vcpu_info)+XEN_vcpu_info_mask
|
|
||||||
setz %ah
|
|
||||||
addb %ah,%ah
|
|
||||||
ENDPATCH(xen_save_fl_direct)
|
|
||||||
ret
|
|
||||||
ENDPROC(xen_save_fl_direct)
|
|
||||||
RELOC(xen_save_fl_direct, 0)
|
|
||||||
|
|
||||||
|
|
||||||
/*
|
|
||||||
In principle the caller should be passing us a value return
|
|
||||||
from xen_save_fl_direct, but for robustness sake we test only
|
|
||||||
the X86_EFLAGS_IF flag rather than the whole byte. After
|
|
||||||
setting the interrupt mask state, it checks for unmasked
|
|
||||||
pending events and enters the hypervisor to get them delivered
|
|
||||||
if so.
|
|
||||||
*/
|
|
||||||
ENTRY(xen_restore_fl_direct)
|
|
||||||
testb $X86_EFLAGS_IF>>8, %ah
|
|
||||||
setz PER_CPU_VAR(xen_vcpu_info)+XEN_vcpu_info_mask
|
|
||||||
/* Preempt here doesn't matter because that will deal with
|
|
||||||
any pending interrupts. The pending check may end up being
|
|
||||||
run on the wrong CPU, but that doesn't hurt. */
|
|
||||||
|
|
||||||
/* check for unmasked and pending */
|
|
||||||
cmpw $0x0001, PER_CPU_VAR(xen_vcpu_info)+XEN_vcpu_info_pending
|
|
||||||
jz 1f
|
|
||||||
2: call check_events
|
|
||||||
1:
|
|
||||||
ENDPATCH(xen_restore_fl_direct)
|
|
||||||
ret
|
|
||||||
ENDPROC(xen_restore_fl_direct)
|
|
||||||
RELOC(xen_restore_fl_direct, 2b+1)
|
|
||||||
|
|
||||||
/*
|
/*
|
||||||
We can't use sysexit directly, because we're not running in ring0.
|
We can't use sysexit directly, because we're not running in ring0.
|
||||||
@ -289,17 +216,3 @@ ENTRY(xen_iret_crit_fixup)
|
|||||||
lea 4(%edi),%esp /* point esp to new frame */
|
lea 4(%edi),%esp /* point esp to new frame */
|
||||||
2: jmp xen_do_upcall
|
2: jmp xen_do_upcall
|
||||||
|
|
||||||
|
|
||||||
/*
|
|
||||||
Force an event check by making a hypercall,
|
|
||||||
but preserve regs before making the call.
|
|
||||||
*/
|
|
||||||
check_events:
|
|
||||||
push %eax
|
|
||||||
push %ecx
|
|
||||||
push %edx
|
|
||||||
call xen_force_evtchn_callback
|
|
||||||
pop %edx
|
|
||||||
pop %ecx
|
|
||||||
pop %eax
|
|
||||||
ret
|
|
||||||
|
|||||||
@ -11,142 +11,14 @@
|
|||||||
generally too large to inline anyway.
|
generally too large to inline anyway.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
#include <linux/linkage.h>
|
|
||||||
|
|
||||||
#include <asm/asm-offsets.h>
|
|
||||||
#include <asm/processor-flags.h>
|
|
||||||
#include <asm/errno.h>
|
#include <asm/errno.h>
|
||||||
#include <asm/segment.h>
|
|
||||||
#include <asm/percpu.h>
|
#include <asm/percpu.h>
|
||||||
|
#include <asm/processor-flags.h>
|
||||||
|
#include <asm/segment.h>
|
||||||
|
|
||||||
#include <xen/interface/xen.h>
|
#include <xen/interface/xen.h>
|
||||||
|
|
||||||
#define RELOC(x, v) .globl x##_reloc; x##_reloc=v
|
#include "xen-asm.h"
|
||||||
#define ENDPATCH(x) .globl x##_end; x##_end=.
|
|
||||||
|
|
||||||
/* Pseudo-flag used for virtual NMI, which we don't implement yet */
|
|
||||||
#define XEN_EFLAGS_NMI 0x80000000
|
|
||||||
|
|
||||||
#if 1
|
|
||||||
/*
|
|
||||||
FIXME: x86_64 now can support direct access to percpu variables
|
|
||||||
via a segment override. Update xen accordingly.
|
|
||||||
*/
|
|
||||||
#define BUG ud2a
|
|
||||||
#endif
|
|
||||||
|
|
||||||
/*
|
|
||||||
Enable events. This clears the event mask and tests the pending
|
|
||||||
event status with one and operation. If there are pending
|
|
||||||
events, then enter the hypervisor to get them handled.
|
|
||||||
*/
|
|
||||||
ENTRY(xen_irq_enable_direct)
|
|
||||||
BUG
|
|
||||||
|
|
||||||
/* Unmask events */
|
|
||||||
movb $0, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
|
|
||||||
|
|
||||||
/* Preempt here doesn't matter because that will deal with
|
|
||||||
any pending interrupts. The pending check may end up being
|
|
||||||
run on the wrong CPU, but that doesn't hurt. */
|
|
||||||
|
|
||||||
/* Test for pending */
|
|
||||||
testb $0xff, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_pending
|
|
||||||
jz 1f
|
|
||||||
|
|
||||||
2: call check_events
|
|
||||||
1:
|
|
||||||
ENDPATCH(xen_irq_enable_direct)
|
|
||||||
ret
|
|
||||||
ENDPROC(xen_irq_enable_direct)
|
|
||||||
RELOC(xen_irq_enable_direct, 2b+1)
|
|
||||||
|
|
||||||
/*
|
|
||||||
Disabling events is simply a matter of making the event mask
|
|
||||||
non-zero.
|
|
||||||
*/
|
|
||||||
ENTRY(xen_irq_disable_direct)
|
|
||||||
BUG
|
|
||||||
|
|
||||||
movb $1, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
|
|
||||||
ENDPATCH(xen_irq_disable_direct)
|
|
||||||
ret
|
|
||||||
ENDPROC(xen_irq_disable_direct)
|
|
||||||
RELOC(xen_irq_disable_direct, 0)
|
|
||||||
|
|
||||||
/*
|
|
||||||
(xen_)save_fl is used to get the current interrupt enable status.
|
|
||||||
Callers expect the status to be in X86_EFLAGS_IF, and other bits
|
|
||||||
may be set in the return value. We take advantage of this by
|
|
||||||
making sure that X86_EFLAGS_IF has the right value (and other bits
|
|
||||||
in that byte are 0), but other bits in the return value are
|
|
||||||
undefined. We need to toggle the state of the bit, because
|
|
||||||
Xen and x86 use opposite senses (mask vs enable).
|
|
||||||
*/
|
|
||||||
ENTRY(xen_save_fl_direct)
|
|
||||||
BUG
|
|
||||||
|
|
||||||
testb $0xff, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
|
|
||||||
setz %ah
|
|
||||||
addb %ah,%ah
|
|
||||||
ENDPATCH(xen_save_fl_direct)
|
|
||||||
ret
|
|
||||||
ENDPROC(xen_save_fl_direct)
|
|
||||||
RELOC(xen_save_fl_direct, 0)
|
|
||||||
|
|
||||||
/*
|
|
||||||
In principle the caller should be passing us a value return
|
|
||||||
from xen_save_fl_direct, but for robustness sake we test only
|
|
||||||
the X86_EFLAGS_IF flag rather than the whole byte. After
|
|
||||||
setting the interrupt mask state, it checks for unmasked
|
|
||||||
pending events and enters the hypervisor to get them delivered
|
|
||||||
if so.
|
|
||||||
*/
|
|
||||||
ENTRY(xen_restore_fl_direct)
|
|
||||||
BUG
|
|
||||||
|
|
||||||
testb $X86_EFLAGS_IF>>8, %ah
|
|
||||||
setz PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
|
|
||||||
/* Preempt here doesn't matter because that will deal with
|
|
||||||
any pending interrupts. The pending check may end up being
|
|
||||||
run on the wrong CPU, but that doesn't hurt. */
|
|
||||||
|
|
||||||
/* check for unmasked and pending */
|
|
||||||
cmpw $0x0001, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_pending
|
|
||||||
jz 1f
|
|
||||||
2: call check_events
|
|
||||||
1:
|
|
||||||
ENDPATCH(xen_restore_fl_direct)
|
|
||||||
ret
|
|
||||||
ENDPROC(xen_restore_fl_direct)
|
|
||||||
RELOC(xen_restore_fl_direct, 2b+1)
|
|
||||||
|
|
||||||
|
|
||||||
/*
|
|
||||||
Force an event check by making a hypercall,
|
|
||||||
but preserve regs before making the call.
|
|
||||||
*/
|
|
||||||
check_events:
|
|
||||||
push %rax
|
|
||||||
push %rcx
|
|
||||||
push %rdx
|
|
||||||
push %rsi
|
|
||||||
push %rdi
|
|
||||||
push %r8
|
|
||||||
push %r9
|
|
||||||
push %r10
|
|
||||||
push %r11
|
|
||||||
call xen_force_evtchn_callback
|
|
||||||
pop %r11
|
|
||||||
pop %r10
|
|
||||||
pop %r9
|
|
||||||
pop %r8
|
|
||||||
pop %rdi
|
|
||||||
pop %rsi
|
|
||||||
pop %rdx
|
|
||||||
pop %rcx
|
|
||||||
pop %rax
|
|
||||||
ret
|
|
||||||
|
|
||||||
ENTRY(xen_adjust_exception_frame)
|
ENTRY(xen_adjust_exception_frame)
|
||||||
mov 8+0(%rsp),%rcx
|
mov 8+0(%rsp),%rcx
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user