# HG changeset patch # User yamahata@xxxxxxxxxxxxx # Date 1184315140 -32400 # Node ID 28aaf94683193e2e50c3d4efd53abb913b7bb321 # Parent 86ac3059ab675a1c80ab02be513afb57b5e63efa binary patch for hyperprivops. PATCHNAME: binary_patch_for_hyperprivops Signed-off-by: Isaku Yamahata diff -r 86ac3059ab67 -r 28aaf9468319 arch/ia64/Kconfig --- a/arch/ia64/Kconfig Tue Jul 10 11:18:07 2007 -0600 +++ b/arch/ia64/Kconfig Fri Jul 13 17:25:40 2007 +0900 @@ -85,6 +85,47 @@ config XEN_IA64_EXPOSE_P2M_USE_DTR default y help use dtr to map the exposed p2m table + +config PARAVIRT_ALT + bool "hyperprivop binary patch" + depends on XEN + default y + help + hyperprivop binary patch. + There are several sensitive(i.e. non-virtualizable) instructions and + performance critical privileged instructions which Xen + paravirtualize as hyperprivops. + For transparent paravirtualization (i.e. single binary should run + on both baremetal and xen environment), xenLinux/IA64 needs + something like "if (is_running_on_xen()) {} else {}" where + is_running_on_xen() is determined at boot time. + This configuration tries to eliminate the overheads for hyperprivops + by annotating such instructions and replacing them with hyperprivops + at boot time. + +config PARAVIRT_NOP_B_PATCH + bool "paravirt branch if native" + depends on XEN + default y + help + paravirt branch if native + There are several paravirtualized paths in hand coded assembly code. + For transparent paravirtualization, there are codes like + GLOBAL_ENTRY(xen_xxx) + 'movl reg=running_on_xen;;' + 'ld4 reg=[reg];;' + 'cmp.e1 pred,p0=reg,r0' + '(pred) br.cond.sptk.many ;;' + To reduce overhead when running on bare metal, just + "br.cond.sptk.many " and replace it with 'nop.b 0' + when running on xen. + +config PARAVIRT_ENTRY + bool "paravirt entry" + depends on XEN + default y + help + paravirt entry. generalization of PARAVIRT_NOP_B_PATCH. config SCHED_NO_NO_OMIT_FRAME_POINTER bool diff -r 86ac3059ab67 -r 28aaf9468319 arch/ia64/kernel/module.c --- a/arch/ia64/kernel/module.c Tue Jul 10 11:18:07 2007 -0600 +++ b/arch/ia64/kernel/module.c Fri Jul 13 17:25:40 2007 +0900 @@ -454,6 +454,14 @@ module_frob_arch_sections (Elf_Ehdr *ehd mod->arch.opd = s; else if (strcmp(".IA_64.unwind", secstrings + s->sh_name) == 0) mod->arch.unwind = s; +#ifdef CONFIG_PARAVIRT_ALT + else if (strcmp(".paravirt_bundles", + secstrings + s->sh_name) == 0) + mod->arch.paravirt_bundles = s; + else if (strcmp(".paravirt_insts", + secstrings + s->sh_name) == 0) + mod->arch.paravirt_insts = s; +#endif if (!mod->arch.core_plt || !mod->arch.init_plt || !mod->arch.got || !mod->arch.opd) { printk(KERN_ERR "%s: sections missing\n", mod->name); @@ -929,6 +937,30 @@ module_finalize (const Elf_Ehdr *hdr, co DEBUGP("%s: init: entry=%p\n", __FUNCTION__, mod->init); if (mod->arch.unwind) register_unwind_table(mod); +#ifdef CONFIG_PARAVIRT_ALT + if (mod->arch.paravirt_bundles) { + struct paravirt_alt_bundle_patch *start = + (struct paravirt_alt_bundle_patch*) + mod->arch.paravirt_bundles->sh_addr; + struct paravirt_alt_bundle_patch *end = + (struct paravirt_alt_bundle_patch*) + (mod->arch.paravirt_bundles->sh_addr + + mod->arch.paravirt_bundles->sh_size); + + xen_alt_bundle_patch_module(start, end); + } + if (mod->arch.paravirt_insts) { + struct paravirt_alt_inst_patch *start = + (struct paravirt_alt_inst_patch*) + mod->arch.paravirt_insts->sh_addr; + struct paravirt_alt_inst_patch *end = + (struct paravirt_alt_inst_patch*) + (mod->arch.paravirt_insts->sh_addr + + mod->arch.paravirt_insts->sh_size); + + xen_alt_inst_patch_module(start, end); + } +#endif return 0; } diff -r 86ac3059ab67 -r 28aaf9468319 arch/ia64/kernel/vmlinux.lds.S --- a/arch/ia64/kernel/vmlinux.lds.S Tue Jul 10 11:18:07 2007 -0600 +++ b/arch/ia64/kernel/vmlinux.lds.S Fri Jul 13 17:25:40 2007 +0900 @@ -151,6 +151,42 @@ SECTIONS *(.data.patch.mckinley_e9) __end___mckinley_e9_bundles = .; } + +#if defined(CONFIG_PARAVIRT_ALT) + . = ALIGN(16); + .paravirt_bundles : AT(ADDR(.paravirt_bundles) - LOAD_OFFSET) + { + __start_paravirt_bundles = .; + *(.paravirt_bundles) + __stop_paravirt_bundles = .; + } + . = ALIGN(16); + .paravirt_insts : AT(ADDR(.paravirt_insts) - LOAD_OFFSET) + { + __start_paravirt_insts = .; + *(.paravirt_insts) + __stop_paravirt_insts = .; + } +#endif +#if defined(CONFIG_PARAVIRT_NOP_B_PATCH) + . = ALIGN(16); + .paravirt_nop_b : AT(ADDR(.paravirt_nop_b) - LOAD_OFFSET) + { + __start_paravirt_nop_b = .; + *(.paravirt_nop_b) + __stop_paravirt_nop_b = .; + + } +#endif +#if defined(CONFIG_PARAVIRT_ENTRY) + . = ALIGN(16); + .paravirt_entry : AT(ADDR(.paravirt_entry) - LOAD_OFFSET) + { + __start_paravirt_entry = .; + *(.paravirt_entry) + __stop_paravirt_entry = .; + } +#endif #if defined(CONFIG_IA64_GENERIC) /* Machine Vector */ diff -r 86ac3059ab67 -r 28aaf9468319 arch/ia64/xen/Makefile --- a/arch/ia64/xen/Makefile Tue Jul 10 11:18:07 2007 -0600 +++ b/arch/ia64/xen/Makefile Fri Jul 13 17:25:40 2007 +0900 @@ -7,3 +7,9 @@ obj-y := hypercall.o xenivt.o xenentry.o xcom_privcmd.o mem.o xen_dma.o obj-$(CONFIG_IA64_GENERIC) += machvec.o +obj-$(CONFIG_PARAVIRT_ALT) += paravirt_core.o paravirt_xen.o \ + paravirt_alt.o privop_s.o +obj-$(CONFIG_PARAVIRT_NOP_B_PATCH) += paravirt_core.o paravirt_xen.o \ + paravirt_nop.o +obj-$(CONFIG_PARAVIRT_ENTRY) += paravirt_core.o paravirt_xen.o \ + paravirt_entry.o paravirtentry.o diff -r 86ac3059ab67 -r 28aaf9468319 arch/ia64/xen/paravirt_alt.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/arch/ia64/xen/paravirt_alt.c Fri Jul 13 17:25:40 2007 +0900 @@ -0,0 +1,91 @@ +/****************************************************************************** + * linux/arch/ia64/xen/paravirt_alt.c + * + * Copyright (c) 2007 Isaku Yamahata + * VA Linux Systems Japan K.K. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + */ + +#include + +extern const char nop_bundle[]; +extern const unsigned long nop_bundle_size; + +static void __init_or_module +fill_nop(void* bundle, unsigned long size) +{ + BUG_ON((((unsigned long)bundle) % sizeof(bundle_t)) != 0); + BUG_ON((size % nop_bundle_size) != 0); + + while (size >= nop_bundle_size) { + memcpy(bundle, nop_bundle, nop_bundle_size); + + bundle += nop_bundle_size; + size -= nop_bundle_size; + } +} + +void __init_or_module +paravirt_alt_bundle_patch_apply(struct paravirt_alt_bundle_patch* start, + struct paravirt_alt_bundle_patch* end, + unsigned long(*patch)(void* bundle, + unsigned long size, + unsigned long type)) +{ + struct paravirt_alt_bundle_patch *p; + + for (p = start; p < end; p++) { + unsigned long used; + + used = (*patch)(p->bundle, p->size, p->type); + if (used == 0) + continue; + + fill_nop(p->bundle + used, p->size - used); + paravirt_flush_i_cache_range(p->bundle, p->size); + } + ia64_sync_i(); + ia64_srlz_i(); +} + +void __init_or_module +paravirt_alt_inst_patch_apply(struct paravirt_alt_inst_patch* start, + struct paravirt_alt_inst_patch* end, + void (*patch)(unsigned long tag, + unsigned long type)) +{ + struct paravirt_alt_inst_patch *p; + + for (p = start; p < end; p++) { + (*patch)(p->tag, p->type); + paravirt_flush_i_cache_range(paravirt_get_bundle(p->tag), + sizeof(bundle_t)); + } + + ia64_sync_i(); + ia64_srlz_i(); +} + +/* + * Local variables: + * mode: C + * c-set-style: "linux" + * c-basic-offset: 8 + * tab-width: 8 + * indent-tabs-mode: t + * End: + */ diff -r 86ac3059ab67 -r 28aaf9468319 arch/ia64/xen/paravirt_core.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/arch/ia64/xen/paravirt_core.c Fri Jul 13 17:25:40 2007 +0900 @@ -0,0 +1,165 @@ +/****************************************************************************** + * linux/arch/ia64/xen/paravirt_core.c + * + * Copyright (c) 2007 Isaku Yamahata + * VA Linux Systems Japan K.K. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + */ + +#include + +/* + * flush_icache_range() can't be used here. + * we are here before cpu_init() which initializes + * ia64_i_cache_stride_shift. flush_icache_range() uses it. + */ +void __init_or_module +paravirt_flush_i_cache_range(const void* instr, unsigned long size) +{ + unsigned long i; + + for (i = 0; i < size; i += sizeof(bundle_t)) + asm volatile ("fc.i %0":: "r"(instr + i): "memory"); +} + +bundle_t* __init_or_module +paravirt_get_bundle(unsigned long tag) +{ + return (bundle_t*)(tag & ~3UL); +} + +unsigned long __init_or_module +paravirt_get_slot(unsigned long tag) +{ + return tag & 3UL; +} + +cmp_inst_t __init_or_module +paravirt_read_slot0(const bundle_t *bundle) +{ + cmp_inst_t inst; + inst.l = bundle->quad0.slot0; + return inst; +} + +cmp_inst_t __init_or_module +paravirt_read_slot1(const bundle_t *bundle) +{ + cmp_inst_t inst; + inst.l = bundle->quad0.slot1_p0 | + ((unsigned long long)bundle->quad1.slot1_p1 << 18UL); + return inst; +} + +cmp_inst_t __init_or_module +paravirt_read_slot2(const bundle_t *bundle) +{ + cmp_inst_t inst; + inst.l = bundle->quad1.slot2; + return inst; +} + +cmp_inst_t __init_or_module +paravirt_read_inst(unsigned long tag) +{ + bundle_t *bundle = paravirt_get_bundle(tag); + unsigned long slot = paravirt_get_slot(tag); + + switch (slot) { + case 0: + return paravirt_read_slot0(bundle); + case 1: + return paravirt_read_slot1(bundle); + case 2: + return paravirt_read_slot2(bundle); + default: + BUG(); + } + /* NOTREACHED */ +} + +void __init_or_module +paravirt_write_slot0(bundle_t *bundle, cmp_inst_t inst) +{ + bundle->quad0.slot0 = inst.l; +} + +void __init_or_module +paravirt_write_slot1(bundle_t *bundle, cmp_inst_t inst) +{ + bundle->quad0.slot1_p0 = inst.l; + bundle->quad1.slot1_p1 = inst.l >> 18UL; +} + +void __init_or_module +paravirt_write_slot2(bundle_t *bundle, cmp_inst_t inst) +{ + bundle->quad1.slot2 = inst.l; +} + +void __init_or_module +paravirt_write_inst(unsigned long tag, cmp_inst_t inst) +{ + bundle_t *bundle = paravirt_get_bundle(tag); + unsigned long slot = paravirt_get_slot(tag); + + switch (slot) { + case 0: + paravirt_write_slot0(bundle, inst); + break; + case 1: + paravirt_write_slot1(bundle, inst); + break; + case 2: + paravirt_write_slot2(bundle, inst); + break; + default: + BUG(); + } + paravirt_flush_i_cache_range(bundle, sizeof(*bundle)); +} + +/* for debug */ +void +print_bundle(const bundle_t *bundle) +{ + const unsigned long *quad = (const unsigned long*)bundle; + cmp_inst_t slot0 = paravirt_read_slot0(bundle); + cmp_inst_t slot1 = paravirt_read_slot1(bundle); + cmp_inst_t slot2 = paravirt_read_slot2(bundle); + + printk("bundle 0x%p 0x%016lx 0x%016lx\n", bundle, quad[0], quad[1]); + printk("bundle template 0x%x\n", + bundle->quad0.template); + printk("slot0 0x%lx slot1_p0 0x%lx slot1_p1 0x%lx slot2 0x%lx\n", + (unsigned long)bundle->quad0.slot0, + (unsigned long)bundle->quad0.slot1_p0, + (unsigned long)bundle->quad1.slot1_p1, + (unsigned long)bundle->quad1.slot2); + printk("slot0 0x%016llx slot1 0x%016llx slot2 0x%016llx\n", + slot0.l, slot1.l, slot2.l); +} + +/* + * Local variables: + * mode: C + * c-set-style: "linux" + * c-basic-offset: 8 + * tab-width: 8 + * indent-tabs-mode: t + * End: + */ diff -r 86ac3059ab67 -r 28aaf9468319 arch/ia64/xen/paravirt_entry.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/arch/ia64/xen/paravirt_entry.c Fri Jul 13 17:25:40 2007 +0900 @@ -0,0 +1,101 @@ +/****************************************************************************** + * Copyright (c) 2007 Isaku Yamahata + * VA Linux Systems Japan K.K. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + */ + +#include +#include + +/* br.cond.sptk.many B1 */ +typedef union inst_b1 { + cmp_inst_t inst; + struct + { + unsigned long qp: 6; + unsigned long bypte: 3; + unsigned long unused: 3; + unsigned long p: 1; + unsigned long imm20b: 20; + unsigned long wh: 2; + unsigned long d: 1; + unsigned long s: 1; + unsigned long opcode: 4; + }; + unsigned long l; +} inst_b1_t; + +static void __init +__paravirt_entry_apply(unsigned long tag, const void* target) +{ + bundle_t *bundle = paravirt_get_bundle(tag); + cmp_inst_t inst = paravirt_read_inst(tag); + unsigned long target25 = (unsigned long)target - (unsigned long)bundle; + inst_b1_t inst_b1; + + inst_b1.l = inst.l; + if (target25 & (1UL << 63)) { + inst_b1.s = 1; + } else { + inst_b1.s = 0; + } + + inst_b1.imm20b = target25 >> 4; + inst.l = inst_b1.l; + + paravirt_write_inst(tag, inst); + paravirt_flush_i_cache_range(bundle, sizeof(*bundle)); +} + +static void __init +paravirt_entry_apply(const struct paravirt_entry_patch* entry_patch, + const struct paravirt_entry* entries, + unsigned int nr_entries) +{ + unsigned int i; + for (i = 0; i < nr_entries; i++) { + if (entry_patch->type == entries[i].type) { + __paravirt_entry_apply(entry_patch->tag, + entries[i].entry); + break; + } + } +} + +void __init +paravirt_entry_patch_apply(const struct paravirt_entry_patch* start, + const struct paravirt_entry_patch* end, + const struct paravirt_entry* entries, + unsigned int nr_entries) +{ + const struct paravirt_entry_patch* p; + for (p = start; p < end; p++) + paravirt_entry_apply(p, entries, nr_entries); + + ia64_sync_i(); + ia64_srlz_i(); +} + +/* + * Local variables: + * mode: C + * c-set-style: "linux" + * c-basic-offset: 8 + * tab-width: 8 + * indent-tabs-mode: t + * End: + */ diff -r 86ac3059ab67 -r 28aaf9468319 arch/ia64/xen/paravirt_nop.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/arch/ia64/xen/paravirt_nop.c Fri Jul 13 17:25:40 2007 +0900 @@ -0,0 +1,49 @@ +/****************************************************************************** + * linux/arch/ia64/xen/paravirt_nop.c + * + * Copyright (c) 2007 Isaku Yamahata + * VA Linux Systems Japan K.K. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + */ + +#include +#include + +void __init_or_module +paravirt_nop_b_patch_apply(const struct paravirt_nop_patch *start, + const struct paravirt_nop_patch *end) +{ + extern const bundle_t nop_b_inst_bundle; + const cmp_inst_t nop_b_inst = paravirt_read_slot0(&nop_b_inst_bundle); + const struct paravirt_nop_patch *p; + + for (p = start; p < end; p++) + paravirt_write_inst(p->tag, nop_b_inst); + + ia64_sync_i(); + ia64_srlz_i(); +} + +/* + * Local variables: + * mode: C + * c-set-style: "linux" + * c-basic-offset: 8 + * tab-width: 8 + * indent-tabs-mode: t + * End: + */ diff -r 86ac3059ab67 -r 28aaf9468319 arch/ia64/xen/paravirt_xen.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/arch/ia64/xen/paravirt_xen.c Fri Jul 13 17:25:40 2007 +0900 @@ -0,0 +1,227 @@ +/****************************************************************************** + * linux/arch/ia64/xen/paravirt_xen.c + * + * Copyright (c) 2007 Isaku Yamahata + * VA Linux Systems Japan K.K. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + */ + +#include +#include +#include +#include +#include +#include /* for bundle_t */ +#include + +#ifdef CONFIG_PARAVIRT_ALT +struct xen_alt_bundle_patch_elem { + const void* bundle; + unsigned long size; + unsigned long type; +}; + +static unsigned long __init_or_module +__xen_alt_bundle_patch(void* bundle, unsigned long size, unsigned long type) +{ + extern const struct xen_alt_bundle_patch_elem xen_alt_bundle_array[]; + extern const unsigned long xen_alt_bundle_array_size; + + unsigned long used = 0; + unsigned long i; + + BUG_ON((((unsigned long)bundle) % sizeof(bundle_t)) != 0); + BUG_ON((size % sizeof(bundle_t)) != 0); + + for (i = 0; + i < xen_alt_bundle_array_size / sizeof (xen_alt_bundle_array[0]); + i++) { + const struct xen_alt_bundle_patch_elem* p = + &xen_alt_bundle_array[i]; + if (p->type == type) { + BUG_ON(p->size > size); + used = p->size; + memcpy(bundle, p->bundle, p->size); + break; + } + } + + return used; +} + +static void __init +xen_alt_bundle_patch(void) +{ + extern struct paravirt_alt_bundle_patch __start_paravirt_bundles[]; + extern struct paravirt_alt_bundle_patch __stop_paravirt_bundles[]; + + paravirt_alt_bundle_patch_apply(__start_paravirt_bundles, + __stop_paravirt_bundles, + &__xen_alt_bundle_patch); +} + +#ifdef CONFIG_MODULES +void +xen_alt_bundle_patch_module(struct paravirt_alt_bundle_patch *start, + struct paravirt_alt_bundle_patch *end) +{ + if (is_running_on_xen()) + paravirt_alt_bundle_patch_apply(start, end, + &__xen_alt_bundle_patch); +} +#endif /* CONFIG_MODULES */ + + +/* + * all the native instructions of hyperprivops are M-form or I-form + * mov ar.=r1 I26, M29 + * mov r1=ar. I28, M31 + * mov r1=cr. M32 + * mov cr.=r1 M33 + * mov r1=psr M36 + * mov indirect=r2 M42 + * mov r1=indirect M43 + * ptc.ga M45 + * thash r1=r2 M46 + * + * break.{m, i} instrucitions format are same. + * So we can safely replace all signle instruction which is target of + * hyperpviops with break.{m, i} imm21 hyperprivops. + */ +struct xen_alt_inst_patch_elem { + const void* bundle; + unsigned long type; +}; + +static void __init_or_module +__xen_alt_inst_patch(unsigned long tag, unsigned long type) +{ + extern const struct xen_alt_inst_patch_elem xen_alt_inst_array[]; + extern const unsigned long xen_alt_inst_array_size; + + unsigned long i; + for (i = 0; + i < xen_alt_inst_array_size / sizeof (xen_alt_inst_array[0]); + i++) { + const struct xen_alt_inst_patch_elem* p = + &xen_alt_inst_array[i]; + if (p->type == type) { + const cmp_inst_t inst = paravirt_read_slot0(p->bundle); + paravirt_write_inst(tag, inst); + break; + } + } +} + +static void __init +xen_alt_inst_patch(void) +{ + extern struct paravirt_alt_inst_patch __start_paravirt_insts[]; + extern struct paravirt_alt_inst_patch __stop_paravirt_insts[]; + + paravirt_alt_inst_patch_apply(__start_paravirt_insts, + __stop_paravirt_insts, + &__xen_alt_inst_patch); +} + +#ifdef CONFIG_MODULES +void +xen_alt_inst_patch_module(struct paravirt_alt_inst_patch *start, + struct paravirt_alt_inst_patch *end) +{ + if (is_running_on_xen()) + paravirt_alt_inst_patch_apply(start, end, + &__xen_alt_inst_patch); +} +#endif /* CONFIG_MODULES */ + +#else +#define xen_alt_bundle_patch() do { } while (0) +#define xen_alt_inst_patch() do { } while (0) +#endif /* CONFIG_PARAVIRT_ALT */ + + +#ifdef CONFIG_PARAVIRT_NOP_B_PATCH +#include +static void __init +xen_nop_b_patch(void) +{ + extern const struct paravirt_nop_patch __start_paravirt_nop_b[]; + extern const struct paravirt_nop_patch __stop_paravirt_nop_b[]; + + paravirt_nop_b_patch_apply(__start_paravirt_nop_b, + __stop_paravirt_nop_b); +} +#else +#define xen_nop_b_patch() do { } while (0) +#endif + + +#ifdef CONFIG_PARAVIRT_ENTRY + +#include + +extern void* xen_switch_to; +extern void* xen_trace_syscall; +extern void* xen_ret_from_clone; +extern void* xen_leave_syscall; +extern void* xen_leave_kernel; +extern void* xen_pal_call_static; + +const static struct paravirt_entry xen_entries[] __initdata = { + {&xen_switch_to, PARAVIRT_ET_SWITCH_TO}, + {&xen_trace_syscall, PARAVIRT_ET_TRACE_SYSCALL}, + {&xen_ret_from_clone, PARAVIRT_ET_RET_FROM_CLONE}, + {&xen_leave_syscall, PARAVIRT_ET_LEAVE_SYSCALL}, + {&xen_leave_kernel, PARAVIRT_ET_LEAVE_KERNEL}, + {&xen_pal_call_static, PARAVIRT_ET_PAL_CALL_STATIC}, +}; + +void __init +xen_entry_patch(void) +{ + extern const struct paravirt_entry_patch __start_paravirt_entry[]; + extern const struct paravirt_entry_patch __stop_paravirt_entry[]; + + paravirt_entry_patch_apply(__start_paravirt_entry, + __stop_paravirt_entry, + xen_entries, + sizeof(xen_entries)/sizeof(xen_entries[0])); +} +#else +#define xen_entry_patch() do { } while (0) +#endif + + +void __init +xen_paravirt_patch(void) +{ + xen_alt_bundle_patch(); + xen_alt_inst_patch(); + xen_nop_b_patch(); + xen_entry_patch(); +} + +/* + * Local variables: + * mode: C + * c-set-style: "linux" + * c-basic-offset: 8 + * tab-width: 8 + * indent-tabs-mode: t + * End: + */ diff -r 86ac3059ab67 -r 28aaf9468319 arch/ia64/xen/paravirtentry.S --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/arch/ia64/xen/paravirtentry.S Fri Jul 13 17:25:40 2007 +0900 @@ -0,0 +1,37 @@ +/****************************************************************************** + * linux/arch/ia64/xen/paravirtentry.S + * + * Copyright (c) 2007 Isaku Yamahata + * VA Linux Systems Japan K.K. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + */ + +#include +#include +#include + +#define BRANCH(sym, type) \ + GLOBAL_ENTRY(paravirt_ ## sym) ; \ + BR_COND_SPTK_MANY(__ia64_ ## sym, type) ; \ + END(paravirt_ ## sym) + + BRANCH(switch_to, PARAVIRT_ST_SWITCH_TO) + BRANCH(trace_syscall, PARAVIRT_ST_TRACE_SYSCALL) + BRANCH(ret_from_clone, PARAVIRT_ST_RET_FROM_CLONE) + BRANCH(leave_syscall, PARAVIRT_ST_LEAVE_SYSCALL) + BRANCH(leave_kernel, PARAVIRT_ST_LEAVE_KERNEL) + BRANCH(pal_call_static, PARAVIRT_ST_PAL_CALL_STATIC) diff -r 86ac3059ab67 -r 28aaf9468319 arch/ia64/xen/privop_s.S --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/arch/ia64/xen/privop_s.S Fri Jul 13 17:25:40 2007 +0900 @@ -0,0 +1,189 @@ +/****************************************************************************** + * linux/arch/ia64/xen/privop_s.S + * + * Copyright (c) 2007 Isaku Yamahata + * VA Linux Systems Japan K.K. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + */ + +#include +#include + +#ifdef CONFIG_MODULES +#define __INIT_OR_MODULE .text +#define __INITDATA_OR_MODULE .data +#else +#define __INIT_OR_MODULE __INIT +#define __INITDATA_OR_MODULE __INITDATA +#endif /* CONFIG_MODULES */ + + __INIT_OR_MODULE + .align 32 + .proc nop_b_inst_bundle + .global nop_b_inst_bundle +nop_b_inst_bundle: + { + nop.b 0 + nop.b 0 + nop.b 0 + } + .endp nop_b_inst_bundle + __FINIT + + __INIT_OR_MODULE + .align 32 + .proc nop_bundle + .global nop_bundle +nop_bundle: +nop_bundle_start: + { + nop 0 + nop 0 + nop 0 + } +nop_bundle_end: + .endp nop_bundle + __FINIT + + __INITDATA_OR_MODULE + .align 8 + .global nop_bundle_size +nop_bundle_size: + data8 nop_bundle_end - nop_bundle_start + +#define DEFINE_PRIVOP(name, instr) \ + .align 32; \ + .proc xen_ ## name ## _instr; \ + xen_ ## name ## _instr:; \ + xen_ ## name ## _instr_start:; \ + {; \ + instr; \ + nop 0; \ + nop 0; \ + }; \ + xen_ ## name ## _instr_end:; \ + .endp xen_ ## name ## _instr; + + __INIT_OR_MODULE + DEFINE_PRIVOP(rfi, XEN_HYPER_RFI) + DEFINE_PRIVOP(rsm_psr_dt, XEN_HYPER_RSM_PSR_DT) + DEFINE_PRIVOP(ssm_psr_dt, XEN_HYPER_SSM_PSR_DT) + DEFINE_PRIVOP(cover, XEN_HYPER_COVER) + DEFINE_PRIVOP(itc_d, XEN_HYPER_ITC_D) + DEFINE_PRIVOP(itc_i, XEN_HYPER_ITC_I) + DEFINE_PRIVOP(ssm_i, XEN_HYPER_SSM_I) + DEFINE_PRIVOP(get_ivr, XEN_HYPER_GET_IVR) + DEFINE_PRIVOP(get_tpr, XEN_HYPER_GET_TPR) + DEFINE_PRIVOP(set_tpr, XEN_HYPER_SET_TPR) + DEFINE_PRIVOP(eoi, XEN_HYPER_EOI) + DEFINE_PRIVOP(set_itm, XEN_HYPER_SET_ITM) + DEFINE_PRIVOP(thash, XEN_HYPER_THASH) + DEFINE_PRIVOP(ptc_ga, XEN_HYPER_PTC_GA) + DEFINE_PRIVOP(itr_d, XEN_HYPER_ITR_D) + DEFINE_PRIVOP(get_rr, XEN_HYPER_GET_RR) + DEFINE_PRIVOP(set_rr, XEN_HYPER_SET_RR) + DEFINE_PRIVOP(set_kr, XEN_HYPER_SET_KR) + DEFINE_PRIVOP(fc, XEN_HYPER_FC) + DEFINE_PRIVOP(get_cpuid, XEN_HYPER_GET_CPUID) + DEFINE_PRIVOP(get_pmd, XEN_HYPER_GET_PMD) + DEFINE_PRIVOP(get_eflag, XEN_HYPER_GET_EFLAG) + DEFINE_PRIVOP(set_eflag, XEN_HYPER_SET_EFLAG) + DEFINE_PRIVOP(get_psr, XEN_HYPER_GET_PSR) + __FINIT + + +#define PARAVIRT_ALT_BUNDLE_ELEM(name, type) \ + data8 xen_ ## name ## _instr; \ + data8 xen_ ## name ## _instr_end - xen_ ## name ## _instr_start;\ + data8 type; + + __INITDATA_OR_MODULE + .align 8 + .global xen_alt_bundle_array +xen_alt_bundle_array: +xen_alt_bundle_array_start: + PARAVIRT_ALT_BUNDLE_ELEM(rfi, HYPERPRIVOP_RFI) + PARAVIRT_ALT_BUNDLE_ELEM(rsm_psr_dt, HYPERPRIVOP_RSM_DT) + PARAVIRT_ALT_BUNDLE_ELEM(ssm_psr_dt, HYPERPRIVOP_SSM_DT) + PARAVIRT_ALT_BUNDLE_ELEM(cover, HYPERPRIVOP_COVER) + PARAVIRT_ALT_BUNDLE_ELEM(itc_d, HYPERPRIVOP_ITC_D) + PARAVIRT_ALT_BUNDLE_ELEM(itc_i, HYPERPRIVOP_ITC_I) + PARAVIRT_ALT_BUNDLE_ELEM(ssm_i, HYPERPRIVOP_SSM_I) + PARAVIRT_ALT_BUNDLE_ELEM(get_ivr, HYPERPRIVOP_GET_IVR) + PARAVIRT_ALT_BUNDLE_ELEM(get_tpr, HYPERPRIVOP_GET_TPR) + PARAVIRT_ALT_BUNDLE_ELEM(set_tpr, HYPERPRIVOP_SET_TPR) + PARAVIRT_ALT_BUNDLE_ELEM(eoi, HYPERPRIVOP_EOI) + PARAVIRT_ALT_BUNDLE_ELEM(set_itm, HYPERPRIVOP_SET_ITM) + PARAVIRT_ALT_BUNDLE_ELEM(thash, HYPERPRIVOP_THASH) + PARAVIRT_ALT_BUNDLE_ELEM(ptc_ga, HYPERPRIVOP_PTC_GA) + PARAVIRT_ALT_BUNDLE_ELEM(itr_d, HYPERPRIVOP_ITR_D) + PARAVIRT_ALT_BUNDLE_ELEM(get_rr, HYPERPRIVOP_GET_RR) + PARAVIRT_ALT_BUNDLE_ELEM(set_rr, HYPERPRIVOP_SET_RR) + PARAVIRT_ALT_BUNDLE_ELEM(set_kr, HYPERPRIVOP_SET_KR) + PARAVIRT_ALT_BUNDLE_ELEM(fc, HYPERPRIVOP_FC) + PARAVIRT_ALT_BUNDLE_ELEM(get_cpuid, HYPERPRIVOP_GET_CPUID) + PARAVIRT_ALT_BUNDLE_ELEM(get_pmd, HYPERPRIVOP_GET_PMD) + PARAVIRT_ALT_BUNDLE_ELEM(get_eflag, HYPERPRIVOP_GET_EFLAG) + PARAVIRT_ALT_BUNDLE_ELEM(set_eflag, HYPERPRIVOP_SET_EFLAG) + PARAVIRT_ALT_BUNDLE_ELEM(get_psr, HYPERPRIVOP_GET_PSR) +xen_alt_bundle_array_end: + + .align 8 + .global xen_alt_bundle_array_size +xen_alt_bundle_array_size: + .long xen_alt_bundle_array_end - xen_alt_bundle_array_start + + +#define PARAVIRT_ALT_INST_ELEM(name, type) \ + data8 xen_ ## name ## _instr; \ + data8 type; + + __INITDATA_OR_MODULE + .align 8 + .global xen_alt_inst_array +xen_alt_inst_array: +xen_alt_inst_array_start: + PARAVIRT_ALT_INST_ELEM(rfi, HYPERPRIVOP_RFI) + PARAVIRT_ALT_INST_ELEM(rsm_psr_dt, HYPERPRIVOP_RSM_DT) + PARAVIRT_ALT_INST_ELEM(ssm_psr_dt, HYPERPRIVOP_SSM_DT) + PARAVIRT_ALT_INST_ELEM(cover, HYPERPRIVOP_COVER) + PARAVIRT_ALT_INST_ELEM(itc_d, HYPERPRIVOP_ITC_D) + PARAVIRT_ALT_INST_ELEM(itc_i, HYPERPRIVOP_ITC_I) + PARAVIRT_ALT_INST_ELEM(ssm_i, HYPERPRIVOP_SSM_I) + PARAVIRT_ALT_INST_ELEM(get_ivr, HYPERPRIVOP_GET_IVR) + PARAVIRT_ALT_INST_ELEM(get_tpr, HYPERPRIVOP_GET_TPR) + PARAVIRT_ALT_INST_ELEM(set_tpr, HYPERPRIVOP_SET_TPR) + PARAVIRT_ALT_INST_ELEM(eoi, HYPERPRIVOP_EOI) + PARAVIRT_ALT_INST_ELEM(set_itm, HYPERPRIVOP_SET_ITM) + PARAVIRT_ALT_INST_ELEM(thash, HYPERPRIVOP_THASH) + PARAVIRT_ALT_INST_ELEM(ptc_ga, HYPERPRIVOP_PTC_GA) + PARAVIRT_ALT_INST_ELEM(itr_d, HYPERPRIVOP_ITR_D) + PARAVIRT_ALT_INST_ELEM(get_rr, HYPERPRIVOP_GET_RR) + PARAVIRT_ALT_INST_ELEM(set_rr, HYPERPRIVOP_SET_RR) + PARAVIRT_ALT_INST_ELEM(set_kr, HYPERPRIVOP_SET_KR) + PARAVIRT_ALT_INST_ELEM(fc, HYPERPRIVOP_FC) + PARAVIRT_ALT_INST_ELEM(get_cpuid, HYPERPRIVOP_GET_CPUID) + PARAVIRT_ALT_INST_ELEM(get_pmd, HYPERPRIVOP_GET_PMD) + PARAVIRT_ALT_INST_ELEM(get_eflag, HYPERPRIVOP_GET_EFLAG) + PARAVIRT_ALT_INST_ELEM(set_eflag, HYPERPRIVOP_SET_EFLAG) + PARAVIRT_ALT_INST_ELEM(get_psr, HYPERPRIVOP_GET_PSR) +xen_alt_inst_array_end: + + .align 8 + .global xen_alt_inst_array_size +xen_alt_inst_array_size: + .long xen_alt_inst_array_end - xen_alt_inst_array_start diff -r 86ac3059ab67 -r 28aaf9468319 arch/ia64/xen/xenentry.S --- a/arch/ia64/xen/xenentry.S Tue Jul 10 11:18:07 2007 -0600 +++ b/arch/ia64/xen/xenentry.S Fri Jul 13 17:25:40 2007 +0900 @@ -21,6 +21,7 @@ #ifdef CONFIG_XEN #include "xenminstate.h" +#include #else #include "minstate.h" #endif @@ -33,17 +34,12 @@ */ #ifdef CONFIG_XEN GLOBAL_ENTRY(xen_switch_to) + BR_IF_NATIVE(__ia64_switch_to, r22, p7) +#else +GLOBAL_ENTRY(ia64_switch_to) +#endif .prologue alloc r16=ar.pfs,1,0,0,0 - movl r22=running_on_xen;; - ld4 r22=[r22];; - cmp.eq p7,p0=r22,r0 -(p7) br.cond.sptk.many __ia64_switch_to;; -#else -GLOBAL_ENTRY(ia64_switch_to) - .prologue - alloc r16=ar.pfs,1,0,0,0 -#endif DO_SAVE_SWITCH_STACK .body @@ -149,15 +145,11 @@ END(ia64_switch_to) */ #ifdef CONFIG_XEN GLOBAL_ENTRY(xen_trace_syscall) + BR_IF_NATIVE(__ia64_trace_syscall, r16, p7) +#else +GLOBAL_ENTRY(ia64_trace_syscall) +#endif PT_REGS_UNWIND_INFO(0) - movl r16=running_on_xen;; - ld4 r16=[r16];; - cmp.eq p7,p0=r16,r0 -(p7) br.cond.sptk.many __ia64_trace_syscall;; -#else -GLOBAL_ENTRY(ia64_trace_syscall) - PT_REGS_UNWIND_INFO(0) -#endif /* * We need to preserve the scratch registers f6-f11 in case the system * call is sigreturn. @@ -236,15 +228,11 @@ END(ia64_trace_syscall) #ifdef CONFIG_XEN GLOBAL_ENTRY(xen_ret_from_clone) - PT_REGS_UNWIND_INFO(0) - movl r16=running_on_xen;; - ld4 r16=[r16];; - cmp.eq p7,p0=r16,r0 -(p7) br.cond.sptk.many __ia64_ret_from_clone;; + BR_IF_NATIVE(__ia64_ret_from_clone, r16, p7) #else GLOBAL_ENTRY(ia64_ret_from_clone) +#endif PT_REGS_UNWIND_INFO(0) -#endif { /* * Some versions of gas generate bad unwind info if the first instruction of a * procedure doesn't go into the first slot of a bundle. This is a workaround. @@ -321,15 +309,11 @@ END(ia64_ret_from_clone) */ #ifdef CONFIG_XEN GLOBAL_ENTRY(xen_leave_syscall) + BR_IF_NATIVE(__ia64_leave_syscall, r22, p7) +#else +ENTRY(ia64_leave_syscall) +#endif PT_REGS_UNWIND_INFO(0) - movl r22=running_on_xen;; - ld4 r22=[r22];; - cmp.eq p7,p0=r22,r0 -(p7) br.cond.sptk.many __ia64_leave_syscall;; -#else -ENTRY(ia64_leave_syscall) - PT_REGS_UNWIND_INFO(0) -#endif /* * work.need_resched etc. mustn't get changed by this CPU before it returns to * user- or fsys-mode, hence we disable interrupts early on. @@ -478,15 +462,11 @@ END(ia64_leave_syscall) #ifdef CONFIG_XEN GLOBAL_ENTRY(xen_leave_kernel) + BR_IF_NATIVE(__ia64_leave_kernel, r22, p7) +#else +GLOBAL_ENTRY(ia64_leave_kernel) +#endif PT_REGS_UNWIND_INFO(0) - movl r22=running_on_xen;; - ld4 r22=[r22];; - cmp.eq p7,p0=r22,r0 -(p7) br.cond.sptk.many __ia64_leave_kernel;; -#else -GLOBAL_ENTRY(ia64_leave_kernel) - PT_REGS_UNWIND_INFO(0) -#endif /* * work.need_resched etc. mustn't get changed by this CPU before it returns to * user- or fsys-mode, hence we disable interrupts early on. diff -r 86ac3059ab67 -r 28aaf9468319 arch/ia64/xen/xenpal.S --- a/arch/ia64/xen/xenpal.S Tue Jul 10 11:18:07 2007 -0600 +++ b/arch/ia64/xen/xenpal.S Fri Jul 13 17:25:40 2007 +0900 @@ -10,16 +10,14 @@ #include #include +#include GLOBAL_ENTRY(xen_pal_call_static) +#ifdef CONFIG_XEN + BR_IF_NATIVE(__ia64_pal_call_static, r22, p7) +#endif .prologue ASM_UNW_PRLG_RP|ASM_UNW_PRLG_PFS, ASM_UNW_PRLG_GRSAVE(5) alloc loc1 = ar.pfs,5,5,0,0 -#ifdef CONFIG_XEN - movl r22=running_on_xen;; - ld4 r22=[r22];; - cmp.eq p7,p0=r22,r0 -(p7) br.cond.spnt.many __ia64_pal_call_static;; -#endif movl loc2 = pal_entry_point 1: { mov r28 = in0 diff -r 86ac3059ab67 -r 28aaf9468319 arch/ia64/xen/xensetup.S --- a/arch/ia64/xen/xensetup.S Tue Jul 10 11:18:07 2007 -0600 +++ b/arch/ia64/xen/xensetup.S Fri Jul 13 17:25:40 2007 +0900 @@ -35,6 +35,16 @@ GLOBAL_ENTRY(early_xen_setup) (isBP) movl r28=XSI_BASE;; (isBP) break 0x1000;; +#ifdef CONFIG_PARAVIRT_ALT + /* patch privops */ +(isBP) mov r4=rp + ;; +(isBP) br.call.sptk.many rp=xen_paravirt_patch + ;; +(isBP) mov rp=r4 + ;; +#endif + br.ret.sptk.many rp ;; END(early_xen_setup) diff -r 86ac3059ab67 -r 28aaf9468319 include/asm-ia64/module.h --- a/include/asm-ia64/module.h Tue Jul 10 11:18:07 2007 -0600 +++ b/include/asm-ia64/module.h Fri Jul 13 17:25:40 2007 +0900 @@ -16,6 +16,12 @@ struct mod_arch_specific { struct elf64_shdr *got; /* global offset table */ struct elf64_shdr *opd; /* official procedure descriptors */ struct elf64_shdr *unwind; /* unwind-table section */ +#ifdef CONFIG_PARAVIRT_ALT + struct elf64_shdr *paravirt_bundles; + /* paravirt_alt_bundle_patch table */ + struct elf64_shdr *paravirt_insts; + /* paravirt_alt_inst_patch table */ +#endif unsigned long gp; /* global-pointer for module */ void *core_unw_table; /* core unwind-table cookie returned by unwinder */ diff -r 86ac3059ab67 -r 28aaf9468319 include/asm-ia64/paravirt_alt.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/include/asm-ia64/paravirt_alt.h Fri Jul 13 17:25:40 2007 +0900 @@ -0,0 +1,359 @@ +/****************************************************************************** + * Copyright (c) 2007 Isaku Yamahata + * VA Linux Systems Japan K.K. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + */ + +#ifndef __ASM_PARAVIRT_ALT_H +#define __ASM_PARAVIRT_ALT_H + +/* for binary patch */ +struct paravirt_alt_bundle_patch { + void* bundle; + unsigned long size; + unsigned long type; +}; + +/* label means the beginning of new bundle */ +#define paravirt_alt_bundle(instr, privop) \ + "\t1:\n" \ + "\t" instr "\n" \ + "\t2:\n" \ + "\t.section .paravirt_bundles, \"a\"\n" \ + "\t.previous\n" \ + "\t.xdata8 \".paravirt_bundles\", 1b, 2b-1b, " \ + __stringify(privop) "\n" + +struct paravirt_alt_inst_patch { + unsigned long tag; + unsigned long type; +}; + +#define paravirt_alt_inst(instr, privop) \ + "\t[1:]\n" \ + "\t" instr "\n" \ + "\t.section .paravirt_insts, \"a\"\n" \ + "\t.previous\n" \ + "\t.xdata8 \".paravirt_insts\", 1b, " \ + __stringify(privop) "\n" + +/************************************************/ +/* Instructions paravirtualized for correctness */ +/************************************************/ +/* Note that "ttag" and "cover" are also privilege-sensitive; "ttag" + * is not currently used (though it may be in a long-format VHPT system!) */ +static inline unsigned long +paravirt_fc(unsigned long addr) +{ + register __u64 ia64_intri_res asm ("r8"); + register __u64 __addr asm ("r8") = addr; + asm volatile (paravirt_alt_inst("fc %1", HYPERPRIVOP_THASH): + "=r"(ia64_intri_res): "0"(__addr): "memory"); + return ia64_intri_res; +} +#define paravirt_fc(addr) paravirt_fc((unsigned long)addr) + +static inline unsigned long +paravirt_thash(unsigned long addr) +{ + register __u64 ia64_intri_res asm ("r8"); + register __u64 __addr asm ("r8") = addr; + asm volatile (paravirt_alt_inst("thash %0=%1", HYPERPRIVOP_THASH): + "=r"(ia64_intri_res): "0"(__addr)); + return ia64_intri_res; +} + +static inline unsigned long +paravirt_get_cpuid(int index) +{ + register __u64 ia64_intri_res asm ("r8"); + register __u64 __index asm ("r8") = index; + asm volatile (paravirt_alt_inst("mov %0=cpuid[%r1]", + HYPERPRIVOP_GET_CPUID): + "=r"(ia64_intri_res): "0O"(__index)); + return ia64_intri_res; +} + +static inline unsigned long +paravirt_get_pmd(int index) +{ + register __u64 ia64_intri_res asm ("r8"); + register __u64 __index asm ("r8") = index; + asm volatile (paravirt_alt_inst("mov %0=pmd[%1]", HYPERPRIVOP_GET_PMD): + "=r"(ia64_intri_res): "0"(__index)); + return ia64_intri_res; +} + +static inline unsigned long +paravirt_get_eflag(void) +{ + register __u64 ia64_intri_res asm ("r8"); + asm volatile (paravirt_alt_inst("mov %0=ar%1", HYPERPRIVOP_GET_EFLAG): + "=r"(ia64_intri_res): + "i"(_IA64_REG_AR_EFLAG - _IA64_REG_AR_KR0): "memory"); + return ia64_intri_res; +} + +static inline void +paravirt_set_eflag(unsigned long val) +{ + register __u64 __val asm ("r8") = val; + asm volatile (paravirt_alt_inst("mov ar%0=%1", HYPERPRIVOP_SET_EFLAG):: + "i"(_IA64_REG_AR_EFLAG - _IA64_REG_AR_KR0), "r"(__val): + "memory"); +} + +/************************************************/ +/* Instructions paravirtualized for performance */ +/************************************************/ + +static inline unsigned long +paravirt_get_psr(void) +{ + register __u64 ia64_intri_res asm ("r8"); + asm volatile (paravirt_alt_inst("mov %0=psr", HYPERPRIVOP_GET_PSR): + "=r"(ia64_intri_res)); + return ia64_intri_res; +} + +static inline unsigned long +paravirt_get_ivr(void) +{ + register __u64 ia64_intri_res asm ("r8"); + asm volatile (paravirt_alt_inst("mov %0=cr%1", HYPERPRIVOP_GET_IVR): + "=r"(ia64_intri_res): + "i" (_IA64_REG_CR_IVR - _IA64_REG_CR_DCR)); + return ia64_intri_res; +} + +static inline unsigned long +paravirt_get_tpr(void) +{ + register __u64 ia64_intri_res asm ("r8"); + asm volatile (paravirt_alt_inst("mov %0=cr%1", HYPERPRIVOP_GET_TPR): + "=r"(ia64_intri_res): + "i" (_IA64_REG_CR_TPR - _IA64_REG_CR_DCR)); + return ia64_intri_res; +} + +static inline void +paravirt_set_tpr(unsigned long val) +{ + register __u64 __val asm ("r8") = val; + asm volatile (paravirt_alt_inst("mov cr%0=%1", HYPERPRIVOP_SET_TPR):: + "i" (_IA64_REG_CR_TPR - _IA64_REG_CR_DCR), "r"(__val): + "memory"); +} + +static inline void +paravirt_eoi(unsigned long val) +{ + register __u64 __val asm ("r8") = val; + asm volatile (paravirt_alt_inst("mov cr%0=%1", HYPERPRIVOP_EOI):: + "i" (_IA64_REG_CR_EOI - _IA64_REG_CR_DCR), "r"(__val): + "memory"); +} + +static inline void +paravirt_set_itm(unsigned long val) +{ + register __u64 __val asm ("r8") = val; + asm volatile (paravirt_alt_inst("mov cr%0=%1", HYPERPRIVOP_SET_ITM):: + "i" (_IA64_REG_CR_ITM - _IA64_REG_CR_DCR), "r"(__val): + "memory"); +} + +static inline void +paravirt_ptcga(unsigned long addr, unsigned long size) +{ + register __u64 __addr asm ("r8") = addr; + register __u64 __size asm ("r9") = size; + asm volatile (paravirt_alt_inst("ptc.ga %0,%1", HYPERPRIVOP_PTC_GA):: + "r"(__addr), "r"(__size): "memory"); + ia64_dv_serialize_data(); +} + +static inline unsigned long +paravirt_get_rr(unsigned long index) +{ + register __u64 ia64_intri_res asm ("r8"); + register __u64 __index asm ("r8") = index; + asm volatile (paravirt_alt_inst("mov %0=rr[%1]", HYPERPRIVOP_GET_RR): + "=r"(ia64_intri_res) : "0" (__index)); + return ia64_intri_res; +} + +static inline void +paravirt_set_rr(unsigned long index, unsigned long val) +{ + register __u64 __index asm ("r8") = index; + register __u64 __val asm ("r9") = val; + asm volatile (paravirt_alt_inst("mov rr[%0]=%1", HYPERPRIVOP_SET_RR):: + "r"(__index), "r"(__val): "memory"); +} + +static inline void +paravirt_set_kr(unsigned long index, unsigned long val) +{ + register __u64 __index asm ("r8") = index - _IA64_REG_AR_KR0; + register __u64 __val asm ("r9") = val; + + //asm volatile ("break %0":: + // "i"(HYPERPRIVOP_SET_KR), "r"(__index), "r"(__val)); +#ifndef BUILD_BUG_ON +#define BUILD_BUG_ON(condition) ((void)sizeof(char[1 - 2*!!(condition)])) +#endif + BUILD_BUG_ON(!__builtin_constant_p(__index)); + switch (index) { + case _IA64_REG_AR_KR0: + asm volatile (paravirt_alt_inst("mov ar%0=%2", + HYPERPRIVOP_SET_KR):: + "i" (_IA64_REG_AR_KR0 - _IA64_REG_AR_KR0), + "r"(__index), "r"(__val): + "memory"); + break; + case _IA64_REG_AR_KR1: + asm volatile (paravirt_alt_inst("mov ar%0=%2", + HYPERPRIVOP_SET_KR):: + "i" (_IA64_REG_AR_KR1 - _IA64_REG_AR_KR0), + "r"(__index), "r"(__val): + "memory"); + break; + case _IA64_REG_AR_KR2: + asm volatile (paravirt_alt_inst("mov ar%0=%2", + HYPERPRIVOP_SET_KR):: + "i" (_IA64_REG_AR_KR2 - _IA64_REG_AR_KR0), + "r"(__index), "r"(__val): + "memory"); + break; + case _IA64_REG_AR_KR3: + asm volatile (paravirt_alt_inst("mov ar%0=%2", + HYPERPRIVOP_SET_KR):: + "i" (_IA64_REG_AR_KR3 - _IA64_REG_AR_KR0), + "r"(__index), "r"(__val): + "memory"); + break; + case _IA64_REG_AR_KR4: + asm volatile (paravirt_alt_inst("mov ar%0=%2", + HYPERPRIVOP_SET_KR):: + "i" (_IA64_REG_AR_KR4 - _IA64_REG_AR_KR0), + "r"(__index), "r"(__val): + "memory"); + break; + case _IA64_REG_AR_KR5: + asm volatile (paravirt_alt_inst("mov ar%0=%2", + HYPERPRIVOP_SET_KR):: + "i" (_IA64_REG_AR_KR5 - _IA64_REG_AR_KR0), + "r"(__index), "r"(__val): + "memory"); + break; + case _IA64_REG_AR_KR6: + asm volatile (paravirt_alt_inst("mov ar%0=%2", + HYPERPRIVOP_SET_KR):: + "i" (_IA64_REG_AR_KR6 - _IA64_REG_AR_KR0), + "r"(__index), "r"(__val): + "memory"); + break; + case _IA64_REG_AR_KR7: + asm volatile (paravirt_alt_inst("mov ar%0=%2", + HYPERPRIVOP_SET_KR):: + "i" (_IA64_REG_AR_KR7 - _IA64_REG_AR_KR0), + "r"(__index), "r"(__val): + "memory"); + break; + default: { + extern void compile_error_ar_kr_index_must_be_copmile_time_constant(void); + compile_error_ar_kr_index_must_be_copmile_time_constant(); + break; + } + } +} + +static inline unsigned long +paravirt_getreg(unsigned long regnum) +{ + __u64 ia64_intri_res; + + switch(regnum) { + case _IA64_REG_PSR: + ia64_intri_res = paravirt_get_psr(); + break; + case _IA64_REG_CR_IVR: + ia64_intri_res = paravirt_get_ivr(); + break; + case _IA64_REG_CR_TPR: + ia64_intri_res = paravirt_get_tpr(); + break; + case _IA64_REG_AR_EFLAG: + ia64_intri_res = paravirt_get_eflag(); + break; + default: + ia64_intri_res = __ia64_getreg(regnum); + break; + } + return ia64_intri_res; + } + +static inline void +paravirt_setreg(unsigned long regnum, unsigned long val) +{ + switch(regnum) { + case _IA64_REG_AR_KR0 ... _IA64_REG_AR_KR7: + paravirt_set_kr(regnum, val); + break; + case _IA64_REG_CR_ITM: + paravirt_set_itm(val); + break; + case _IA64_REG_CR_TPR: + paravirt_set_tpr(val); + break; + case _IA64_REG_CR_EOI: + paravirt_eoi(val); + break; + case _IA64_REG_AR_EFLAG: + paravirt_set_eflag(val); + break; + default: + __ia64_setreg(regnum,val); + break; + } +} + +void +paravirt_alt_bundle_patch_apply(struct paravirt_alt_bundle_patch* start, + struct paravirt_alt_bundle_patch* end, + unsigned long(*patch)(void* inst, + unsigned long size, + unsigned long type)); + +void +paravirt_alt_inst_patch_apply(struct paravirt_alt_inst_patch* start, + struct paravirt_alt_inst_patch* end, + void (*patch)(unsigned long tag, + unsigned long type)); + +#endif /* __ASM_PARAVIRT_ALT_H */ + +/* + * Local variables: + * mode: C + * c-set-style: "linux" + * c-basic-offset: 8 + * tab-width: 8 + * indent-tabs-mode: t + * End: + */ diff -r 86ac3059ab67 -r 28aaf9468319 include/asm-ia64/paravirt_core.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/include/asm-ia64/paravirt_core.h Fri Jul 13 17:25:40 2007 +0900 @@ -0,0 +1,53 @@ +/****************************************************************************** + * Copyright (c) 2007 Isaku Yamahata + * VA Linux Systems Japan K.K. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + */ + +#ifndef __ASM_PARAVIRT_CORE_H +#define __ASM_PARAVIRT_CORE_H + +#include + +void paravirt_flush_i_cache_range(const void* instr, unsigned long size); + +bundle_t* paravirt_get_bundle(unsigned long tag); +unsigned long paravirt_get_slot(unsigned long tag); + +cmp_inst_t paravirt_read_slot0(const bundle_t *bundle); +cmp_inst_t paravirt_read_slot1(const bundle_t *bundle); +cmp_inst_t paravirt_read_slot2(const bundle_t *bundle); +cmp_inst_t paravirt_read_inst(unsigned long tag); + +void paravirt_write_slot0(bundle_t *bundle, cmp_inst_t inst); +void paravirt_write_slot1(bundle_t *bundle, cmp_inst_t inst); +void paravirt_write_slot2(bundle_t *bundle, cmp_inst_t inst); +void paravirt_write_inst(unsigned long tag, cmp_inst_t inst); + +void print_bundle(const bundle_t *bundle); + +#endif /* __ASM_PARAVIRT_CORE_H */ + +/* + * Local variables: + * mode: C + * c-set-style: "linux" + * c-basic-offset: 8 + * tab-width: 8 + * indent-tabs-mode: t + * End: + */ diff -r 86ac3059ab67 -r 28aaf9468319 include/asm-ia64/paravirt_entry.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/include/asm-ia64/paravirt_entry.h Fri Jul 13 17:25:40 2007 +0900 @@ -0,0 +1,78 @@ +/****************************************************************************** + * Copyright (c) 2007 Isaku Yamahata + * VA Linux Systems Japan K.K. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + */ + +#ifndef __ASM_PARAVIRT_ENTRY_H +#define __ASM_PARAVIRT_ENTRY_H + +/* + * void* leave_kernel; + * void* leave_syscall; + * void* trace_syscall; + * void* ret_from_clone; + * struct task_struct* (*switch_to)(void* next_task); + * struct ia64_pal_retval (*pal_call_static)(u64, u64, u64, u64, u64); + */ + +#define PARAVIRT_ET_LEAVE_KERNEL __IA64_UL_CONST(0x0) +#define PARAVIRT_ET_LEAVE_SYSCALL __IA64_UL_CONST(0x1) +#define PARAVIRT_ET_TRACE_SYSCALL __IA64_UL_CONST(0x2) +#define PARAVIRT_ET_RET_FROM_CLONE __IA64_UL_CONST(0x3) +#define PARAVIRT_ET_SWITCH_TO __IA64_UL_CONST(0x4) +#define PARAVIRT_ET_PAL_CALL_STATIC __IA64_UL_CONST(0x5) + +#ifdef __ASSEMBLY__ + +#define BR_COND_SPTK_MANY(target, type) \ + [1:] ; \ + br.cond.sptk.many target;; ; \ + .section .paravirt_entry, "a" ; \ + .previous ; \ + .xdata8 ".paravirt_entry", 1b, type + +#else /* __ASSEMBLY__ */ + +struct paravirt_entry_patch { + unsigned long tag; + unsigned long type; +}; + +struct paravirt_entry { + void* entry; + unsigned long type; +}; + +void +paravirt_entry_patch_apply(const struct paravirt_entry_patch* start, + const struct paravirt_entry_patch* end, + const struct paravirt_entry* entries, + unsigned int nr_entries); + +#endif /* __ASSEMBLY__ */ + +#endif /* __ASM_PARAVIRT_ENTRY_H */ +/* + * Local variables: + * mode: C + * c-set-style: "linux" + * c-basic-offset: 8 + * tab-width: 8 + * indent-tabs-mode: t + * End: + */ diff -r 86ac3059ab67 -r 28aaf9468319 include/asm-ia64/paravirt_nop.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/include/asm-ia64/paravirt_nop.h Fri Jul 13 17:25:40 2007 +0900 @@ -0,0 +1,72 @@ +/****************************************************************************** + * Copyright (c) 2007 Isaku Yamahata + * VA Linux Systems Japan K.K. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + */ + +#ifndef __ASM_PARAVIRT_OPS_H +#define __ASM_PARAVIRT_OPS_H + +#ifdef __ASSEMBLY__ + +#ifdef CONFIG_XEN + +#ifdef CONFIG_PARAVIRT_ENTRY +#define BR_IF_NATIVE(target, reg_unused, pred_unused) /* nothing */ +#elif defined(CONFIG_PARAVIRT_NOP_B_PATCH) +#define BR_IF_NATIVE(target, reg_unused, pred_unused) \ + .body ; \ + [1:] ; \ + br.cond.sptk.many target;; ; \ + .section .paravirt_nop_b, "a" ; \ + .previous ; \ + .xdata8 ".paravirt_nop_b", 1b +#else +#define BR_IF_NATIVE(target, reg, pred) \ + .body ; \ + movl reg=running_on_xen;; ; \ + ld4 reg=[reg];; ; \ + cmp.eq pred,p0=reg,r0 ; \ + (pred) br.cond.sptk.many target;; +#endif + +#endif + +#else /* __ASSEMBLEY__ */ + +struct paravirt_nop_patch { + unsigned long tag; +}; + +void +paravirt_nop_b_patch_apply(const struct paravirt_nop_patch *start, + const struct paravirt_nop_patch *end); + + +#endif /* __ASSEMBLEY__ */ + +#endif /* __ASM_PARAVIRT_OPS_H */ + +/* + * Local variables: + * mode: C + * c-set-style: "linux" + * c-basic-offset: 8 + * tab-width: 8 + * indent-tabs-mode: t + * End: + */ diff -r 86ac3059ab67 -r 28aaf9468319 include/asm-ia64/xen/privop.h --- a/include/asm-ia64/xen/privop.h Tue Jul 10 11:18:07 2007 -0600 +++ b/include/asm-ia64/xen/privop.h Fri Jul 13 17:25:40 2007 +0900 @@ -6,6 +6,11 @@ * Dan Magenheimer * * Paravirtualizations of privileged operations for Xen/ia64 + * + * + * inline privop and paravirt_alt support + * Copyright (c) 2007 Isaku Yamahata + * VA Linux Systems Japan K.K. * */ @@ -509,16 +514,53 @@ do { \ #define ia64_ptri __ia64_ptri #define ia64_ptrd __ia64_ptrd +#if defined(ASM_SUPPORTED) && defined(CONFIG_PARAVIRT_ALT) + +#undef ia64_fc +#undef ia64_thash +#undef ia64_get_cpuid +#undef ia64_get_pmd +#undef ia64_ptcga +#undef ia64_set_rr +#undef ia64_get_rr +#undef ia64_getreg +#undef ia64_setreg + +#include + +#define ia64_fc(addr) paravirt_fc(addr) +#define ia64_thash(addr) paravirt_thash(addr) +#define ia64_get_cpuid(i) paravirt_get_cpuid(i) +#define ia64_get_pmd(i) paravirt_get_pmd(i) +#define ia64_ptcga(addr, size) paravirt_ptcga((addr), (size)) +#define ia64_set_rr(index, val) paravirt_set_rr((index), (val)) +#define ia64_get_rr(index) paravirt_get_rr(index) +#define ia64_getreg(regnum) paravirt_getreg(regnum) +#define ia64_setreg(regnum, val) paravirt_setreg((regnum), (val)) + +void xen_alt_bundle_patch_module(struct paravirt_alt_bundle_patch *start, struct paravirt_alt_bundle_patch *end); + +#endif /* CONFIG_PARAVIRT_ALT */ + #endif /* !__ASSEMBLY__ */ /* these routines utilize privilege-sensitive or performance-sensitive * privileged instructions so the code must be replaced with * paravirtualized versions */ +#ifdef CONFIG_PARAVIRT_ENTRY +#define ia64_leave_kernel paravirt_leave_kernel +#define ia64_leave_syscall paravirt_leave_syscall +#define ia64_trace_syscall paravirt_trace_syscall +#define ia64_ret_from_clone paravirt_ret_from_clone +#define ia64_switch_to paravirt_switch_to +#define ia64_pal_call_static paravirt_pal_call_static +#else #define ia64_leave_kernel xen_leave_kernel #define ia64_leave_syscall xen_leave_syscall #define ia64_trace_syscall xen_trace_syscall #define ia64_ret_from_clone xen_ret_from_clone #define ia64_switch_to xen_switch_to #define ia64_pal_call_static xen_pal_call_static +#endif #endif /* _ASM_IA64_XEN_PRIVOP_H */