1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
27 #include <linux/stringify.h>
36 #define OpImplicit 1ull /* No generic decode */
37 #define OpReg 2ull /* Register */
38 #define OpMem 3ull /* Memory */
39 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
40 #define OpDI 5ull /* ES:DI/EDI/RDI */
41 #define OpMem64 6ull /* Memory, 64-bit */
42 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
43 #define OpDX 8ull /* DX register */
44 #define OpCL 9ull /* CL register (for shifts) */
45 #define OpImmByte 10ull /* 8-bit sign extended immediate */
46 #define OpOne 11ull /* Implied 1 */
47 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
48 #define OpMem16 13ull /* Memory operand (16-bit). */
49 #define OpMem32 14ull /* Memory operand (32-bit). */
50 #define OpImmU 15ull /* Immediate operand, zero extended */
51 #define OpSI 16ull /* SI/ESI/RSI */
52 #define OpImmFAddr 17ull /* Immediate far address */
53 #define OpMemFAddr 18ull /* Far address in memory */
54 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
55 #define OpES 20ull /* ES */
56 #define OpCS 21ull /* CS */
57 #define OpSS 22ull /* SS */
58 #define OpDS 23ull /* DS */
59 #define OpFS 24ull /* FS */
60 #define OpGS 25ull /* GS */
61 #define OpMem8 26ull /* 8-bit zero extended memory operand */
62 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
63 #define OpXLat 28ull /* memory at BX/EBX/RBX + zero-extended AL */
64 #define OpAccLo 29ull /* Low part of extended acc (AX/AX/EAX/RAX) */
65 #define OpAccHi 30ull /* High part of extended acc (-/DX/EDX/RDX) */
67 #define OpBits 5 /* Width of operand field */
68 #define OpMask ((1ull << OpBits) - 1)
71 * Opcode effective-address decode tables.
72 * Note that we only emulate instructions that have at least one memory
73 * operand (excluding implicit stack references). We assume that stack
74 * references and instruction fetches will never occur in special memory
75 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
79 /* Operand sizes: 8-bit operands or specified/overridden size. */
80 #define ByteOp (1<<0) /* 8-bit operands. */
81 /* Destination operand type. */
83 #define ImplicitOps (OpImplicit << DstShift)
84 #define DstReg (OpReg << DstShift)
85 #define DstMem (OpMem << DstShift)
86 #define DstAcc (OpAcc << DstShift)
87 #define DstDI (OpDI << DstShift)
88 #define DstMem64 (OpMem64 << DstShift)
89 #define DstImmUByte (OpImmUByte << DstShift)
90 #define DstDX (OpDX << DstShift)
91 #define DstAccLo (OpAccLo << DstShift)
92 #define DstMask (OpMask << DstShift)
93 /* Source operand type. */
95 #define SrcNone (OpNone << SrcShift)
96 #define SrcReg (OpReg << SrcShift)
97 #define SrcMem (OpMem << SrcShift)
98 #define SrcMem16 (OpMem16 << SrcShift)
99 #define SrcMem32 (OpMem32 << SrcShift)
100 #define SrcImm (OpImm << SrcShift)
101 #define SrcImmByte (OpImmByte << SrcShift)
102 #define SrcOne (OpOne << SrcShift)
103 #define SrcImmUByte (OpImmUByte << SrcShift)
104 #define SrcImmU (OpImmU << SrcShift)
105 #define SrcSI (OpSI << SrcShift)
106 #define SrcXLat (OpXLat << SrcShift)
107 #define SrcImmFAddr (OpImmFAddr << SrcShift)
108 #define SrcMemFAddr (OpMemFAddr << SrcShift)
109 #define SrcAcc (OpAcc << SrcShift)
110 #define SrcImmU16 (OpImmU16 << SrcShift)
111 #define SrcImm64 (OpImm64 << SrcShift)
112 #define SrcDX (OpDX << SrcShift)
113 #define SrcMem8 (OpMem8 << SrcShift)
114 #define SrcAccHi (OpAccHi << SrcShift)
115 #define SrcMask (OpMask << SrcShift)
116 #define BitOp (1<<11)
117 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
118 #define String (1<<13) /* String instruction (rep capable) */
119 #define Stack (1<<14) /* Stack instruction (push/pop) */
120 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
121 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
122 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
123 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
124 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
125 #define Escape (5<<15) /* Escape to coprocessor instruction */
126 #define Sse (1<<18) /* SSE Vector instruction */
127 /* Generic ModRM decode. */
128 #define ModRM (1<<19)
129 /* Destination is only written; never read. */
132 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
133 #define EmulateOnUD (1<<22) /* Emulate if unsupported by the host */
134 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
135 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
136 #define Undefined (1<<25) /* No Such Instruction */
137 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
138 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
140 #define PageTable (1 << 29) /* instruction used to write page table */
141 #define NotImpl (1 << 30) /* instruction is not implemented */
142 /* Source 2 operand type */
143 #define Src2Shift (31)
144 #define Src2None (OpNone << Src2Shift)
145 #define Src2Mem (OpMem << Src2Shift)
146 #define Src2CL (OpCL << Src2Shift)
147 #define Src2ImmByte (OpImmByte << Src2Shift)
148 #define Src2One (OpOne << Src2Shift)
149 #define Src2Imm (OpImm << Src2Shift)
150 #define Src2ES (OpES << Src2Shift)
151 #define Src2CS (OpCS << Src2Shift)
152 #define Src2SS (OpSS << Src2Shift)
153 #define Src2DS (OpDS << Src2Shift)
154 #define Src2FS (OpFS << Src2Shift)
155 #define Src2GS (OpGS << Src2Shift)
156 #define Src2Mask (OpMask << Src2Shift)
157 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
158 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
159 #define Unaligned ((u64)1 << 42) /* Explicitly unaligned (e.g. MOVDQU) */
160 #define Avx ((u64)1 << 43) /* Advanced Vector Extensions */
161 #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
162 #define NoWrite ((u64)1 << 45) /* No writeback */
163 #define SrcWrite ((u64)1 << 46) /* Write back src operand */
164 #define NoMod ((u64)1 << 47) /* Mod field is ignored */
165 #define Intercept ((u64)1 << 48) /* Has valid intercept field */
166 #define CheckPerm ((u64)1 << 49) /* Has valid check_perm field */
168 #define DstXacc (DstAccLo | SrcAccHi | SrcWrite)
170 #define X2(x...) x, x
171 #define X3(x...) X2(x), x
172 #define X4(x...) X2(x), X2(x)
173 #define X5(x...) X4(x), x
174 #define X6(x...) X4(x), X2(x)
175 #define X7(x...) X4(x), X3(x)
176 #define X8(x...) X4(x), X4(x)
177 #define X16(x...) X8(x), X8(x)
179 #define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
180 #define FASTOP_SIZE 8
183 * fastop functions have a special calling convention:
188 * flags: rflags (in/out)
189 * ex: rsi (in:fastop pointer, out:zero if exception)
191 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
192 * different operand sizes can be reached by calculation, rather than a jump
193 * table (which would be bigger than the code).
195 * fastop functions are declared as taking a never-defined fastop parameter,
196 * so they can't be called from C directly.
205 int (*execute)(struct x86_emulate_ctxt *ctxt);
206 const struct opcode *group;
207 const struct group_dual *gdual;
208 const struct gprefix *gprefix;
209 const struct escape *esc;
210 void (*fastop)(struct fastop *fake);
212 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
216 struct opcode mod012[8];
217 struct opcode mod3[8];
221 struct opcode pfx_no;
222 struct opcode pfx_66;
223 struct opcode pfx_f2;
224 struct opcode pfx_f3;
229 struct opcode high[64];
232 /* EFLAGS bit definitions. */
233 #define EFLG_ID (1<<21)
234 #define EFLG_VIP (1<<20)
235 #define EFLG_VIF (1<<19)
236 #define EFLG_AC (1<<18)
237 #define EFLG_VM (1<<17)
238 #define EFLG_RF (1<<16)
239 #define EFLG_IOPL (3<<12)
240 #define EFLG_NT (1<<14)
241 #define EFLG_OF (1<<11)
242 #define EFLG_DF (1<<10)
243 #define EFLG_IF (1<<9)
244 #define EFLG_TF (1<<8)
245 #define EFLG_SF (1<<7)
246 #define EFLG_ZF (1<<6)
247 #define EFLG_AF (1<<4)
248 #define EFLG_PF (1<<2)
249 #define EFLG_CF (1<<0)
251 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
252 #define EFLG_RESERVED_ONE_MASK 2
254 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
256 if (!(ctxt->regs_valid & (1 << nr))) {
257 ctxt->regs_valid |= 1 << nr;
258 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
260 return ctxt->_regs[nr];
263 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
265 ctxt->regs_valid |= 1 << nr;
266 ctxt->regs_dirty |= 1 << nr;
267 return &ctxt->_regs[nr];
270 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
273 return reg_write(ctxt, nr);
276 static void writeback_registers(struct x86_emulate_ctxt *ctxt)
280 for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
281 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
284 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
286 ctxt->regs_dirty = 0;
287 ctxt->regs_valid = 0;
291 * These EFLAGS bits are restored from saved value during emulation, and
292 * any changes are written back to the saved value after emulation.
294 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
302 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));
304 #define FOP_ALIGN ".align " __stringify(FASTOP_SIZE) " \n\t"
305 #define FOP_RET "ret \n\t"
307 #define FOP_START(op) \
308 extern void em_##op(struct fastop *fake); \
309 asm(".pushsection .text, \"ax\" \n\t" \
310 ".global em_" #op " \n\t" \
317 #define FOPNOP() FOP_ALIGN FOP_RET
319 #define FOP1E(op, dst) \
320 FOP_ALIGN "10: " #op " %" #dst " \n\t" FOP_RET
322 #define FOP1EEX(op, dst) \
323 FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception)
325 #define FASTOP1(op) \
330 ON64(FOP1E(op##q, rax)) \
333 /* 1-operand, using src2 (for MUL/DIV r/m) */
334 #define FASTOP1SRC2(op, name) \
339 ON64(FOP1E(op, rcx)) \
342 /* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
343 #define FASTOP1SRC2EX(op, name) \
348 ON64(FOP1EEX(op, rcx)) \
351 #define FOP2E(op, dst, src) \
352 FOP_ALIGN #op " %" #src ", %" #dst " \n\t" FOP_RET
354 #define FASTOP2(op) \
356 FOP2E(op##b, al, dl) \
357 FOP2E(op##w, ax, dx) \
358 FOP2E(op##l, eax, edx) \
359 ON64(FOP2E(op##q, rax, rdx)) \
362 /* 2 operand, word only */
363 #define FASTOP2W(op) \
366 FOP2E(op##w, ax, dx) \
367 FOP2E(op##l, eax, edx) \
368 ON64(FOP2E(op##q, rax, rdx)) \
371 /* 2 operand, src is CL */
372 #define FASTOP2CL(op) \
374 FOP2E(op##b, al, cl) \
375 FOP2E(op##w, ax, cl) \
376 FOP2E(op##l, eax, cl) \
377 ON64(FOP2E(op##q, rax, cl)) \
380 #define FOP3E(op, dst, src, src2) \
381 FOP_ALIGN #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET
383 /* 3-operand, word-only, src2=cl */
384 #define FASTOP3WCL(op) \
387 FOP3E(op##w, ax, dx, cl) \
388 FOP3E(op##l, eax, edx, cl) \
389 ON64(FOP3E(op##q, rax, rdx, cl)) \
392 /* Special case for SETcc - 1 instruction per cc */
393 #define FOP_SETCC(op) ".align 4; " #op " %al; ret \n\t"
395 asm(".global kvm_fastop_exception \n"
396 "kvm_fastop_exception: xor %esi, %esi; ret");
417 FOP_START(salc) "pushf; sbb %al, %al; popf \n\t" FOP_RET
420 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
421 enum x86_intercept intercept,
422 enum x86_intercept_stage stage)
424 struct x86_instruction_info info = {
425 .intercept = intercept,
426 .rep_prefix = ctxt->rep_prefix,
427 .modrm_mod = ctxt->modrm_mod,
428 .modrm_reg = ctxt->modrm_reg,
429 .modrm_rm = ctxt->modrm_rm,
430 .src_val = ctxt->src.val64,
431 .dst_val = ctxt->dst.val64,
432 .src_bytes = ctxt->src.bytes,
433 .dst_bytes = ctxt->dst.bytes,
434 .ad_bytes = ctxt->ad_bytes,
435 .next_rip = ctxt->eip,
438 return ctxt->ops->intercept(ctxt, &info, stage);
441 static void assign_masked(ulong *dest, ulong src, ulong mask)
443 *dest = (*dest & ~mask) | (src & mask);
446 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
448 return (1UL << (ctxt->ad_bytes << 3)) - 1;
451 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
454 struct desc_struct ss;
456 if (ctxt->mode == X86EMUL_MODE_PROT64)
458 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
459 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
462 static int stack_size(struct x86_emulate_ctxt *ctxt)
464 return (__fls(stack_mask(ctxt)) + 1) >> 3;
467 /* Access/update address held in a register, based on addressing mode. */
468 static inline unsigned long
469 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
471 if (ctxt->ad_bytes == sizeof(unsigned long))
474 return reg & ad_mask(ctxt);
477 static inline unsigned long
478 register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
480 return address_mask(ctxt, reg);
483 static void masked_increment(ulong *reg, ulong mask, int inc)
485 assign_masked(reg, *reg + inc, mask);
489 register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
493 if (ctxt->ad_bytes == sizeof(unsigned long))
496 mask = ad_mask(ctxt);
497 masked_increment(reg, mask, inc);
500 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
502 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
505 static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
507 register_address_increment(ctxt, &ctxt->_eip, rel);
510 static u32 desc_limit_scaled(struct desc_struct *desc)
512 u32 limit = get_desc_limit(desc);
514 return desc->g ? (limit << 12) | 0xfff : limit;
517 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
519 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
522 return ctxt->ops->get_cached_segment_base(ctxt, seg);
525 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
526 u32 error, bool valid)
528 ctxt->exception.vector = vec;
529 ctxt->exception.error_code = error;
530 ctxt->exception.error_code_valid = valid;
531 return X86EMUL_PROPAGATE_FAULT;
534 static int emulate_db(struct x86_emulate_ctxt *ctxt)
536 return emulate_exception(ctxt, DB_VECTOR, 0, false);
539 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
541 return emulate_exception(ctxt, GP_VECTOR, err, true);
544 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
546 return emulate_exception(ctxt, SS_VECTOR, err, true);
549 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
551 return emulate_exception(ctxt, UD_VECTOR, 0, false);
554 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
556 return emulate_exception(ctxt, TS_VECTOR, err, true);
559 static int emulate_de(struct x86_emulate_ctxt *ctxt)
561 return emulate_exception(ctxt, DE_VECTOR, 0, false);
564 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
566 return emulate_exception(ctxt, NM_VECTOR, 0, false);
569 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
572 struct desc_struct desc;
574 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
578 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
583 struct desc_struct desc;
585 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
586 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
590 * x86 defines three classes of vector instructions: explicitly
591 * aligned, explicitly unaligned, and the rest, which change behaviour
592 * depending on whether they're AVX encoded or not.
594 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
595 * subject to the same check.
597 static bool insn_aligned(struct x86_emulate_ctxt *ctxt, unsigned size)
599 if (likely(size < 16))
602 if (ctxt->d & Aligned)
604 else if (ctxt->d & Unaligned)
606 else if (ctxt->d & Avx)
612 static int __linearize(struct x86_emulate_ctxt *ctxt,
613 struct segmented_address addr,
614 unsigned size, bool write, bool fetch,
617 struct desc_struct desc;
624 la = seg_base(ctxt, addr.seg) + addr.ea;
625 switch (ctxt->mode) {
626 case X86EMUL_MODE_PROT64:
627 if (((signed long)la << 16) >> 16 != la)
628 return emulate_gp(ctxt, 0);
631 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
635 /* code segment in protected mode or read-only data segment */
636 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
637 || !(desc.type & 2)) && write)
639 /* unreadable code segment */
640 if (!fetch && (desc.type & 8) && !(desc.type & 2))
642 lim = desc_limit_scaled(&desc);
643 if ((desc.type & 8) || !(desc.type & 4)) {
644 /* expand-up segment */
645 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
648 /* expand-down segment */
649 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
651 lim = desc.d ? 0xffffffff : 0xffff;
652 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
655 cpl = ctxt->ops->cpl(ctxt);
656 if (!(desc.type & 8)) {
660 } else if ((desc.type & 8) && !(desc.type & 4)) {
661 /* nonconforming code segment */
664 } else if ((desc.type & 8) && (desc.type & 4)) {
665 /* conforming code segment */
671 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
673 if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0))
674 return emulate_gp(ctxt, 0);
676 return X86EMUL_CONTINUE;
678 if (addr.seg == VCPU_SREG_SS)
679 return emulate_ss(ctxt, sel);
681 return emulate_gp(ctxt, sel);
684 static int linearize(struct x86_emulate_ctxt *ctxt,
685 struct segmented_address addr,
686 unsigned size, bool write,
689 return __linearize(ctxt, addr, size, write, false, linear);
693 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
694 struct segmented_address addr,
701 rc = linearize(ctxt, addr, size, false, &linear);
702 if (rc != X86EMUL_CONTINUE)
704 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
708 * Fetch the next byte of the instruction being emulated which is pointed to
709 * by ctxt->_eip, then increment ctxt->_eip.
711 * Also prefetch the remaining bytes of the instruction without crossing page
712 * boundary if they are not in fetch_cache yet.
714 static int do_insn_fetch_byte(struct x86_emulate_ctxt *ctxt, u8 *dest)
716 struct fetch_cache *fc = &ctxt->fetch;
720 if (ctxt->_eip == fc->end) {
721 unsigned long linear;
722 struct segmented_address addr = { .seg = VCPU_SREG_CS,
724 cur_size = fc->end - fc->start;
725 size = min(15UL - cur_size,
726 PAGE_SIZE - offset_in_page(ctxt->_eip));
727 rc = __linearize(ctxt, addr, size, false, true, &linear);
728 if (unlikely(rc != X86EMUL_CONTINUE))
730 rc = ctxt->ops->fetch(ctxt, linear, fc->data + cur_size,
731 size, &ctxt->exception);
732 if (unlikely(rc != X86EMUL_CONTINUE))
736 *dest = fc->data[ctxt->_eip - fc->start];
738 return X86EMUL_CONTINUE;
741 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
742 void *dest, unsigned size)
746 /* x86 instructions are limited to 15 bytes. */
747 if (unlikely(ctxt->_eip + size - ctxt->eip > 15))
748 return X86EMUL_UNHANDLEABLE;
750 rc = do_insn_fetch_byte(ctxt, dest++);
751 if (rc != X86EMUL_CONTINUE)
754 return X86EMUL_CONTINUE;
757 /* Fetch next part of the instruction being emulated. */
758 #define insn_fetch(_type, _ctxt) \
759 ({ unsigned long _x; \
760 rc = do_insn_fetch(_ctxt, &_x, sizeof(_type)); \
761 if (rc != X86EMUL_CONTINUE) \
766 #define insn_fetch_arr(_arr, _size, _ctxt) \
767 ({ rc = do_insn_fetch(_ctxt, _arr, (_size)); \
768 if (rc != X86EMUL_CONTINUE) \
773 * Given the 'reg' portion of a ModRM byte, and a register block, return a
774 * pointer into the block that addresses the relevant register.
775 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
777 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
781 int highbyte_regs = (ctxt->rex_prefix == 0) && byteop;
783 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
784 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
786 p = reg_rmw(ctxt, modrm_reg);
790 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
791 struct segmented_address addr,
792 u16 *size, unsigned long *address, int op_bytes)
799 rc = segmented_read_std(ctxt, addr, size, 2);
800 if (rc != X86EMUL_CONTINUE)
803 rc = segmented_read_std(ctxt, addr, address, op_bytes);
817 FASTOP1SRC2(mul, mul_ex);
818 FASTOP1SRC2(imul, imul_ex);
819 FASTOP1SRC2EX(div, div_ex);
820 FASTOP1SRC2EX(idiv, idiv_ex);
849 static u8 test_cc(unsigned int condition, unsigned long flags)
852 void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
854 flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
855 asm("push %[flags]; popf; call *%[fastop]"
856 : "=a"(rc) : [fastop]"r"(fop), [flags]"r"(flags));
860 static void fetch_register_operand(struct operand *op)
864 op->val = *(u8 *)op->addr.reg;
867 op->val = *(u16 *)op->addr.reg;
870 op->val = *(u32 *)op->addr.reg;
873 op->val = *(u64 *)op->addr.reg;
878 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
880 ctxt->ops->get_fpu(ctxt);
882 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
883 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
884 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
885 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
886 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
887 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
888 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
889 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
891 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
892 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
893 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
894 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
895 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
896 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
897 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
898 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
902 ctxt->ops->put_fpu(ctxt);
905 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
908 ctxt->ops->get_fpu(ctxt);
910 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
911 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
912 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
913 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
914 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
915 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
916 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
917 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
919 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
920 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
921 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
922 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
923 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
924 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
925 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
926 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
930 ctxt->ops->put_fpu(ctxt);
933 static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
935 ctxt->ops->get_fpu(ctxt);
937 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
938 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
939 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
940 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
941 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
942 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
943 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
944 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
947 ctxt->ops->put_fpu(ctxt);
950 static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
952 ctxt->ops->get_fpu(ctxt);
954 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
955 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
956 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
957 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
958 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
959 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
960 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
961 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
964 ctxt->ops->put_fpu(ctxt);
967 static int em_fninit(struct x86_emulate_ctxt *ctxt)
969 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
970 return emulate_nm(ctxt);
972 ctxt->ops->get_fpu(ctxt);
973 asm volatile("fninit");
974 ctxt->ops->put_fpu(ctxt);
975 return X86EMUL_CONTINUE;
978 static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
982 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
983 return emulate_nm(ctxt);
985 ctxt->ops->get_fpu(ctxt);
986 asm volatile("fnstcw %0": "+m"(fcw));
987 ctxt->ops->put_fpu(ctxt);
989 /* force 2 byte destination */
993 return X86EMUL_CONTINUE;
996 static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
1000 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1001 return emulate_nm(ctxt);
1003 ctxt->ops->get_fpu(ctxt);
1004 asm volatile("fnstsw %0": "+m"(fsw));
1005 ctxt->ops->put_fpu(ctxt);
1007 /* force 2 byte destination */
1008 ctxt->dst.bytes = 2;
1009 ctxt->dst.val = fsw;
1011 return X86EMUL_CONTINUE;
1014 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1017 unsigned reg = ctxt->modrm_reg;
1019 if (!(ctxt->d & ModRM))
1020 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1022 if (ctxt->d & Sse) {
1026 read_sse_reg(ctxt, &op->vec_val, reg);
1029 if (ctxt->d & Mmx) {
1038 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1039 op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp);
1041 fetch_register_operand(op);
1042 op->orig_val = op->val;
1045 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1047 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1048 ctxt->modrm_seg = VCPU_SREG_SS;
1051 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1055 int index_reg, base_reg, scale;
1056 int rc = X86EMUL_CONTINUE;
1059 ctxt->modrm_reg = ((ctxt->rex_prefix << 1) & 8); /* REX.R */
1060 index_reg = (ctxt->rex_prefix << 2) & 8; /* REX.X */
1061 base_reg = (ctxt->rex_prefix << 3) & 8; /* REX.B */
1063 ctxt->modrm_mod = (ctxt->modrm & 0xc0) >> 6;
1064 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1065 ctxt->modrm_rm = base_reg | (ctxt->modrm & 0x07);
1066 ctxt->modrm_seg = VCPU_SREG_DS;
1068 if (ctxt->modrm_mod == 3 || (ctxt->d & NoMod)) {
1070 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1071 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,
1073 if (ctxt->d & Sse) {
1076 op->addr.xmm = ctxt->modrm_rm;
1077 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
1080 if (ctxt->d & Mmx) {
1083 op->addr.mm = ctxt->modrm_rm & 7;
1086 fetch_register_operand(op);
1092 if (ctxt->ad_bytes == 2) {
1093 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1094 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1095 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1096 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1098 /* 16-bit ModR/M decode. */
1099 switch (ctxt->modrm_mod) {
1101 if (ctxt->modrm_rm == 6)
1102 modrm_ea += insn_fetch(u16, ctxt);
1105 modrm_ea += insn_fetch(s8, ctxt);
1108 modrm_ea += insn_fetch(u16, ctxt);
1111 switch (ctxt->modrm_rm) {
1113 modrm_ea += bx + si;
1116 modrm_ea += bx + di;
1119 modrm_ea += bp + si;
1122 modrm_ea += bp + di;
1131 if (ctxt->modrm_mod != 0)
1138 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1139 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1140 ctxt->modrm_seg = VCPU_SREG_SS;
1141 modrm_ea = (u16)modrm_ea;
1143 /* 32/64-bit ModR/M decode. */
1144 if ((ctxt->modrm_rm & 7) == 4) {
1145 sib = insn_fetch(u8, ctxt);
1146 index_reg |= (sib >> 3) & 7;
1147 base_reg |= sib & 7;
1150 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1151 modrm_ea += insn_fetch(s32, ctxt);
1153 modrm_ea += reg_read(ctxt, base_reg);
1154 adjust_modrm_seg(ctxt, base_reg);
1157 modrm_ea += reg_read(ctxt, index_reg) << scale;
1158 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1159 if (ctxt->mode == X86EMUL_MODE_PROT64)
1160 ctxt->rip_relative = 1;
1162 base_reg = ctxt->modrm_rm;
1163 modrm_ea += reg_read(ctxt, base_reg);
1164 adjust_modrm_seg(ctxt, base_reg);
1166 switch (ctxt->modrm_mod) {
1168 if (ctxt->modrm_rm == 5)
1169 modrm_ea += insn_fetch(s32, ctxt);
1172 modrm_ea += insn_fetch(s8, ctxt);
1175 modrm_ea += insn_fetch(s32, ctxt);
1179 op->addr.mem.ea = modrm_ea;
1184 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1187 int rc = X86EMUL_CONTINUE;
1190 switch (ctxt->ad_bytes) {
1192 op->addr.mem.ea = insn_fetch(u16, ctxt);
1195 op->addr.mem.ea = insn_fetch(u32, ctxt);
1198 op->addr.mem.ea = insn_fetch(u64, ctxt);
1205 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1209 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1210 mask = ~((long)ctxt->dst.bytes * 8 - 1);
1212 if (ctxt->src.bytes == 2)
1213 sv = (s16)ctxt->src.val & (s16)mask;
1214 else if (ctxt->src.bytes == 4)
1215 sv = (s32)ctxt->src.val & (s32)mask;
1217 sv = (s64)ctxt->src.val & (s64)mask;
1219 ctxt->dst.addr.mem.ea += (sv >> 3);
1222 /* only subword offset */
1223 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1226 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1227 unsigned long addr, void *dest, unsigned size)
1230 struct read_cache *mc = &ctxt->mem_read;
1232 if (mc->pos < mc->end)
1235 WARN_ON((mc->end + size) >= sizeof(mc->data));
1237 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1239 if (rc != X86EMUL_CONTINUE)
1245 memcpy(dest, mc->data + mc->pos, size);
1247 return X86EMUL_CONTINUE;
1250 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1251 struct segmented_address addr,
1258 rc = linearize(ctxt, addr, size, false, &linear);
1259 if (rc != X86EMUL_CONTINUE)
1261 return read_emulated(ctxt, linear, data, size);
1264 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1265 struct segmented_address addr,
1272 rc = linearize(ctxt, addr, size, true, &linear);
1273 if (rc != X86EMUL_CONTINUE)
1275 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1279 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1280 struct segmented_address addr,
1281 const void *orig_data, const void *data,
1287 rc = linearize(ctxt, addr, size, true, &linear);
1288 if (rc != X86EMUL_CONTINUE)
1290 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1291 size, &ctxt->exception);
1294 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1295 unsigned int size, unsigned short port,
1298 struct read_cache *rc = &ctxt->io_read;
1300 if (rc->pos == rc->end) { /* refill pio read ahead */
1301 unsigned int in_page, n;
1302 unsigned int count = ctxt->rep_prefix ?
1303 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1304 in_page = (ctxt->eflags & EFLG_DF) ?
1305 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1306 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1307 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1311 rc->pos = rc->end = 0;
1312 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1317 if (ctxt->rep_prefix && (ctxt->d & String) &&
1318 !(ctxt->eflags & EFLG_DF)) {
1319 ctxt->dst.data = rc->data + rc->pos;
1320 ctxt->dst.type = OP_MEM_STR;
1321 ctxt->dst.count = (rc->end - rc->pos) / size;
1324 memcpy(dest, rc->data + rc->pos, size);
1330 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1331 u16 index, struct desc_struct *desc)
1336 ctxt->ops->get_idt(ctxt, &dt);
1338 if (dt.size < index * 8 + 7)
1339 return emulate_gp(ctxt, index << 3 | 0x2);
1341 addr = dt.address + index * 8;
1342 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1346 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1347 u16 selector, struct desc_ptr *dt)
1349 const struct x86_emulate_ops *ops = ctxt->ops;
1352 if (selector & 1 << 2) {
1353 struct desc_struct desc;
1356 memset (dt, 0, sizeof *dt);
1357 if (!ops->get_segment(ctxt, &sel, &desc, &base3,
1361 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1362 dt->address = get_desc_base(&desc) | ((u64)base3 << 32);
1364 ops->get_gdt(ctxt, dt);
1367 /* allowed just for 8 bytes segments */
1368 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1369 u16 selector, struct desc_struct *desc,
1373 u16 index = selector >> 3;
1376 get_descriptor_table_ptr(ctxt, selector, &dt);
1378 if (dt.size < index * 8 + 7)
1379 return emulate_gp(ctxt, selector & 0xfffc);
1381 *desc_addr_p = addr = dt.address + index * 8;
1382 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1386 /* allowed just for 8 bytes segments */
1387 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1388 u16 selector, struct desc_struct *desc)
1391 u16 index = selector >> 3;
1394 get_descriptor_table_ptr(ctxt, selector, &dt);
1396 if (dt.size < index * 8 + 7)
1397 return emulate_gp(ctxt, selector & 0xfffc);
1399 addr = dt.address + index * 8;
1400 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1404 /* Does not support long mode */
1405 static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1406 u16 selector, int seg, u8 cpl, bool in_task_switch)
1408 struct desc_struct seg_desc, old_desc;
1410 unsigned err_vec = GP_VECTOR;
1412 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1418 memset(&seg_desc, 0, sizeof seg_desc);
1420 if (ctxt->mode == X86EMUL_MODE_REAL) {
1421 /* set real mode segment descriptor (keep limit etc. for
1423 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1424 set_desc_base(&seg_desc, selector << 4);
1426 } else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {
1427 /* VM86 needs a clean new segment descriptor */
1428 set_desc_base(&seg_desc, selector << 4);
1429 set_desc_limit(&seg_desc, 0xffff);
1439 /* NULL selector is not valid for TR, CS and SS (except for long mode) */
1440 if ((seg == VCPU_SREG_CS
1441 || (seg == VCPU_SREG_SS
1442 && (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl))
1443 || seg == VCPU_SREG_TR)
1447 /* TR should be in GDT only */
1448 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1451 if (null_selector) /* for NULL selector skip all following checks */
1454 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1455 if (ret != X86EMUL_CONTINUE)
1458 err_code = selector & 0xfffc;
1459 err_vec = GP_VECTOR;
1461 /* can't load system descriptor into segment selector */
1462 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1466 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1475 * segment is not a writable data segment or segment
1476 * selector's RPL != CPL or segment selector's RPL != CPL
1478 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1482 if (in_task_switch && rpl != dpl)
1485 if (!(seg_desc.type & 8))
1488 if (seg_desc.type & 4) {
1494 if (rpl > cpl || dpl != cpl)
1497 /* CS(RPL) <- CPL */
1498 selector = (selector & 0xfffc) | cpl;
1501 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1503 old_desc = seg_desc;
1504 seg_desc.type |= 2; /* busy */
1505 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1506 sizeof(seg_desc), &ctxt->exception);
1507 if (ret != X86EMUL_CONTINUE)
1510 case VCPU_SREG_LDTR:
1511 if (seg_desc.s || seg_desc.type != 2)
1514 default: /* DS, ES, FS, or GS */
1516 * segment is not a data or readable code segment or
1517 * ((segment is a data or nonconforming code segment)
1518 * and (both RPL and CPL > DPL))
1520 if ((seg_desc.type & 0xa) == 0x8 ||
1521 (((seg_desc.type & 0xc) != 0xc) &&
1522 (rpl > dpl && cpl > dpl)))
1528 /* mark segment as accessed */
1530 ret = write_segment_descriptor(ctxt, selector, &seg_desc);
1531 if (ret != X86EMUL_CONTINUE)
1533 } else if (ctxt->mode == X86EMUL_MODE_PROT64) {
1534 ret = ctxt->ops->read_std(ctxt, desc_addr+8, &base3,
1535 sizeof(base3), &ctxt->exception);
1536 if (ret != X86EMUL_CONTINUE)
1540 ctxt->ops->set_segment(ctxt, selector, &seg_desc, base3, seg);
1541 return X86EMUL_CONTINUE;
1543 emulate_exception(ctxt, err_vec, err_code, true);
1544 return X86EMUL_PROPAGATE_FAULT;
1547 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1548 u16 selector, int seg)
1550 u8 cpl = ctxt->ops->cpl(ctxt);
1551 return __load_segment_descriptor(ctxt, selector, seg, cpl, false);
1554 static void write_register_operand(struct operand *op)
1556 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1557 switch (op->bytes) {
1559 *(u8 *)op->addr.reg = (u8)op->val;
1562 *(u16 *)op->addr.reg = (u16)op->val;
1565 *op->addr.reg = (u32)op->val;
1566 break; /* 64b: zero-extend */
1568 *op->addr.reg = op->val;
1573 static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)
1577 write_register_operand(op);
1580 if (ctxt->lock_prefix)
1581 return segmented_cmpxchg(ctxt,
1587 return segmented_write(ctxt,
1593 return segmented_write(ctxt,
1596 op->bytes * op->count);
1599 write_sse_reg(ctxt, &op->vec_val, op->addr.xmm);
1602 write_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
1610 return X86EMUL_CONTINUE;
1613 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1615 struct segmented_address addr;
1617 rsp_increment(ctxt, -bytes);
1618 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1619 addr.seg = VCPU_SREG_SS;
1621 return segmented_write(ctxt, addr, data, bytes);
1624 static int em_push(struct x86_emulate_ctxt *ctxt)
1626 /* Disable writeback. */
1627 ctxt->dst.type = OP_NONE;
1628 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1631 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1632 void *dest, int len)
1635 struct segmented_address addr;
1637 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1638 addr.seg = VCPU_SREG_SS;
1639 rc = segmented_read(ctxt, addr, dest, len);
1640 if (rc != X86EMUL_CONTINUE)
1643 rsp_increment(ctxt, len);
1647 static int em_pop(struct x86_emulate_ctxt *ctxt)
1649 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1652 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1653 void *dest, int len)
1656 unsigned long val, change_mask;
1657 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1658 int cpl = ctxt->ops->cpl(ctxt);
1660 rc = emulate_pop(ctxt, &val, len);
1661 if (rc != X86EMUL_CONTINUE)
1664 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1665 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1667 switch(ctxt->mode) {
1668 case X86EMUL_MODE_PROT64:
1669 case X86EMUL_MODE_PROT32:
1670 case X86EMUL_MODE_PROT16:
1672 change_mask |= EFLG_IOPL;
1674 change_mask |= EFLG_IF;
1676 case X86EMUL_MODE_VM86:
1678 return emulate_gp(ctxt, 0);
1679 change_mask |= EFLG_IF;
1681 default: /* real mode */
1682 change_mask |= (EFLG_IOPL | EFLG_IF);
1686 *(unsigned long *)dest =
1687 (ctxt->eflags & ~change_mask) | (val & change_mask);
1692 static int em_popf(struct x86_emulate_ctxt *ctxt)
1694 ctxt->dst.type = OP_REG;
1695 ctxt->dst.addr.reg = &ctxt->eflags;
1696 ctxt->dst.bytes = ctxt->op_bytes;
1697 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1700 static int em_enter(struct x86_emulate_ctxt *ctxt)
1703 unsigned frame_size = ctxt->src.val;
1704 unsigned nesting_level = ctxt->src2.val & 31;
1708 return X86EMUL_UNHANDLEABLE;
1710 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1711 rc = push(ctxt, &rbp, stack_size(ctxt));
1712 if (rc != X86EMUL_CONTINUE)
1714 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1716 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1717 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1719 return X86EMUL_CONTINUE;
1722 static int em_leave(struct x86_emulate_ctxt *ctxt)
1724 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1726 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1729 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1731 int seg = ctxt->src2.val;
1733 ctxt->src.val = get_segment_selector(ctxt, seg);
1735 return em_push(ctxt);
1738 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1740 int seg = ctxt->src2.val;
1741 unsigned long selector;
1744 rc = emulate_pop(ctxt, &selector, ctxt->op_bytes);
1745 if (rc != X86EMUL_CONTINUE)
1748 if (ctxt->modrm_reg == VCPU_SREG_SS)
1749 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
1751 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1755 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1757 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
1758 int rc = X86EMUL_CONTINUE;
1759 int reg = VCPU_REGS_RAX;
1761 while (reg <= VCPU_REGS_RDI) {
1762 (reg == VCPU_REGS_RSP) ?
1763 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
1766 if (rc != X86EMUL_CONTINUE)
1775 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1777 ctxt->src.val = (unsigned long)ctxt->eflags;
1778 return em_push(ctxt);
1781 static int em_popa(struct x86_emulate_ctxt *ctxt)
1783 int rc = X86EMUL_CONTINUE;
1784 int reg = VCPU_REGS_RDI;
1786 while (reg >= VCPU_REGS_RAX) {
1787 if (reg == VCPU_REGS_RSP) {
1788 rsp_increment(ctxt, ctxt->op_bytes);
1792 rc = emulate_pop(ctxt, reg_rmw(ctxt, reg), ctxt->op_bytes);
1793 if (rc != X86EMUL_CONTINUE)
1800 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1802 const struct x86_emulate_ops *ops = ctxt->ops;
1809 /* TODO: Add limit checks */
1810 ctxt->src.val = ctxt->eflags;
1812 if (rc != X86EMUL_CONTINUE)
1815 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1817 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1819 if (rc != X86EMUL_CONTINUE)
1822 ctxt->src.val = ctxt->_eip;
1824 if (rc != X86EMUL_CONTINUE)
1827 ops->get_idt(ctxt, &dt);
1829 eip_addr = dt.address + (irq << 2);
1830 cs_addr = dt.address + (irq << 2) + 2;
1832 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1833 if (rc != X86EMUL_CONTINUE)
1836 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1837 if (rc != X86EMUL_CONTINUE)
1840 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1841 if (rc != X86EMUL_CONTINUE)
1849 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1853 invalidate_registers(ctxt);
1854 rc = __emulate_int_real(ctxt, irq);
1855 if (rc == X86EMUL_CONTINUE)
1856 writeback_registers(ctxt);
1860 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
1862 switch(ctxt->mode) {
1863 case X86EMUL_MODE_REAL:
1864 return __emulate_int_real(ctxt, irq);
1865 case X86EMUL_MODE_VM86:
1866 case X86EMUL_MODE_PROT16:
1867 case X86EMUL_MODE_PROT32:
1868 case X86EMUL_MODE_PROT64:
1870 /* Protected mode interrupts unimplemented yet */
1871 return X86EMUL_UNHANDLEABLE;
1875 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
1877 int rc = X86EMUL_CONTINUE;
1878 unsigned long temp_eip = 0;
1879 unsigned long temp_eflags = 0;
1880 unsigned long cs = 0;
1881 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1882 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1883 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1884 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1886 /* TODO: Add stack limit check */
1888 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
1890 if (rc != X86EMUL_CONTINUE)
1893 if (temp_eip & ~0xffff)
1894 return emulate_gp(ctxt, 0);
1896 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1898 if (rc != X86EMUL_CONTINUE)
1901 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
1903 if (rc != X86EMUL_CONTINUE)
1906 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1908 if (rc != X86EMUL_CONTINUE)
1911 ctxt->_eip = temp_eip;
1914 if (ctxt->op_bytes == 4)
1915 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1916 else if (ctxt->op_bytes == 2) {
1917 ctxt->eflags &= ~0xffff;
1918 ctxt->eflags |= temp_eflags;
1921 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1922 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1927 static int em_iret(struct x86_emulate_ctxt *ctxt)
1929 switch(ctxt->mode) {
1930 case X86EMUL_MODE_REAL:
1931 return emulate_iret_real(ctxt);
1932 case X86EMUL_MODE_VM86:
1933 case X86EMUL_MODE_PROT16:
1934 case X86EMUL_MODE_PROT32:
1935 case X86EMUL_MODE_PROT64:
1937 /* iret from protected mode unimplemented yet */
1938 return X86EMUL_UNHANDLEABLE;
1942 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
1947 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1949 rc = load_segment_descriptor(ctxt, sel, VCPU_SREG_CS);
1950 if (rc != X86EMUL_CONTINUE)
1954 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
1955 return X86EMUL_CONTINUE;
1958 static int em_grp45(struct x86_emulate_ctxt *ctxt)
1960 int rc = X86EMUL_CONTINUE;
1962 switch (ctxt->modrm_reg) {
1963 case 2: /* call near abs */ {
1965 old_eip = ctxt->_eip;
1966 ctxt->_eip = ctxt->src.val;
1967 ctxt->src.val = old_eip;
1971 case 4: /* jmp abs */
1972 ctxt->_eip = ctxt->src.val;
1974 case 5: /* jmp far */
1975 rc = em_jmp_far(ctxt);
1984 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
1986 u64 old = ctxt->dst.orig_val64;
1988 if (ctxt->dst.bytes == 16)
1989 return X86EMUL_UNHANDLEABLE;
1991 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
1992 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
1993 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
1994 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
1995 ctxt->eflags &= ~EFLG_ZF;
1997 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
1998 (u32) reg_read(ctxt, VCPU_REGS_RBX);
2000 ctxt->eflags |= EFLG_ZF;
2002 return X86EMUL_CONTINUE;
2005 static int em_ret(struct x86_emulate_ctxt *ctxt)
2007 ctxt->dst.type = OP_REG;
2008 ctxt->dst.addr.reg = &ctxt->_eip;
2009 ctxt->dst.bytes = ctxt->op_bytes;
2010 return em_pop(ctxt);
2013 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2017 int cpl = ctxt->ops->cpl(ctxt);
2019 rc = emulate_pop(ctxt, &ctxt->_eip, ctxt->op_bytes);
2020 if (rc != X86EMUL_CONTINUE)
2022 if (ctxt->op_bytes == 4)
2023 ctxt->_eip = (u32)ctxt->_eip;
2024 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2025 if (rc != X86EMUL_CONTINUE)
2027 /* Outer-privilege level return is not implemented */
2028 if (ctxt->mode >= X86EMUL_MODE_PROT16 && (cs & 3) > cpl)
2029 return X86EMUL_UNHANDLEABLE;
2030 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2034 static int em_ret_far_imm(struct x86_emulate_ctxt *ctxt)
2038 rc = em_ret_far(ctxt);
2039 if (rc != X86EMUL_CONTINUE)
2041 rsp_increment(ctxt, ctxt->src.val);
2042 return X86EMUL_CONTINUE;
2045 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2047 /* Save real source value, then compare EAX against destination. */
2048 ctxt->dst.orig_val = ctxt->dst.val;
2049 ctxt->dst.val = reg_read(ctxt, VCPU_REGS_RAX);
2050 ctxt->src.orig_val = ctxt->src.val;
2051 ctxt->src.val = ctxt->dst.orig_val;
2052 fastop(ctxt, em_cmp);
2054 if (ctxt->eflags & EFLG_ZF) {
2055 /* Success: write back to memory. */
2056 ctxt->dst.val = ctxt->src.orig_val;
2058 /* Failure: write the value we saw to EAX. */
2059 ctxt->dst.type = OP_REG;
2060 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2061 ctxt->dst.val = ctxt->dst.orig_val;
2063 return X86EMUL_CONTINUE;
2066 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2068 int seg = ctxt->src2.val;
2072 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2074 rc = load_segment_descriptor(ctxt, sel, seg);
2075 if (rc != X86EMUL_CONTINUE)
2078 ctxt->dst.val = ctxt->src.val;
2083 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2084 struct desc_struct *cs, struct desc_struct *ss)
2086 cs->l = 0; /* will be adjusted later */
2087 set_desc_base(cs, 0); /* flat segment */
2088 cs->g = 1; /* 4kb granularity */
2089 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2090 cs->type = 0x0b; /* Read, Execute, Accessed */
2092 cs->dpl = 0; /* will be adjusted later */
2097 set_desc_base(ss, 0); /* flat segment */
2098 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2099 ss->g = 1; /* 4kb granularity */
2101 ss->type = 0x03; /* Read/Write, Accessed */
2102 ss->d = 1; /* 32bit stack segment */
2109 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2111 u32 eax, ebx, ecx, edx;
2114 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2115 return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2116 && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2117 && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
2120 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2122 const struct x86_emulate_ops *ops = ctxt->ops;
2123 u32 eax, ebx, ecx, edx;
2126 * syscall should always be enabled in longmode - so only become
2127 * vendor specific (cpuid) if other modes are active...
2129 if (ctxt->mode == X86EMUL_MODE_PROT64)
2134 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2136 * Intel ("GenuineIntel")
2137 * remark: Intel CPUs only support "syscall" in 64bit
2138 * longmode. Also an 64bit guest with a
2139 * 32bit compat-app running will #UD !! While this
2140 * behaviour can be fixed (by emulating) into AMD
2141 * response - CPUs of AMD can't behave like Intel.
2143 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2144 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2145 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
2148 /* AMD ("AuthenticAMD") */
2149 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2150 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2151 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
2154 /* AMD ("AMDisbetter!") */
2155 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2156 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2157 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2160 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
2164 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2166 const struct x86_emulate_ops *ops = ctxt->ops;
2167 struct desc_struct cs, ss;
2172 /* syscall is not available in real mode */
2173 if (ctxt->mode == X86EMUL_MODE_REAL ||
2174 ctxt->mode == X86EMUL_MODE_VM86)
2175 return emulate_ud(ctxt);
2177 if (!(em_syscall_is_enabled(ctxt)))
2178 return emulate_ud(ctxt);
2180 ops->get_msr(ctxt, MSR_EFER, &efer);
2181 setup_syscalls_segments(ctxt, &cs, &ss);
2183 if (!(efer & EFER_SCE))
2184 return emulate_ud(ctxt);
2186 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2188 cs_sel = (u16)(msr_data & 0xfffc);
2189 ss_sel = (u16)(msr_data + 8);
2191 if (efer & EFER_LMA) {
2195 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2196 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2198 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2199 if (efer & EFER_LMA) {
2200 #ifdef CONFIG_X86_64
2201 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags & ~EFLG_RF;
2204 ctxt->mode == X86EMUL_MODE_PROT64 ?
2205 MSR_LSTAR : MSR_CSTAR, &msr_data);
2206 ctxt->_eip = msr_data;
2208 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2209 ctxt->eflags &= ~(msr_data | EFLG_RF);
2213 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2214 ctxt->_eip = (u32)msr_data;
2216 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2219 return X86EMUL_CONTINUE;
2222 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2224 const struct x86_emulate_ops *ops = ctxt->ops;
2225 struct desc_struct cs, ss;
2230 ops->get_msr(ctxt, MSR_EFER, &efer);
2231 /* inject #GP if in real mode */
2232 if (ctxt->mode == X86EMUL_MODE_REAL)
2233 return emulate_gp(ctxt, 0);
2236 * Not recognized on AMD in compat mode (but is recognized in legacy
2239 if ((ctxt->mode == X86EMUL_MODE_PROT32) && (efer & EFER_LMA)
2240 && !vendor_intel(ctxt))
2241 return emulate_ud(ctxt);
2243 /* XXX sysenter/sysexit have not been tested in 64bit mode.
2244 * Therefore, we inject an #UD.
2246 if (ctxt->mode == X86EMUL_MODE_PROT64)
2247 return emulate_ud(ctxt);
2249 setup_syscalls_segments(ctxt, &cs, &ss);
2251 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2252 switch (ctxt->mode) {
2253 case X86EMUL_MODE_PROT32:
2254 if ((msr_data & 0xfffc) == 0x0)
2255 return emulate_gp(ctxt, 0);
2257 case X86EMUL_MODE_PROT64:
2258 if (msr_data == 0x0)
2259 return emulate_gp(ctxt, 0);
2265 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2266 cs_sel = (u16)msr_data;
2267 cs_sel &= ~SELECTOR_RPL_MASK;
2268 ss_sel = cs_sel + 8;
2269 ss_sel &= ~SELECTOR_RPL_MASK;
2270 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
2275 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2276 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2278 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2279 ctxt->_eip = msr_data;
2281 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2282 *reg_write(ctxt, VCPU_REGS_RSP) = msr_data;
2284 return X86EMUL_CONTINUE;
2287 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2289 const struct x86_emulate_ops *ops = ctxt->ops;
2290 struct desc_struct cs, ss;
2293 u16 cs_sel = 0, ss_sel = 0;
2295 /* inject #GP if in real mode or Virtual 8086 mode */
2296 if (ctxt->mode == X86EMUL_MODE_REAL ||
2297 ctxt->mode == X86EMUL_MODE_VM86)
2298 return emulate_gp(ctxt, 0);
2300 setup_syscalls_segments(ctxt, &cs, &ss);
2302 if ((ctxt->rex_prefix & 0x8) != 0x0)
2303 usermode = X86EMUL_MODE_PROT64;
2305 usermode = X86EMUL_MODE_PROT32;
2309 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2311 case X86EMUL_MODE_PROT32:
2312 cs_sel = (u16)(msr_data + 16);
2313 if ((msr_data & 0xfffc) == 0x0)
2314 return emulate_gp(ctxt, 0);
2315 ss_sel = (u16)(msr_data + 24);
2317 case X86EMUL_MODE_PROT64:
2318 cs_sel = (u16)(msr_data + 32);
2319 if (msr_data == 0x0)
2320 return emulate_gp(ctxt, 0);
2321 ss_sel = cs_sel + 8;
2326 cs_sel |= SELECTOR_RPL_MASK;
2327 ss_sel |= SELECTOR_RPL_MASK;
2329 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2330 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2332 ctxt->_eip = reg_read(ctxt, VCPU_REGS_RDX);
2333 *reg_write(ctxt, VCPU_REGS_RSP) = reg_read(ctxt, VCPU_REGS_RCX);
2335 return X86EMUL_CONTINUE;
2338 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2341 if (ctxt->mode == X86EMUL_MODE_REAL)
2343 if (ctxt->mode == X86EMUL_MODE_VM86)
2345 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2346 return ctxt->ops->cpl(ctxt) > iopl;
2349 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2352 const struct x86_emulate_ops *ops = ctxt->ops;
2353 struct desc_struct tr_seg;
2356 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2357 unsigned mask = (1 << len) - 1;
2360 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2363 if (desc_limit_scaled(&tr_seg) < 103)
2365 base = get_desc_base(&tr_seg);
2366 #ifdef CONFIG_X86_64
2367 base |= ((u64)base3) << 32;
2369 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2370 if (r != X86EMUL_CONTINUE)
2372 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2374 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2375 if (r != X86EMUL_CONTINUE)
2377 if ((perm >> bit_idx) & mask)
2382 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2388 if (emulator_bad_iopl(ctxt))
2389 if (!emulator_io_port_access_allowed(ctxt, port, len))
2392 ctxt->perm_ok = true;
2397 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2398 struct tss_segment_16 *tss)
2400 tss->ip = ctxt->_eip;
2401 tss->flag = ctxt->eflags;
2402 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
2403 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
2404 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
2405 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
2406 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
2407 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
2408 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
2409 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
2411 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2412 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2413 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2414 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2415 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2418 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2419 struct tss_segment_16 *tss)
2424 ctxt->_eip = tss->ip;
2425 ctxt->eflags = tss->flag | 2;
2426 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
2427 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
2428 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
2429 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
2430 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
2431 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
2432 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
2433 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
2436 * SDM says that segment selectors are loaded before segment
2439 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2440 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2441 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2442 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2443 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2448 * Now load segment descriptors. If fault happens at this stage
2449 * it is handled in a context of new task
2451 ret = __load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR, cpl, true);
2452 if (ret != X86EMUL_CONTINUE)
2454 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl, true);
2455 if (ret != X86EMUL_CONTINUE)
2457 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl, true);
2458 if (ret != X86EMUL_CONTINUE)
2460 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl, true);
2461 if (ret != X86EMUL_CONTINUE)
2463 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl, true);
2464 if (ret != X86EMUL_CONTINUE)
2467 return X86EMUL_CONTINUE;
2470 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2471 u16 tss_selector, u16 old_tss_sel,
2472 ulong old_tss_base, struct desc_struct *new_desc)
2474 const struct x86_emulate_ops *ops = ctxt->ops;
2475 struct tss_segment_16 tss_seg;
2477 u32 new_tss_base = get_desc_base(new_desc);
2479 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2481 if (ret != X86EMUL_CONTINUE)
2482 /* FIXME: need to provide precise fault address */
2485 save_state_to_tss16(ctxt, &tss_seg);
2487 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2489 if (ret != X86EMUL_CONTINUE)
2490 /* FIXME: need to provide precise fault address */
2493 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2495 if (ret != X86EMUL_CONTINUE)
2496 /* FIXME: need to provide precise fault address */
2499 if (old_tss_sel != 0xffff) {
2500 tss_seg.prev_task_link = old_tss_sel;
2502 ret = ops->write_std(ctxt, new_tss_base,
2503 &tss_seg.prev_task_link,
2504 sizeof tss_seg.prev_task_link,
2506 if (ret != X86EMUL_CONTINUE)
2507 /* FIXME: need to provide precise fault address */
2511 return load_state_from_tss16(ctxt, &tss_seg);
2514 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2515 struct tss_segment_32 *tss)
2517 /* CR3 and ldt selector are not saved intentionally */
2518 tss->eip = ctxt->_eip;
2519 tss->eflags = ctxt->eflags;
2520 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
2521 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
2522 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
2523 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
2524 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
2525 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
2526 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
2527 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
2529 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2530 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2531 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2532 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2533 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2534 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2537 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2538 struct tss_segment_32 *tss)
2543 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2544 return emulate_gp(ctxt, 0);
2545 ctxt->_eip = tss->eip;
2546 ctxt->eflags = tss->eflags | 2;
2548 /* General purpose registers */
2549 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
2550 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
2551 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
2552 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
2553 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
2554 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
2555 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
2556 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
2559 * SDM says that segment selectors are loaded before segment
2560 * descriptors. This is important because CPL checks will
2563 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2564 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2565 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2566 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2567 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2568 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2569 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2572 * If we're switching between Protected Mode and VM86, we need to make
2573 * sure to update the mode before loading the segment descriptors so
2574 * that the selectors are interpreted correctly.
2576 if (ctxt->eflags & X86_EFLAGS_VM) {
2577 ctxt->mode = X86EMUL_MODE_VM86;
2580 ctxt->mode = X86EMUL_MODE_PROT32;
2585 * Now load segment descriptors. If fault happenes at this stage
2586 * it is handled in a context of new task
2588 ret = __load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR, cpl, true);
2589 if (ret != X86EMUL_CONTINUE)
2591 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl, true);
2592 if (ret != X86EMUL_CONTINUE)
2594 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl, true);
2595 if (ret != X86EMUL_CONTINUE)
2597 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl, true);
2598 if (ret != X86EMUL_CONTINUE)
2600 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl, true);
2601 if (ret != X86EMUL_CONTINUE)
2603 ret = __load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS, cpl, true);
2604 if (ret != X86EMUL_CONTINUE)
2606 ret = __load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS, cpl, true);
2607 if (ret != X86EMUL_CONTINUE)
2610 return X86EMUL_CONTINUE;
2613 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2614 u16 tss_selector, u16 old_tss_sel,
2615 ulong old_tss_base, struct desc_struct *new_desc)
2617 const struct x86_emulate_ops *ops = ctxt->ops;
2618 struct tss_segment_32 tss_seg;
2620 u32 new_tss_base = get_desc_base(new_desc);
2621 u32 eip_offset = offsetof(struct tss_segment_32, eip);
2622 u32 ldt_sel_offset = offsetof(struct tss_segment_32, ldt_selector);
2624 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2626 if (ret != X86EMUL_CONTINUE)
2627 /* FIXME: need to provide precise fault address */
2630 save_state_to_tss32(ctxt, &tss_seg);
2632 /* Only GP registers and segment selectors are saved */
2633 ret = ops->write_std(ctxt, old_tss_base + eip_offset, &tss_seg.eip,
2634 ldt_sel_offset - eip_offset, &ctxt->exception);
2635 if (ret != X86EMUL_CONTINUE)
2636 /* FIXME: need to provide precise fault address */
2639 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2641 if (ret != X86EMUL_CONTINUE)
2642 /* FIXME: need to provide precise fault address */
2645 if (old_tss_sel != 0xffff) {
2646 tss_seg.prev_task_link = old_tss_sel;
2648 ret = ops->write_std(ctxt, new_tss_base,
2649 &tss_seg.prev_task_link,
2650 sizeof tss_seg.prev_task_link,
2652 if (ret != X86EMUL_CONTINUE)
2653 /* FIXME: need to provide precise fault address */
2657 return load_state_from_tss32(ctxt, &tss_seg);
2660 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2661 u16 tss_selector, int idt_index, int reason,
2662 bool has_error_code, u32 error_code)
2664 const struct x86_emulate_ops *ops = ctxt->ops;
2665 struct desc_struct curr_tss_desc, next_tss_desc;
2667 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2668 ulong old_tss_base =
2669 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2673 /* FIXME: old_tss_base == ~0 ? */
2675 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
2676 if (ret != X86EMUL_CONTINUE)
2678 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
2679 if (ret != X86EMUL_CONTINUE)
2682 /* FIXME: check that next_tss_desc is tss */
2685 * Check privileges. The three cases are task switch caused by...
2687 * 1. jmp/call/int to task gate: Check against DPL of the task gate
2688 * 2. Exception/IRQ/iret: No check is performed
2689 * 3. jmp/call to TSS: Check against DPL of the TSS
2691 if (reason == TASK_SWITCH_GATE) {
2692 if (idt_index != -1) {
2693 /* Software interrupts */
2694 struct desc_struct task_gate_desc;
2697 ret = read_interrupt_descriptor(ctxt, idt_index,
2699 if (ret != X86EMUL_CONTINUE)
2702 dpl = task_gate_desc.dpl;
2703 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2704 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
2706 } else if (reason != TASK_SWITCH_IRET) {
2707 int dpl = next_tss_desc.dpl;
2708 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2709 return emulate_gp(ctxt, tss_selector);
2713 desc_limit = desc_limit_scaled(&next_tss_desc);
2714 if (!next_tss_desc.p ||
2715 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2716 desc_limit < 0x2b)) {
2717 emulate_ts(ctxt, tss_selector & 0xfffc);
2718 return X86EMUL_PROPAGATE_FAULT;
2721 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2722 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2723 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2726 if (reason == TASK_SWITCH_IRET)
2727 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2729 /* set back link to prev task only if NT bit is set in eflags
2730 note that old_tss_sel is not used after this point */
2731 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2732 old_tss_sel = 0xffff;
2734 if (next_tss_desc.type & 8)
2735 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2736 old_tss_base, &next_tss_desc);
2738 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2739 old_tss_base, &next_tss_desc);
2740 if (ret != X86EMUL_CONTINUE)
2743 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2744 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2746 if (reason != TASK_SWITCH_IRET) {
2747 next_tss_desc.type |= (1 << 1); /* set busy flag */
2748 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2751 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2752 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2754 if (has_error_code) {
2755 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2756 ctxt->lock_prefix = 0;
2757 ctxt->src.val = (unsigned long) error_code;
2758 ret = em_push(ctxt);
2764 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2765 u16 tss_selector, int idt_index, int reason,
2766 bool has_error_code, u32 error_code)
2770 invalidate_registers(ctxt);
2771 ctxt->_eip = ctxt->eip;
2772 ctxt->dst.type = OP_NONE;
2774 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
2775 has_error_code, error_code);
2777 if (rc == X86EMUL_CONTINUE) {
2778 ctxt->eip = ctxt->_eip;
2779 writeback_registers(ctxt);
2782 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2785 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
2788 int df = (ctxt->eflags & EFLG_DF) ? -op->count : op->count;
2790 register_address_increment(ctxt, reg_rmw(ctxt, reg), df * op->bytes);
2791 op->addr.mem.ea = register_address(ctxt, reg_read(ctxt, reg));
2794 static int em_das(struct x86_emulate_ctxt *ctxt)
2797 bool af, cf, old_cf;
2799 cf = ctxt->eflags & X86_EFLAGS_CF;
2805 af = ctxt->eflags & X86_EFLAGS_AF;
2806 if ((al & 0x0f) > 9 || af) {
2808 cf = old_cf | (al >= 250);
2813 if (old_al > 0x99 || old_cf) {
2819 /* Set PF, ZF, SF */
2820 ctxt->src.type = OP_IMM;
2822 ctxt->src.bytes = 1;
2823 fastop(ctxt, em_or);
2824 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2826 ctxt->eflags |= X86_EFLAGS_CF;
2828 ctxt->eflags |= X86_EFLAGS_AF;
2829 return X86EMUL_CONTINUE;
2832 static int em_aam(struct x86_emulate_ctxt *ctxt)
2836 if (ctxt->src.val == 0)
2837 return emulate_de(ctxt);
2839 al = ctxt->dst.val & 0xff;
2840 ah = al / ctxt->src.val;
2841 al %= ctxt->src.val;
2843 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);
2845 /* Set PF, ZF, SF */
2846 ctxt->src.type = OP_IMM;
2848 ctxt->src.bytes = 1;
2849 fastop(ctxt, em_or);
2851 return X86EMUL_CONTINUE;
2854 static int em_aad(struct x86_emulate_ctxt *ctxt)
2856 u8 al = ctxt->dst.val & 0xff;
2857 u8 ah = (ctxt->dst.val >> 8) & 0xff;
2859 al = (al + (ah * ctxt->src.val)) & 0xff;
2861 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
2863 /* Set PF, ZF, SF */
2864 ctxt->src.type = OP_IMM;
2866 ctxt->src.bytes = 1;
2867 fastop(ctxt, em_or);
2869 return X86EMUL_CONTINUE;
2872 static int em_call(struct x86_emulate_ctxt *ctxt)
2874 long rel = ctxt->src.val;
2876 ctxt->src.val = (unsigned long)ctxt->_eip;
2878 return em_push(ctxt);
2881 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2887 old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2888 old_eip = ctxt->_eip;
2890 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2891 if (load_segment_descriptor(ctxt, sel, VCPU_SREG_CS))
2892 return X86EMUL_CONTINUE;
2895 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
2897 ctxt->src.val = old_cs;
2899 if (rc != X86EMUL_CONTINUE)
2902 ctxt->src.val = old_eip;
2903 return em_push(ctxt);
2906 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
2910 ctxt->dst.type = OP_REG;
2911 ctxt->dst.addr.reg = &ctxt->_eip;
2912 ctxt->dst.bytes = ctxt->op_bytes;
2913 rc = emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
2914 if (rc != X86EMUL_CONTINUE)
2916 rsp_increment(ctxt, ctxt->src.val);
2917 return X86EMUL_CONTINUE;
2920 static int em_xchg(struct x86_emulate_ctxt *ctxt)
2922 /* Write back the register source. */
2923 ctxt->src.val = ctxt->dst.val;
2924 write_register_operand(&ctxt->src);
2926 /* Write back the memory destination with implicit LOCK prefix. */
2927 ctxt->dst.val = ctxt->src.orig_val;
2928 ctxt->lock_prefix = 1;
2929 return X86EMUL_CONTINUE;
2932 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
2934 ctxt->dst.val = ctxt->src2.val;
2935 return fastop(ctxt, em_imul);
2938 static int em_cwd(struct x86_emulate_ctxt *ctxt)
2940 ctxt->dst.type = OP_REG;
2941 ctxt->dst.bytes = ctxt->src.bytes;
2942 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
2943 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
2945 return X86EMUL_CONTINUE;
2948 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
2952 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
2953 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
2954 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
2955 return X86EMUL_CONTINUE;
2958 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
2962 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
2963 return emulate_gp(ctxt, 0);
2964 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
2965 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
2966 return X86EMUL_CONTINUE;
2969 static int em_mov(struct x86_emulate_ctxt *ctxt)
2971 memcpy(ctxt->dst.valptr, ctxt->src.valptr, sizeof(ctxt->src.valptr));
2972 return X86EMUL_CONTINUE;
2975 #define FFL(x) bit(X86_FEATURE_##x)
2977 static int em_movbe(struct x86_emulate_ctxt *ctxt)
2979 u32 ebx, ecx, edx, eax = 1;
2983 * Check MOVBE is set in the guest-visible CPUID leaf.
2985 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2986 if (!(ecx & FFL(MOVBE)))
2987 return emulate_ud(ctxt);
2989 switch (ctxt->op_bytes) {
2992 * From MOVBE definition: "...When the operand size is 16 bits,
2993 * the upper word of the destination register remains unchanged
2996 * Both casting ->valptr and ->val to u16 breaks strict aliasing
2997 * rules so we have to do the operation almost per hand.
2999 tmp = (u16)ctxt->src.val;
3000 ctxt->dst.val &= ~0xffffUL;
3001 ctxt->dst.val |= (unsigned long)swab16(tmp);
3004 ctxt->dst.val = swab32((u32)ctxt->src.val);
3007 ctxt->dst.val = swab64(ctxt->src.val);
3010 return X86EMUL_PROPAGATE_FAULT;
3012 return X86EMUL_CONTINUE;
3015 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3017 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
3018 return emulate_gp(ctxt, 0);
3020 /* Disable writeback. */
3021 ctxt->dst.type = OP_NONE;
3022 return X86EMUL_CONTINUE;
3025 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
3029 if (ctxt->mode == X86EMUL_MODE_PROT64)
3030 val = ctxt->src.val & ~0ULL;
3032 val = ctxt->src.val & ~0U;
3034 /* #UD condition is already handled. */
3035 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
3036 return emulate_gp(ctxt, 0);
3038 /* Disable writeback. */
3039 ctxt->dst.type = OP_NONE;
3040 return X86EMUL_CONTINUE;
3043 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3047 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3048 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3049 if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
3050 return emulate_gp(ctxt, 0);
3052 return X86EMUL_CONTINUE;
3055 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3059 if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3060 return emulate_gp(ctxt, 0);
3062 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3063 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3064 return X86EMUL_CONTINUE;
3067 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3069 if (ctxt->modrm_reg > VCPU_SREG_GS)
3070 return emulate_ud(ctxt);
3072 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
3073 return X86EMUL_CONTINUE;
3076 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3078 u16 sel = ctxt->src.val;
3080 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3081 return emulate_ud(ctxt);
3083 if (ctxt->modrm_reg == VCPU_SREG_SS)
3084 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3086 /* Disable writeback. */
3087 ctxt->dst.type = OP_NONE;
3088 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3091 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3093 u16 sel = ctxt->src.val;
3095 /* Disable writeback. */
3096 ctxt->dst.type = OP_NONE;
3097 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3100 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3102 u16 sel = ctxt->src.val;
3104 /* Disable writeback. */
3105 ctxt->dst.type = OP_NONE;
3106 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3109 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3114 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3115 if (rc == X86EMUL_CONTINUE)
3116 ctxt->ops->invlpg(ctxt, linear);
3117 /* Disable writeback. */
3118 ctxt->dst.type = OP_NONE;
3119 return X86EMUL_CONTINUE;
3122 static int em_clts(struct x86_emulate_ctxt *ctxt)
3126 cr0 = ctxt->ops->get_cr(ctxt, 0);
3128 ctxt->ops->set_cr(ctxt, 0, cr0);
3129 return X86EMUL_CONTINUE;
3132 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
3136 if (ctxt->modrm_mod != 3 || ctxt->modrm_rm != 1)
3137 return X86EMUL_UNHANDLEABLE;
3139 rc = ctxt->ops->fix_hypercall(ctxt);
3140 if (rc != X86EMUL_CONTINUE)
3143 /* Let the processor re-execute the fixed hypercall */
3144 ctxt->_eip = ctxt->eip;
3145 /* Disable writeback. */
3146 ctxt->dst.type = OP_NONE;
3147 return X86EMUL_CONTINUE;
3150 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3151 void (*get)(struct x86_emulate_ctxt *ctxt,
3152 struct desc_ptr *ptr))
3154 struct desc_ptr desc_ptr;
3156 if (ctxt->mode == X86EMUL_MODE_PROT64)
3158 get(ctxt, &desc_ptr);
3159 if (ctxt->op_bytes == 2) {
3161 desc_ptr.address &= 0x00ffffff;
3163 /* Disable writeback. */
3164 ctxt->dst.type = OP_NONE;
3165 return segmented_write(ctxt, ctxt->dst.addr.mem,
3166 &desc_ptr, 2 + ctxt->op_bytes);
3169 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3171 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3174 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3176 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3179 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3181 struct desc_ptr desc_ptr;
3184 if (ctxt->mode == X86EMUL_MODE_PROT64)
3186 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3187 &desc_ptr.size, &desc_ptr.address,
3189 if (rc != X86EMUL_CONTINUE)
3191 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3192 /* Disable writeback. */
3193 ctxt->dst.type = OP_NONE;
3194 return X86EMUL_CONTINUE;
3197 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
3201 rc = ctxt->ops->fix_hypercall(ctxt);
3203 /* Disable writeback. */
3204 ctxt->dst.type = OP_NONE;
3208 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3210 struct desc_ptr desc_ptr;
3213 if (ctxt->mode == X86EMUL_MODE_PROT64)
3215 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3216 &desc_ptr.size, &desc_ptr.address,
3218 if (rc != X86EMUL_CONTINUE)
3220 ctxt->ops->set_idt(ctxt, &desc_ptr);
3221 /* Disable writeback. */
3222 ctxt->dst.type = OP_NONE;
3223 return X86EMUL_CONTINUE;
3226 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3228 if (ctxt->dst.type == OP_MEM)
3229 ctxt->dst.bytes = 2;
3230 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3231 return X86EMUL_CONTINUE;
3234 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3236 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3237 | (ctxt->src.val & 0x0f));
3238 ctxt->dst.type = OP_NONE;
3239 return X86EMUL_CONTINUE;
3242 static int em_loop(struct x86_emulate_ctxt *ctxt)
3244 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX), -1);
3245 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3246 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3247 jmp_rel(ctxt, ctxt->src.val);
3249 return X86EMUL_CONTINUE;
3252 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3254 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3255 jmp_rel(ctxt, ctxt->src.val);
3257 return X86EMUL_CONTINUE;
3260 static int em_in(struct x86_emulate_ctxt *ctxt)
3262 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3264 return X86EMUL_IO_NEEDED;
3266 return X86EMUL_CONTINUE;
3269 static int em_out(struct x86_emulate_ctxt *ctxt)
3271 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3273 /* Disable writeback. */
3274 ctxt->dst.type = OP_NONE;
3275 return X86EMUL_CONTINUE;
3278 static int em_cli(struct x86_emulate_ctxt *ctxt)
3280 if (emulator_bad_iopl(ctxt))
3281 return emulate_gp(ctxt, 0);
3283 ctxt->eflags &= ~X86_EFLAGS_IF;
3284 return X86EMUL_CONTINUE;
3287 static int em_sti(struct x86_emulate_ctxt *ctxt)
3289 if (emulator_bad_iopl(ctxt))
3290 return emulate_gp(ctxt, 0);
3292 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3293 ctxt->eflags |= X86_EFLAGS_IF;
3294 return X86EMUL_CONTINUE;
3297 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3299 u32 eax, ebx, ecx, edx;
3301 eax = reg_read(ctxt, VCPU_REGS_RAX);
3302 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3303 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3304 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3305 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3306 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3307 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
3308 return X86EMUL_CONTINUE;
3311 static int em_sahf(struct x86_emulate_ctxt *ctxt)
3315 flags = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF;
3316 flags &= *reg_rmw(ctxt, VCPU_REGS_RAX) >> 8;
3318 ctxt->eflags &= ~0xffUL;
3319 ctxt->eflags |= flags | X86_EFLAGS_FIXED;
3320 return X86EMUL_CONTINUE;
3323 static int em_lahf(struct x86_emulate_ctxt *ctxt)
3325 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
3326 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
3327 return X86EMUL_CONTINUE;
3330 static int em_bswap(struct x86_emulate_ctxt *ctxt)
3332 switch (ctxt->op_bytes) {
3333 #ifdef CONFIG_X86_64
3335 asm("bswap %0" : "+r"(ctxt->dst.val));
3339 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
3342 return X86EMUL_CONTINUE;
3345 static bool valid_cr(int nr)
3357 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
3359 if (!valid_cr(ctxt->modrm_reg))
3360 return emulate_ud(ctxt);
3362 return X86EMUL_CONTINUE;
3365 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
3367 u64 new_val = ctxt->src.val64;
3368 int cr = ctxt->modrm_reg;
3371 static u64 cr_reserved_bits[] = {
3372 0xffffffff00000000ULL,
3373 0, 0, 0, /* CR3 checked later */
3380 return emulate_ud(ctxt);
3382 if (new_val & cr_reserved_bits[cr])
3383 return emulate_gp(ctxt, 0);
3388 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
3389 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
3390 return emulate_gp(ctxt, 0);
3392 cr4 = ctxt->ops->get_cr(ctxt, 4);
3393 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3395 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
3396 !(cr4 & X86_CR4_PAE))
3397 return emulate_gp(ctxt, 0);
3404 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3405 if (efer & EFER_LMA)
3406 rsvd = CR3_L_MODE_RESERVED_BITS;
3409 return emulate_gp(ctxt, 0);
3414 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3416 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
3417 return emulate_gp(ctxt, 0);
3423 return X86EMUL_CONTINUE;
3426 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
3430 ctxt->ops->get_dr(ctxt, 7, &dr7);
3432 /* Check if DR7.Global_Enable is set */
3433 return dr7 & (1 << 13);
3436 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
3438 int dr = ctxt->modrm_reg;
3442 return emulate_ud(ctxt);
3444 cr4 = ctxt->ops->get_cr(ctxt, 4);
3445 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
3446 return emulate_ud(ctxt);
3448 if (check_dr7_gd(ctxt))
3449 return emulate_db(ctxt);
3451 return X86EMUL_CONTINUE;
3454 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
3456 u64 new_val = ctxt->src.val64;
3457 int dr = ctxt->modrm_reg;
3459 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
3460 return emulate_gp(ctxt, 0);
3462 return check_dr_read(ctxt);
3465 static int check_svme(struct x86_emulate_ctxt *ctxt)
3469 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3471 if (!(efer & EFER_SVME))
3472 return emulate_ud(ctxt);
3474 return X86EMUL_CONTINUE;
3477 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
3479 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
3481 /* Valid physical address? */
3482 if (rax & 0xffff000000000000ULL)
3483 return emulate_gp(ctxt, 0);
3485 return check_svme(ctxt);
3488 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
3490 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3492 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
3493 return emulate_ud(ctxt);
3495 return X86EMUL_CONTINUE;
3498 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
3500 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3501 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
3503 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
3504 ctxt->ops->check_pmc(ctxt, rcx))
3505 return emulate_gp(ctxt, 0);
3507 return X86EMUL_CONTINUE;
3510 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
3512 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
3513 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
3514 return emulate_gp(ctxt, 0);
3516 return X86EMUL_CONTINUE;
3519 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
3521 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
3522 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
3523 return emulate_gp(ctxt, 0);
3525 return X86EMUL_CONTINUE;
3528 #define D(_y) { .flags = (_y) }
3529 #define DI(_y, _i) { .flags = (_y)|Intercept, .intercept = x86_intercept_##_i }
3530 #define DIP(_y, _i, _p) { .flags = (_y)|Intercept|CheckPerm, \
3531 .intercept = x86_intercept_##_i, .check_perm = (_p) }
3532 #define N D(NotImpl)
3533 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3534 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
3535 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
3536 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
3537 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3538 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
3539 #define II(_f, _e, _i) \
3540 { .flags = (_f)|Intercept, .u.execute = (_e), .intercept = x86_intercept_##_i }
3541 #define IIP(_f, _e, _i, _p) \
3542 { .flags = (_f)|Intercept|CheckPerm, .u.execute = (_e), \
3543 .intercept = x86_intercept_##_i, .check_perm = (_p) }
3544 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3546 #define D2bv(_f) D((_f) | ByteOp), D(_f)
3547 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3548 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
3549 #define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
3550 #define I2bvIP(_f, _e, _i, _p) \
3551 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
3553 #define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
3554 F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3555 F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3557 static const struct opcode group7_rm1[] = {
3558 DI(SrcNone | Priv, monitor),
3559 DI(SrcNone | Priv, mwait),
3563 static const struct opcode group7_rm3[] = {
3564 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
3565 II(SrcNone | Prot | EmulateOnUD, em_vmmcall, vmmcall),
3566 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
3567 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
3568 DIP(SrcNone | Prot | Priv, stgi, check_svme),
3569 DIP(SrcNone | Prot | Priv, clgi, check_svme),
3570 DIP(SrcNone | Prot | Priv, skinit, check_svme),
3571 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
3574 static const struct opcode group7_rm7[] = {
3576 DIP(SrcNone, rdtscp, check_rdtsc),
3580 static const struct opcode group1[] = {
3582 F(Lock | PageTable, em_or),
3585 F(Lock | PageTable, em_and),
3591 static const struct opcode group1A[] = {
3592 I(DstMem | SrcNone | Mov | Stack, em_pop), N, N, N, N, N, N, N,
3595 static const struct opcode group2[] = {
3596 F(DstMem | ModRM, em_rol),
3597 F(DstMem | ModRM, em_ror),
3598 F(DstMem | ModRM, em_rcl),
3599 F(DstMem | ModRM, em_rcr),
3600 F(DstMem | ModRM, em_shl),
3601 F(DstMem | ModRM, em_shr),
3602 F(DstMem | ModRM, em_shl),
3603 F(DstMem | ModRM, em_sar),
3606 static const struct opcode group3[] = {
3607 F(DstMem | SrcImm | NoWrite, em_test),
3608 F(DstMem | SrcImm | NoWrite, em_test),
3609 F(DstMem | SrcNone | Lock, em_not),
3610 F(DstMem | SrcNone | Lock, em_neg),
3611 F(DstXacc | Src2Mem, em_mul_ex),
3612 F(DstXacc | Src2Mem, em_imul_ex),
3613 F(DstXacc | Src2Mem, em_div_ex),
3614 F(DstXacc | Src2Mem, em_idiv_ex),
3617 static const struct opcode group4[] = {
3618 F(ByteOp | DstMem | SrcNone | Lock, em_inc),
3619 F(ByteOp | DstMem | SrcNone | Lock, em_dec),
3623 static const struct opcode group5[] = {
3624 F(DstMem | SrcNone | Lock, em_inc),
3625 F(DstMem | SrcNone | Lock, em_dec),
3626 I(SrcMem | Stack, em_grp45),
3627 I(SrcMemFAddr | ImplicitOps | Stack, em_call_far),
3628 I(SrcMem | Stack, em_grp45),
3629 I(SrcMemFAddr | ImplicitOps, em_grp45),
3630 I(SrcMem | Stack, em_grp45), D(Undefined),
3633 static const struct opcode group6[] = {
3636 II(Prot | Priv | SrcMem16, em_lldt, lldt),
3637 II(Prot | Priv | SrcMem16, em_ltr, ltr),
3641 static const struct group_dual group7 = { {
3642 II(Mov | DstMem, em_sgdt, sgdt),
3643 II(Mov | DstMem, em_sidt, sidt),
3644 II(SrcMem | Priv, em_lgdt, lgdt),
3645 II(SrcMem | Priv, em_lidt, lidt),
3646 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3647 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3648 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3650 I(SrcNone | Priv | EmulateOnUD, em_vmcall),
3652 N, EXT(0, group7_rm3),
3653 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3654 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3658 static const struct opcode group8[] = {
3660 F(DstMem | SrcImmByte | NoWrite, em_bt),
3661 F(DstMem | SrcImmByte | Lock | PageTable, em_bts),
3662 F(DstMem | SrcImmByte | Lock, em_btr),
3663 F(DstMem | SrcImmByte | Lock | PageTable, em_btc),
3666 static const struct group_dual group9 = { {
3667 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
3669 N, N, N, N, N, N, N, N,
3672 static const struct opcode group11[] = {
3673 I(DstMem | SrcImm | Mov | PageTable, em_mov),
3677 static const struct gprefix pfx_0f_6f_0f_7f = {
3678 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
3681 static const struct gprefix pfx_vmovntpx = {
3682 I(0, em_mov), N, N, N,
3685 static const struct gprefix pfx_0f_28_0f_29 = {
3686 I(Aligned, em_mov), I(Aligned, em_mov), N, N,
3689 static const struct escape escape_d9 = { {
3690 N, N, N, N, N, N, N, I(DstMem, em_fnstcw),
3693 N, N, N, N, N, N, N, N,
3695 N, N, N, N, N, N, N, N,
3697 N, N, N, N, N, N, N, N,
3699 N, N, N, N, N, N, N, N,
3701 N, N, N, N, N, N, N, N,
3703 N, N, N, N, N, N, N, N,
3705 N, N, N, N, N, N, N, N,
3707 N, N, N, N, N, N, N, N,
3710 static const struct escape escape_db = { {
3711 N, N, N, N, N, N, N, N,
3714 N, N, N, N, N, N, N, N,
3716 N, N, N, N, N, N, N, N,
3718 N, N, N, N, N, N, N, N,
3720 N, N, N, N, N, N, N, N,
3722 N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
3724 N, N, N, N, N, N, N, N,
3726 N, N, N, N, N, N, N, N,
3728 N, N, N, N, N, N, N, N,
3731 static const struct escape escape_dd = { {
3732 N, N, N, N, N, N, N, I(DstMem, em_fnstsw),
3735 N, N, N, N, N, N, N, N,
3737 N, N, N, N, N, N, N, N,
3739 N, N, N, N, N, N, N, N,
3741 N, N, N, N, N, N, N, N,
3743 N, N, N, N, N, N, N, N,
3745 N, N, N, N, N, N, N, N,
3747 N, N, N, N, N, N, N, N,
3749 N, N, N, N, N, N, N, N,
3752 static const struct opcode opcode_table[256] = {
3754 F6ALU(Lock, em_add),
3755 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
3756 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
3758 F6ALU(Lock | PageTable, em_or),
3759 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
3762 F6ALU(Lock, em_adc),
3763 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
3764 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
3766 F6ALU(Lock, em_sbb),
3767 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
3768 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
3770 F6ALU(Lock | PageTable, em_and), N, N,
3772 F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3774 F6ALU(Lock, em_xor), N, N,
3776 F6ALU(NoWrite, em_cmp), N, N,
3778 X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
3780 X8(I(SrcReg | Stack, em_push)),
3782 X8(I(DstReg | Stack, em_pop)),
3784 I(ImplicitOps | Stack | No64, em_pusha),
3785 I(ImplicitOps | Stack | No64, em_popa),
3786 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
3789 I(SrcImm | Mov | Stack, em_push),
3790 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3791 I(SrcImmByte | Mov | Stack, em_push),
3792 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3793 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
3794 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
3798 G(ByteOp | DstMem | SrcImm, group1),
3799 G(DstMem | SrcImm, group1),
3800 G(ByteOp | DstMem | SrcImm | No64, group1),
3801 G(DstMem | SrcImmByte, group1),
3802 F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
3803 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
3805 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
3806 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3807 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
3808 D(ModRM | SrcMem | NoAccess | DstReg),
3809 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
3812 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3814 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3815 I(SrcImmFAddr | No64, em_call_far), N,
3816 II(ImplicitOps | Stack, em_pushf, pushf),
3817 II(ImplicitOps | Stack, em_popf, popf),
3818 I(ImplicitOps, em_sahf), I(ImplicitOps, em_lahf),
3820 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3821 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
3822 I2bv(SrcSI | DstDI | Mov | String, em_mov),
3823 F2bv(SrcSI | DstDI | String | NoWrite, em_cmp),
3825 F2bv(DstAcc | SrcImm | NoWrite, em_test),
3826 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
3827 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3828 F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp),
3830 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3832 X8(I(DstReg | SrcImm64 | Mov, em_mov)),
3834 G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
3835 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3836 I(ImplicitOps | Stack, em_ret),
3837 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
3838 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
3839 G(ByteOp, group11), G(0, group11),
3841 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
3842 I(ImplicitOps | Stack | SrcImmU16, em_ret_far_imm),
3843 I(ImplicitOps | Stack, em_ret_far),
3844 D(ImplicitOps), DI(SrcImmByte, intn),
3845 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
3847 G(Src2One | ByteOp, group2), G(Src2One, group2),
3848 G(Src2CL | ByteOp, group2), G(Src2CL, group2),
3849 I(DstAcc | SrcImmUByte | No64, em_aam),
3850 I(DstAcc | SrcImmUByte | No64, em_aad),
3851 F(DstAcc | ByteOp | No64, em_salc),
3852 I(DstAcc | SrcXLat | ByteOp, em_mov),
3854 N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
3856 X3(I(SrcImmByte, em_loop)),
3857 I(SrcImmByte, em_jcxz),
3858 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
3859 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
3861 I(SrcImm | Stack, em_call), D(SrcImm | ImplicitOps),
3862 I(SrcImmFAddr | No64, em_jmp_far), D(SrcImmByte | ImplicitOps),
3863 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
3864 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
3866 N, DI(ImplicitOps, icebp), N, N,
3867 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
3868 G(ByteOp, group3), G(0, group3),
3870 D(ImplicitOps), D(ImplicitOps),
3871 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
3872 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
3875 static const struct opcode twobyte_table[256] = {
3877 G(0, group6), GD(0, &group7), N, N,
3878 N, I(ImplicitOps | EmulateOnUD, em_syscall),
3879 II(ImplicitOps | Priv, em_clts, clts), N,
3880 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3881 N, D(ImplicitOps | ModRM), N, N,
3883 N, N, N, N, N, N, N, N,
3884 D(ImplicitOps | ModRM), N, N, N, N, N, N, D(ImplicitOps | ModRM),
3886 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, cr_read, check_cr_read),
3887 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, dr_read, check_dr_read),
3888 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_cr_write, cr_write,
3890 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_dr_write, dr_write,
3893 GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_28_0f_29),
3894 GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_28_0f_29),
3895 N, GP(ModRM | DstMem | SrcReg | Sse | Mov | Aligned, &pfx_vmovntpx),
3898 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
3899 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
3900 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
3901 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
3902 I(ImplicitOps | EmulateOnUD, em_sysenter),
3903 I(ImplicitOps | Priv | EmulateOnUD, em_sysexit),
3905 N, N, N, N, N, N, N, N,
3907 X16(D(DstReg | SrcMem | ModRM)),
3909 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3914 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3919 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3923 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3925 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
3926 II(ImplicitOps, em_cpuid, cpuid),
3927 F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
3928 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
3929 F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
3931 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
3932 DI(ImplicitOps, rsm),
3933 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
3934 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
3935 F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
3936 D(ModRM), F(DstReg | SrcMem | ModRM, em_imul),
3938 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_cmpxchg),
3939 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
3940 F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
3941 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
3942 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
3943 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3947 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
3948 F(DstReg | SrcMem | ModRM, em_bsf), F(DstReg | SrcMem | ModRM, em_bsr),
3949 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3951 F2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd),
3952 N, D(DstMem | SrcReg | ModRM | Mov),
3953 N, N, N, GD(0, &group9),
3955 X8(I(DstReg, em_bswap)),
3957 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3959 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3961 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
3964 static const struct gprefix three_byte_0f_38_f0 = {
3965 I(DstReg | SrcMem | Mov, em_movbe), N, N, N
3968 static const struct gprefix three_byte_0f_38_f1 = {
3969 I(DstMem | SrcReg | Mov, em_movbe), N, N, N
3973 * Insns below are selected by the prefix which indexed by the third opcode
3976 static const struct opcode opcode_map_0f_38[256] = {
3978 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
3980 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
3982 GP(EmulateOnUD | ModRM | Prefix, &three_byte_0f_38_f0),
3983 GP(EmulateOnUD | ModRM | Prefix, &three_byte_0f_38_f1),
4002 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
4006 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4012 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
4013 unsigned size, bool sign_extension)
4015 int rc = X86EMUL_CONTINUE;
4019 op->addr.mem.ea = ctxt->_eip;
4020 /* NB. Immediates are sign-extended as necessary. */
4021 switch (op->bytes) {
4023 op->val = insn_fetch(s8, ctxt);
4026 op->val = insn_fetch(s16, ctxt);
4029 op->val = insn_fetch(s32, ctxt);
4032 op->val = insn_fetch(s64, ctxt);
4035 if (!sign_extension) {
4036 switch (op->bytes) {
4044 op->val &= 0xffffffff;
4052 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
4055 int rc = X86EMUL_CONTINUE;
4059 decode_register_operand(ctxt, op);
4062 rc = decode_imm(ctxt, op, 1, false);
4065 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4069 if (ctxt->d & BitOp)
4070 fetch_bit_operand(ctxt);
4071 op->orig_val = op->val;
4074 ctxt->memop.bytes = (ctxt->op_bytes == 8) ? 16 : 8;
4078 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4079 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4080 fetch_register_operand(op);
4081 op->orig_val = op->val;
4085 op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes;
4086 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4087 fetch_register_operand(op);
4088 op->orig_val = op->val;
4091 if (ctxt->d & ByteOp) {
4096 op->bytes = ctxt->op_bytes;
4097 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4098 fetch_register_operand(op);
4099 op->orig_val = op->val;
4103 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4105 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RDI));
4106 op->addr.mem.seg = VCPU_SREG_ES;
4113 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4114 fetch_register_operand(op);
4118 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
4121 rc = decode_imm(ctxt, op, 1, true);
4128 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
4131 rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
4134 ctxt->memop.bytes = 1;
4135 if (ctxt->memop.type == OP_REG) {
4136 ctxt->memop.addr.reg = decode_register(ctxt,
4137 ctxt->modrm_rm, true);
4138 fetch_register_operand(&ctxt->memop);
4142 ctxt->memop.bytes = 2;
4145 ctxt->memop.bytes = 4;
4148 rc = decode_imm(ctxt, op, 2, false);
4151 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
4155 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4157 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RSI));
4158 op->addr.mem.seg = ctxt->seg_override;
4164 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4166 register_address(ctxt,
4167 reg_read(ctxt, VCPU_REGS_RBX) +
4168 (reg_read(ctxt, VCPU_REGS_RAX) & 0xff));
4169 op->addr.mem.seg = ctxt->seg_override;
4174 op->addr.mem.ea = ctxt->_eip;
4175 op->bytes = ctxt->op_bytes + 2;
4176 insn_fetch_arr(op->valptr, op->bytes, ctxt);
4179 ctxt->memop.bytes = ctxt->op_bytes + 2;
4182 op->val = VCPU_SREG_ES;
4185 op->val = VCPU_SREG_CS;
4188 op->val = VCPU_SREG_SS;
4191 op->val = VCPU_SREG_DS;
4194 op->val = VCPU_SREG_FS;
4197 op->val = VCPU_SREG_GS;
4200 /* Special instructions do their own operand decoding. */
4202 op->type = OP_NONE; /* Disable writeback. */
4210 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
4212 int rc = X86EMUL_CONTINUE;
4213 int mode = ctxt->mode;
4214 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
4215 bool op_prefix = false;
4216 bool has_seg_override = false;
4217 struct opcode opcode;
4219 ctxt->memop.type = OP_NONE;
4220 ctxt->memopp = NULL;
4221 ctxt->_eip = ctxt->eip;
4222 ctxt->fetch.start = ctxt->_eip;
4223 ctxt->fetch.end = ctxt->fetch.start + insn_len;
4224 ctxt->opcode_len = 1;
4226 memcpy(ctxt->fetch.data, insn, insn_len);
4229 case X86EMUL_MODE_REAL:
4230 case X86EMUL_MODE_VM86:
4231 case X86EMUL_MODE_PROT16:
4232 def_op_bytes = def_ad_bytes = 2;
4234 case X86EMUL_MODE_PROT32:
4235 def_op_bytes = def_ad_bytes = 4;
4237 #ifdef CONFIG_X86_64
4238 case X86EMUL_MODE_PROT64:
4244 return EMULATION_FAILED;
4247 ctxt->op_bytes = def_op_bytes;
4248 ctxt->ad_bytes = def_ad_bytes;
4250 /* Legacy prefixes. */
4252 switch (ctxt->b = insn_fetch(u8, ctxt)) {
4253 case 0x66: /* operand-size override */
4255 /* switch between 2/4 bytes */
4256 ctxt->op_bytes = def_op_bytes ^ 6;
4258 case 0x67: /* address-size override */
4259 if (mode == X86EMUL_MODE_PROT64)
4260 /* switch between 4/8 bytes */
4261 ctxt->ad_bytes = def_ad_bytes ^ 12;
4263 /* switch between 2/4 bytes */
4264 ctxt->ad_bytes = def_ad_bytes ^ 6;
4266 case 0x26: /* ES override */
4267 case 0x2e: /* CS override */
4268 case 0x36: /* SS override */
4269 case 0x3e: /* DS override */
4270 has_seg_override = true;
4271 ctxt->seg_override = (ctxt->b >> 3) & 3;
4273 case 0x64: /* FS override */
4274 case 0x65: /* GS override */
4275 has_seg_override = true;
4276 ctxt->seg_override = ctxt->b & 7;
4278 case 0x40 ... 0x4f: /* REX */
4279 if (mode != X86EMUL_MODE_PROT64)
4281 ctxt->rex_prefix = ctxt->b;
4283 case 0xf0: /* LOCK */
4284 ctxt->lock_prefix = 1;
4286 case 0xf2: /* REPNE/REPNZ */
4287 case 0xf3: /* REP/REPE/REPZ */
4288 ctxt->rep_prefix = ctxt->b;
4294 /* Any legacy prefix after a REX prefix nullifies its effect. */
4296 ctxt->rex_prefix = 0;
4302 if (ctxt->rex_prefix & 8)
4303 ctxt->op_bytes = 8; /* REX.W */
4305 /* Opcode byte(s). */
4306 opcode = opcode_table[ctxt->b];
4307 /* Two-byte opcode? */
4308 if (ctxt->b == 0x0f) {
4309 ctxt->opcode_len = 2;
4310 ctxt->b = insn_fetch(u8, ctxt);
4311 opcode = twobyte_table[ctxt->b];
4313 /* 0F_38 opcode map */
4314 if (ctxt->b == 0x38) {
4315 ctxt->opcode_len = 3;
4316 ctxt->b = insn_fetch(u8, ctxt);
4317 opcode = opcode_map_0f_38[ctxt->b];
4320 ctxt->d = opcode.flags;
4322 if (ctxt->d & ModRM)
4323 ctxt->modrm = insn_fetch(u8, ctxt);
4325 /* vex-prefix instructions are not implemented */
4326 if (ctxt->opcode_len == 1 && (ctxt->b == 0xc5 || ctxt->b == 0xc4) &&
4327 (mode == X86EMUL_MODE_PROT64 ||
4328 (mode >= X86EMUL_MODE_PROT16 && (ctxt->modrm & 0x80)))) {
4332 while (ctxt->d & GroupMask) {
4333 switch (ctxt->d & GroupMask) {
4335 goffset = (ctxt->modrm >> 3) & 7;
4336 opcode = opcode.u.group[goffset];
4339 goffset = (ctxt->modrm >> 3) & 7;
4340 if ((ctxt->modrm >> 6) == 3)
4341 opcode = opcode.u.gdual->mod3[goffset];
4343 opcode = opcode.u.gdual->mod012[goffset];
4346 goffset = ctxt->modrm & 7;
4347 opcode = opcode.u.group[goffset];
4350 if (ctxt->rep_prefix && op_prefix)
4351 return EMULATION_FAILED;
4352 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
4353 switch (simd_prefix) {
4354 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
4355 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
4356 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
4357 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
4361 if (ctxt->modrm > 0xbf)
4362 opcode = opcode.u.esc->high[ctxt->modrm - 0xc0];
4364 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
4367 return EMULATION_FAILED;
4370 ctxt->d &= ~(u64)GroupMask;
4371 ctxt->d |= opcode.flags;
4376 return EMULATION_FAILED;
4378 ctxt->execute = opcode.u.execute;
4380 if (unlikely(ctxt->d &
4381 (NotImpl|EmulateOnUD|Stack|Op3264|Sse|Mmx|Intercept|CheckPerm))) {
4383 * These are copied unconditionally here, and checked unconditionally
4384 * in x86_emulate_insn.
4386 ctxt->check_perm = opcode.check_perm;
4387 ctxt->intercept = opcode.intercept;
4389 if (ctxt->d & NotImpl)
4390 return EMULATION_FAILED;
4392 if (!(ctxt->d & EmulateOnUD) && ctxt->ud)
4393 return EMULATION_FAILED;
4395 if (mode == X86EMUL_MODE_PROT64 && (ctxt->d & Stack))
4398 if (ctxt->d & Op3264) {
4399 if (mode == X86EMUL_MODE_PROT64)
4406 ctxt->op_bytes = 16;
4407 else if (ctxt->d & Mmx)
4411 /* ModRM and SIB bytes. */
4412 if (ctxt->d & ModRM) {
4413 rc = decode_modrm(ctxt, &ctxt->memop);
4414 if (!has_seg_override) {
4415 has_seg_override = true;
4416 ctxt->seg_override = ctxt->modrm_seg;
4418 } else if (ctxt->d & MemAbs)
4419 rc = decode_abs(ctxt, &ctxt->memop);
4420 if (rc != X86EMUL_CONTINUE)
4423 if (!has_seg_override)
4424 ctxt->seg_override = VCPU_SREG_DS;
4426 ctxt->memop.addr.mem.seg = ctxt->seg_override;
4428 if (ctxt->memop.type == OP_MEM && ctxt->ad_bytes != 8)
4429 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
4432 * Decode and fetch the source operand: register, memory
4435 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
4436 if (rc != X86EMUL_CONTINUE)
4440 * Decode and fetch the second source operand: register, memory
4443 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
4444 if (rc != X86EMUL_CONTINUE)
4447 /* Decode and fetch the destination operand: register or memory. */
4448 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
4451 if (ctxt->memopp && ctxt->memopp->type == OP_MEM && ctxt->rip_relative)
4452 ctxt->memopp->addr.mem.ea += ctxt->_eip;
4454 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
4457 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
4459 return ctxt->d & PageTable;
4462 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
4464 /* The second termination condition only applies for REPE
4465 * and REPNE. Test if the repeat string operation prefix is
4466 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
4467 * corresponding termination condition according to:
4468 * - if REPE/REPZ and ZF = 0 then done
4469 * - if REPNE/REPNZ and ZF = 1 then done
4471 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
4472 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
4473 && (((ctxt->rep_prefix == REPE_PREFIX) &&
4474 ((ctxt->eflags & EFLG_ZF) == 0))
4475 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
4476 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
4482 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
4486 ctxt->ops->get_fpu(ctxt);
4487 asm volatile("1: fwait \n\t"
4489 ".pushsection .fixup,\"ax\" \n\t"
4491 "movb $1, %[fault] \n\t"
4494 _ASM_EXTABLE(1b, 3b)
4495 : [fault]"+qm"(fault));
4496 ctxt->ops->put_fpu(ctxt);
4498 if (unlikely(fault))
4499 return emulate_exception(ctxt, MF_VECTOR, 0, false);
4501 return X86EMUL_CONTINUE;
4504 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
4507 if (op->type == OP_MM)
4508 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
4511 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))
4513 ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
4514 if (!(ctxt->d & ByteOp))
4515 fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
4516 asm("push %[flags]; popf; call *%[fastop]; pushf; pop %[flags]\n"
4517 : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),
4519 : "c"(ctxt->src2.val));
4520 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
4521 if (!fop) /* exception is returned in fop variable */
4522 return emulate_de(ctxt);
4523 return X86EMUL_CONTINUE;
4526 void init_decode_cache(struct x86_emulate_ctxt *ctxt)
4528 memset(&ctxt->rip_relative, 0,
4529 (void *)&ctxt->modrm - (void *)&ctxt->rip_relative);
4531 ctxt->io_read.pos = 0;
4532 ctxt->io_read.end = 0;
4533 ctxt->mem_read.end = 0;
4536 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
4538 const struct x86_emulate_ops *ops = ctxt->ops;
4539 int rc = X86EMUL_CONTINUE;
4540 int saved_dst_type = ctxt->dst.type;
4542 ctxt->mem_read.pos = 0;
4544 /* LOCK prefix is allowed only with some instructions */
4545 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
4546 rc = emulate_ud(ctxt);
4550 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
4551 rc = emulate_ud(ctxt);
4555 if (unlikely(ctxt->d &
4556 (No64|Undefined|Sse|Mmx|Intercept|CheckPerm|Priv|Prot|String))) {
4557 if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||
4558 (ctxt->d & Undefined)) {
4559 rc = emulate_ud(ctxt);
4563 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
4564 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
4565 rc = emulate_ud(ctxt);
4569 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
4570 rc = emulate_nm(ctxt);
4574 if (ctxt->d & Mmx) {
4575 rc = flush_pending_x87_faults(ctxt);
4576 if (rc != X86EMUL_CONTINUE)
4579 * Now that we know the fpu is exception safe, we can fetch
4582 fetch_possible_mmx_operand(ctxt, &ctxt->src);
4583 fetch_possible_mmx_operand(ctxt, &ctxt->src2);
4584 if (!(ctxt->d & Mov))
4585 fetch_possible_mmx_operand(ctxt, &ctxt->dst);
4588 if (unlikely(ctxt->guest_mode) && (ctxt->d & Intercept)) {
4589 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4590 X86_ICPT_PRE_EXCEPT);
4591 if (rc != X86EMUL_CONTINUE)
4595 /* Privileged instruction can be executed only in CPL=0 */
4596 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
4597 rc = emulate_gp(ctxt, 0);
4601 /* Instruction can only be executed in protected mode */
4602 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
4603 rc = emulate_ud(ctxt);
4607 /* Do instruction specific permission checks */
4608 if (ctxt->d & CheckPerm) {
4609 rc = ctxt->check_perm(ctxt);
4610 if (rc != X86EMUL_CONTINUE)
4614 if (unlikely(ctxt->guest_mode) && (ctxt->d & Intercept)) {
4615 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4616 X86_ICPT_POST_EXCEPT);
4617 if (rc != X86EMUL_CONTINUE)
4621 if (ctxt->rep_prefix && (ctxt->d & String)) {
4622 /* All REP prefixes have the same first termination condition */
4623 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
4624 ctxt->eip = ctxt->_eip;
4630 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
4631 rc = segmented_read(ctxt, ctxt->src.addr.mem,
4632 ctxt->src.valptr, ctxt->src.bytes);
4633 if (rc != X86EMUL_CONTINUE)
4635 ctxt->src.orig_val64 = ctxt->src.val64;
4638 if (ctxt->src2.type == OP_MEM) {
4639 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
4640 &ctxt->src2.val, ctxt->src2.bytes);
4641 if (rc != X86EMUL_CONTINUE)
4645 if ((ctxt->d & DstMask) == ImplicitOps)
4649 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
4650 /* optimisation - avoid slow emulated read if Mov */
4651 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
4652 &ctxt->dst.val, ctxt->dst.bytes);
4653 if (rc != X86EMUL_CONTINUE)
4656 ctxt->dst.orig_val = ctxt->dst.val;
4660 if (unlikely(ctxt->guest_mode) && (ctxt->d & Intercept)) {
4661 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4662 X86_ICPT_POST_MEMACCESS);
4663 if (rc != X86EMUL_CONTINUE)
4667 if (ctxt->execute) {
4668 if (ctxt->d & Fastop) {
4669 void (*fop)(struct fastop *) = (void *)ctxt->execute;
4670 rc = fastop(ctxt, fop);
4671 if (rc != X86EMUL_CONTINUE)
4675 rc = ctxt->execute(ctxt);
4676 if (rc != X86EMUL_CONTINUE)
4681 if (ctxt->opcode_len == 2)
4683 else if (ctxt->opcode_len == 3)
4684 goto threebyte_insn;
4687 case 0x63: /* movsxd */
4688 if (ctxt->mode != X86EMUL_MODE_PROT64)
4689 goto cannot_emulate;
4690 ctxt->dst.val = (s32) ctxt->src.val;
4692 case 0x70 ... 0x7f: /* jcc (short) */
4693 if (test_cc(ctxt->b, ctxt->eflags))
4694 jmp_rel(ctxt, ctxt->src.val);
4696 case 0x8d: /* lea r16/r32, m */
4697 ctxt->dst.val = ctxt->src.addr.mem.ea;
4699 case 0x90 ... 0x97: /* nop / xchg reg, rax */
4700 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
4701 ctxt->dst.type = OP_NONE;
4705 case 0x98: /* cbw/cwde/cdqe */
4706 switch (ctxt->op_bytes) {
4707 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
4708 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
4709 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
4712 case 0xcc: /* int3 */
4713 rc = emulate_int(ctxt, 3);
4715 case 0xcd: /* int n */
4716 rc = emulate_int(ctxt, ctxt->src.val);
4718 case 0xce: /* into */
4719 if (ctxt->eflags & EFLG_OF)
4720 rc = emulate_int(ctxt, 4);
4722 case 0xe9: /* jmp rel */
4723 case 0xeb: /* jmp rel short */
4724 jmp_rel(ctxt, ctxt->src.val);
4725 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4727 case 0xf4: /* hlt */
4728 ctxt->ops->halt(ctxt);
4730 case 0xf5: /* cmc */
4731 /* complement carry flag from eflags reg */
4732 ctxt->eflags ^= EFLG_CF;
4734 case 0xf8: /* clc */
4735 ctxt->eflags &= ~EFLG_CF;
4737 case 0xf9: /* stc */
4738 ctxt->eflags |= EFLG_CF;
4740 case 0xfc: /* cld */
4741 ctxt->eflags &= ~EFLG_DF;
4743 case 0xfd: /* std */
4744 ctxt->eflags |= EFLG_DF;
4747 goto cannot_emulate;
4750 if (rc != X86EMUL_CONTINUE)
4754 if (ctxt->d & SrcWrite) {
4755 BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR);
4756 rc = writeback(ctxt, &ctxt->src);
4757 if (rc != X86EMUL_CONTINUE)
4760 if (!(ctxt->d & NoWrite)) {
4761 rc = writeback(ctxt, &ctxt->dst);
4762 if (rc != X86EMUL_CONTINUE)
4767 * restore dst type in case the decoding will be reused
4768 * (happens for string instruction )
4770 ctxt->dst.type = saved_dst_type;
4772 if ((ctxt->d & SrcMask) == SrcSI)
4773 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
4775 if ((ctxt->d & DstMask) == DstDI)
4776 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
4778 if (ctxt->rep_prefix && (ctxt->d & String)) {
4780 struct read_cache *r = &ctxt->io_read;
4781 if ((ctxt->d & SrcMask) == SrcSI)
4782 count = ctxt->src.count;
4784 count = ctxt->dst.count;
4785 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX),
4788 if (!string_insn_completed(ctxt)) {
4790 * Re-enter guest when pio read ahead buffer is empty
4791 * or, if it is not used, after each 1024 iteration.
4793 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
4794 (r->end == 0 || r->end != r->pos)) {
4796 * Reset read cache. Usually happens before
4797 * decode, but since instruction is restarted
4798 * we have to do it here.
4800 ctxt->mem_read.end = 0;
4801 writeback_registers(ctxt);
4802 return EMULATION_RESTART;
4804 goto done; /* skip rip writeback */
4808 ctxt->eip = ctxt->_eip;
4811 if (rc == X86EMUL_PROPAGATE_FAULT)
4812 ctxt->have_exception = true;
4813 if (rc == X86EMUL_INTERCEPTED)
4814 return EMULATION_INTERCEPTED;
4816 if (rc == X86EMUL_CONTINUE)
4817 writeback_registers(ctxt);
4819 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4823 case 0x09: /* wbinvd */
4824 (ctxt->ops->wbinvd)(ctxt);
4826 case 0x08: /* invd */
4827 case 0x0d: /* GrpP (prefetch) */
4828 case 0x18: /* Grp16 (prefetch/nop) */
4829 case 0x1f: /* nop */
4831 case 0x20: /* mov cr, reg */
4832 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
4834 case 0x21: /* mov from dr to reg */
4835 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
4837 case 0x40 ... 0x4f: /* cmov */
4838 if (test_cc(ctxt->b, ctxt->eflags))
4839 ctxt->dst.val = ctxt->src.val;
4840 else if (ctxt->mode != X86EMUL_MODE_PROT64 ||
4841 ctxt->op_bytes != 4)
4842 ctxt->dst.type = OP_NONE; /* no writeback */
4844 case 0x80 ... 0x8f: /* jnz rel, etc*/
4845 if (test_cc(ctxt->b, ctxt->eflags))
4846 jmp_rel(ctxt, ctxt->src.val);
4848 case 0x90 ... 0x9f: /* setcc r/m8 */
4849 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
4851 case 0xae: /* clflush */
4853 case 0xb6 ... 0xb7: /* movzx */
4854 ctxt->dst.bytes = ctxt->op_bytes;
4855 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
4856 : (u16) ctxt->src.val;
4858 case 0xbe ... 0xbf: /* movsx */
4859 ctxt->dst.bytes = ctxt->op_bytes;
4860 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
4861 (s16) ctxt->src.val;
4863 case 0xc3: /* movnti */
4864 ctxt->dst.bytes = ctxt->op_bytes;
4865 ctxt->dst.val = (ctxt->op_bytes == 8) ? (u64) ctxt->src.val :
4866 (u32) ctxt->src.val;
4869 goto cannot_emulate;
4874 if (rc != X86EMUL_CONTINUE)
4880 return EMULATION_FAILED;
4883 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
4885 invalidate_registers(ctxt);
4888 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
4890 writeback_registers(ctxt);