1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
27 #include <linux/stringify.h>
36 #define OpImplicit 1ull /* No generic decode */
37 #define OpReg 2ull /* Register */
38 #define OpMem 3ull /* Memory */
39 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
40 #define OpDI 5ull /* ES:DI/EDI/RDI */
41 #define OpMem64 6ull /* Memory, 64-bit */
42 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
43 #define OpDX 8ull /* DX register */
44 #define OpCL 9ull /* CL register (for shifts) */
45 #define OpImmByte 10ull /* 8-bit sign extended immediate */
46 #define OpOne 11ull /* Implied 1 */
47 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
48 #define OpMem16 13ull /* Memory operand (16-bit). */
49 #define OpMem32 14ull /* Memory operand (32-bit). */
50 #define OpImmU 15ull /* Immediate operand, zero extended */
51 #define OpSI 16ull /* SI/ESI/RSI */
52 #define OpImmFAddr 17ull /* Immediate far address */
53 #define OpMemFAddr 18ull /* Far address in memory */
54 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
55 #define OpES 20ull /* ES */
56 #define OpCS 21ull /* CS */
57 #define OpSS 22ull /* SS */
58 #define OpDS 23ull /* DS */
59 #define OpFS 24ull /* FS */
60 #define OpGS 25ull /* GS */
61 #define OpMem8 26ull /* 8-bit zero extended memory operand */
62 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
63 #define OpXLat 28ull /* memory at BX/EBX/RBX + zero-extended AL */
64 #define OpAccLo 29ull /* Low part of extended acc (AX/AX/EAX/RAX) */
65 #define OpAccHi 30ull /* High part of extended acc (-/DX/EDX/RDX) */
67 #define OpBits 5 /* Width of operand field */
68 #define OpMask ((1ull << OpBits) - 1)
71 * Opcode effective-address decode tables.
72 * Note that we only emulate instructions that have at least one memory
73 * operand (excluding implicit stack references). We assume that stack
74 * references and instruction fetches will never occur in special memory
75 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
79 /* Operand sizes: 8-bit operands or specified/overridden size. */
80 #define ByteOp (1<<0) /* 8-bit operands. */
81 /* Destination operand type. */
83 #define ImplicitOps (OpImplicit << DstShift)
84 #define DstReg (OpReg << DstShift)
85 #define DstMem (OpMem << DstShift)
86 #define DstAcc (OpAcc << DstShift)
87 #define DstDI (OpDI << DstShift)
88 #define DstMem64 (OpMem64 << DstShift)
89 #define DstImmUByte (OpImmUByte << DstShift)
90 #define DstDX (OpDX << DstShift)
91 #define DstAccLo (OpAccLo << DstShift)
92 #define DstMask (OpMask << DstShift)
93 /* Source operand type. */
95 #define SrcNone (OpNone << SrcShift)
96 #define SrcReg (OpReg << SrcShift)
97 #define SrcMem (OpMem << SrcShift)
98 #define SrcMem16 (OpMem16 << SrcShift)
99 #define SrcMem32 (OpMem32 << SrcShift)
100 #define SrcImm (OpImm << SrcShift)
101 #define SrcImmByte (OpImmByte << SrcShift)
102 #define SrcOne (OpOne << SrcShift)
103 #define SrcImmUByte (OpImmUByte << SrcShift)
104 #define SrcImmU (OpImmU << SrcShift)
105 #define SrcSI (OpSI << SrcShift)
106 #define SrcXLat (OpXLat << SrcShift)
107 #define SrcImmFAddr (OpImmFAddr << SrcShift)
108 #define SrcMemFAddr (OpMemFAddr << SrcShift)
109 #define SrcAcc (OpAcc << SrcShift)
110 #define SrcImmU16 (OpImmU16 << SrcShift)
111 #define SrcImm64 (OpImm64 << SrcShift)
112 #define SrcDX (OpDX << SrcShift)
113 #define SrcMem8 (OpMem8 << SrcShift)
114 #define SrcAccHi (OpAccHi << SrcShift)
115 #define SrcMask (OpMask << SrcShift)
116 #define BitOp (1<<11)
117 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
118 #define String (1<<13) /* String instruction (rep capable) */
119 #define Stack (1<<14) /* Stack instruction (push/pop) */
120 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
121 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
122 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
123 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
124 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
125 #define Escape (5<<15) /* Escape to coprocessor instruction */
126 #define Sse (1<<18) /* SSE Vector instruction */
127 /* Generic ModRM decode. */
128 #define ModRM (1<<19)
129 /* Destination is only written; never read. */
132 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
133 #define EmulateOnUD (1<<22) /* Emulate if unsupported by the host */
134 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
135 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
136 #define Undefined (1<<25) /* No Such Instruction */
137 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
138 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
140 #define PageTable (1 << 29) /* instruction used to write page table */
141 #define NotImpl (1 << 30) /* instruction is not implemented */
142 /* Source 2 operand type */
143 #define Src2Shift (31)
144 #define Src2None (OpNone << Src2Shift)
145 #define Src2Mem (OpMem << Src2Shift)
146 #define Src2CL (OpCL << Src2Shift)
147 #define Src2ImmByte (OpImmByte << Src2Shift)
148 #define Src2One (OpOne << Src2Shift)
149 #define Src2Imm (OpImm << Src2Shift)
150 #define Src2ES (OpES << Src2Shift)
151 #define Src2CS (OpCS << Src2Shift)
152 #define Src2SS (OpSS << Src2Shift)
153 #define Src2DS (OpDS << Src2Shift)
154 #define Src2FS (OpFS << Src2Shift)
155 #define Src2GS (OpGS << Src2Shift)
156 #define Src2Mask (OpMask << Src2Shift)
157 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
158 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
159 #define Unaligned ((u64)1 << 42) /* Explicitly unaligned (e.g. MOVDQU) */
160 #define Avx ((u64)1 << 43) /* Advanced Vector Extensions */
161 #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
162 #define NoWrite ((u64)1 << 45) /* No writeback */
163 #define SrcWrite ((u64)1 << 46) /* Write back src operand */
164 #define NoMod ((u64)1 << 47) /* Mod field is ignored */
166 #define DstXacc (DstAccLo | SrcAccHi | SrcWrite)
168 #define X2(x...) x, x
169 #define X3(x...) X2(x), x
170 #define X4(x...) X2(x), X2(x)
171 #define X5(x...) X4(x), x
172 #define X6(x...) X4(x), X2(x)
173 #define X7(x...) X4(x), X3(x)
174 #define X8(x...) X4(x), X4(x)
175 #define X16(x...) X8(x), X8(x)
177 #define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
178 #define FASTOP_SIZE 8
181 * fastop functions have a special calling convention:
186 * flags: rflags (in/out)
187 * ex: rsi (in:fastop pointer, out:zero if exception)
189 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
190 * different operand sizes can be reached by calculation, rather than a jump
191 * table (which would be bigger than the code).
193 * fastop functions are declared as taking a never-defined fastop parameter,
194 * so they can't be called from C directly.
203 int (*execute)(struct x86_emulate_ctxt *ctxt);
204 const struct opcode *group;
205 const struct group_dual *gdual;
206 const struct gprefix *gprefix;
207 const struct escape *esc;
208 void (*fastop)(struct fastop *fake);
210 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
214 struct opcode mod012[8];
215 struct opcode mod3[8];
219 struct opcode pfx_no;
220 struct opcode pfx_66;
221 struct opcode pfx_f2;
222 struct opcode pfx_f3;
227 struct opcode high[64];
230 /* EFLAGS bit definitions. */
231 #define EFLG_ID (1<<21)
232 #define EFLG_VIP (1<<20)
233 #define EFLG_VIF (1<<19)
234 #define EFLG_AC (1<<18)
235 #define EFLG_VM (1<<17)
236 #define EFLG_RF (1<<16)
237 #define EFLG_IOPL (3<<12)
238 #define EFLG_NT (1<<14)
239 #define EFLG_OF (1<<11)
240 #define EFLG_DF (1<<10)
241 #define EFLG_IF (1<<9)
242 #define EFLG_TF (1<<8)
243 #define EFLG_SF (1<<7)
244 #define EFLG_ZF (1<<6)
245 #define EFLG_AF (1<<4)
246 #define EFLG_PF (1<<2)
247 #define EFLG_CF (1<<0)
249 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
250 #define EFLG_RESERVED_ONE_MASK 2
252 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
254 if (!(ctxt->regs_valid & (1 << nr))) {
255 ctxt->regs_valid |= 1 << nr;
256 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
258 return ctxt->_regs[nr];
261 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
263 ctxt->regs_valid |= 1 << nr;
264 ctxt->regs_dirty |= 1 << nr;
265 return &ctxt->_regs[nr];
268 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
271 return reg_write(ctxt, nr);
274 static void writeback_registers(struct x86_emulate_ctxt *ctxt)
278 for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
279 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
282 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
284 ctxt->regs_dirty = 0;
285 ctxt->regs_valid = 0;
289 * These EFLAGS bits are restored from saved value during emulation, and
290 * any changes are written back to the saved value after emulation.
292 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
300 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));
302 #define FOP_ALIGN ".align " __stringify(FASTOP_SIZE) " \n\t"
303 #define FOP_RET "ret \n\t"
305 #define FOP_START(op) \
306 extern void em_##op(struct fastop *fake); \
307 asm(".pushsection .text, \"ax\" \n\t" \
308 ".global em_" #op " \n\t" \
315 #define FOPNOP() FOP_ALIGN FOP_RET
317 #define FOP1E(op, dst) \
318 FOP_ALIGN "10: " #op " %" #dst " \n\t" FOP_RET
320 #define FOP1EEX(op, dst) \
321 FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception)
323 #define FASTOP1(op) \
328 ON64(FOP1E(op##q, rax)) \
331 /* 1-operand, using src2 (for MUL/DIV r/m) */
332 #define FASTOP1SRC2(op, name) \
337 ON64(FOP1E(op, rcx)) \
340 /* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
341 #define FASTOP1SRC2EX(op, name) \
346 ON64(FOP1EEX(op, rcx)) \
349 #define FOP2E(op, dst, src) \
350 FOP_ALIGN #op " %" #src ", %" #dst " \n\t" FOP_RET
352 #define FASTOP2(op) \
354 FOP2E(op##b, al, dl) \
355 FOP2E(op##w, ax, dx) \
356 FOP2E(op##l, eax, edx) \
357 ON64(FOP2E(op##q, rax, rdx)) \
360 /* 2 operand, word only */
361 #define FASTOP2W(op) \
364 FOP2E(op##w, ax, dx) \
365 FOP2E(op##l, eax, edx) \
366 ON64(FOP2E(op##q, rax, rdx)) \
369 /* 2 operand, src is CL */
370 #define FASTOP2CL(op) \
372 FOP2E(op##b, al, cl) \
373 FOP2E(op##w, ax, cl) \
374 FOP2E(op##l, eax, cl) \
375 ON64(FOP2E(op##q, rax, cl)) \
378 #define FOP3E(op, dst, src, src2) \
379 FOP_ALIGN #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET
381 /* 3-operand, word-only, src2=cl */
382 #define FASTOP3WCL(op) \
385 FOP3E(op##w, ax, dx, cl) \
386 FOP3E(op##l, eax, edx, cl) \
387 ON64(FOP3E(op##q, rax, rdx, cl)) \
390 /* Special case for SETcc - 1 instruction per cc */
391 #define FOP_SETCC(op) ".align 4; " #op " %al; ret \n\t"
393 asm(".global kvm_fastop_exception \n"
394 "kvm_fastop_exception: xor %esi, %esi; ret");
415 FOP_START(salc) "pushf; sbb %al, %al; popf \n\t" FOP_RET
418 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
419 enum x86_intercept intercept,
420 enum x86_intercept_stage stage)
422 struct x86_instruction_info info = {
423 .intercept = intercept,
424 .rep_prefix = ctxt->rep_prefix,
425 .modrm_mod = ctxt->modrm_mod,
426 .modrm_reg = ctxt->modrm_reg,
427 .modrm_rm = ctxt->modrm_rm,
428 .src_val = ctxt->src.val64,
429 .src_bytes = ctxt->src.bytes,
430 .dst_bytes = ctxt->dst.bytes,
431 .ad_bytes = ctxt->ad_bytes,
432 .next_rip = ctxt->eip,
435 return ctxt->ops->intercept(ctxt, &info, stage);
438 static void assign_masked(ulong *dest, ulong src, ulong mask)
440 *dest = (*dest & ~mask) | (src & mask);
443 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
445 return (1UL << (ctxt->ad_bytes << 3)) - 1;
448 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
451 struct desc_struct ss;
453 if (ctxt->mode == X86EMUL_MODE_PROT64)
455 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
456 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
459 static int stack_size(struct x86_emulate_ctxt *ctxt)
461 return (__fls(stack_mask(ctxt)) + 1) >> 3;
464 /* Access/update address held in a register, based on addressing mode. */
465 static inline unsigned long
466 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
468 if (ctxt->ad_bytes == sizeof(unsigned long))
471 return reg & ad_mask(ctxt);
474 static inline unsigned long
475 register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
477 return address_mask(ctxt, reg);
480 static void masked_increment(ulong *reg, ulong mask, int inc)
482 assign_masked(reg, *reg + inc, mask);
486 register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
490 if (ctxt->ad_bytes == sizeof(unsigned long))
493 mask = ad_mask(ctxt);
494 masked_increment(reg, mask, inc);
497 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
499 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
502 static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
504 register_address_increment(ctxt, &ctxt->_eip, rel);
507 static u32 desc_limit_scaled(struct desc_struct *desc)
509 u32 limit = get_desc_limit(desc);
511 return desc->g ? (limit << 12) | 0xfff : limit;
514 static void set_seg_override(struct x86_emulate_ctxt *ctxt, int seg)
516 ctxt->has_seg_override = true;
517 ctxt->seg_override = seg;
520 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
522 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
525 return ctxt->ops->get_cached_segment_base(ctxt, seg);
528 static unsigned seg_override(struct x86_emulate_ctxt *ctxt)
530 if (!ctxt->has_seg_override)
533 return ctxt->seg_override;
536 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
537 u32 error, bool valid)
539 ctxt->exception.vector = vec;
540 ctxt->exception.error_code = error;
541 ctxt->exception.error_code_valid = valid;
542 return X86EMUL_PROPAGATE_FAULT;
545 static int emulate_db(struct x86_emulate_ctxt *ctxt)
547 return emulate_exception(ctxt, DB_VECTOR, 0, false);
550 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
552 return emulate_exception(ctxt, GP_VECTOR, err, true);
555 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
557 return emulate_exception(ctxt, SS_VECTOR, err, true);
560 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
562 return emulate_exception(ctxt, UD_VECTOR, 0, false);
565 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
567 return emulate_exception(ctxt, TS_VECTOR, err, true);
570 static int emulate_de(struct x86_emulate_ctxt *ctxt)
572 return emulate_exception(ctxt, DE_VECTOR, 0, false);
575 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
577 return emulate_exception(ctxt, NM_VECTOR, 0, false);
580 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
583 struct desc_struct desc;
585 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
589 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
594 struct desc_struct desc;
596 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
597 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
601 * x86 defines three classes of vector instructions: explicitly
602 * aligned, explicitly unaligned, and the rest, which change behaviour
603 * depending on whether they're AVX encoded or not.
605 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
606 * subject to the same check.
608 static bool insn_aligned(struct x86_emulate_ctxt *ctxt, unsigned size)
610 if (likely(size < 16))
613 if (ctxt->d & Aligned)
615 else if (ctxt->d & Unaligned)
617 else if (ctxt->d & Avx)
623 static int __linearize(struct x86_emulate_ctxt *ctxt,
624 struct segmented_address addr,
625 unsigned size, bool write, bool fetch,
628 struct desc_struct desc;
635 la = seg_base(ctxt, addr.seg) + addr.ea;
636 switch (ctxt->mode) {
637 case X86EMUL_MODE_PROT64:
638 if (((signed long)la << 16) >> 16 != la)
639 return emulate_gp(ctxt, 0);
642 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
646 /* code segment in protected mode or read-only data segment */
647 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
648 || !(desc.type & 2)) && write)
650 /* unreadable code segment */
651 if (!fetch && (desc.type & 8) && !(desc.type & 2))
653 lim = desc_limit_scaled(&desc);
654 if ((desc.type & 8) || !(desc.type & 4)) {
655 /* expand-up segment */
656 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
659 /* expand-down segment */
660 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
662 lim = desc.d ? 0xffffffff : 0xffff;
663 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
666 cpl = ctxt->ops->cpl(ctxt);
667 if (!(desc.type & 8)) {
671 } else if ((desc.type & 8) && !(desc.type & 4)) {
672 /* nonconforming code segment */
675 } else if ((desc.type & 8) && (desc.type & 4)) {
676 /* conforming code segment */
682 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
684 if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0))
685 return emulate_gp(ctxt, 0);
687 return X86EMUL_CONTINUE;
689 if (addr.seg == VCPU_SREG_SS)
690 return emulate_ss(ctxt, sel);
692 return emulate_gp(ctxt, sel);
695 static int linearize(struct x86_emulate_ctxt *ctxt,
696 struct segmented_address addr,
697 unsigned size, bool write,
700 return __linearize(ctxt, addr, size, write, false, linear);
704 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
705 struct segmented_address addr,
712 rc = linearize(ctxt, addr, size, false, &linear);
713 if (rc != X86EMUL_CONTINUE)
715 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
719 * Fetch the next byte of the instruction being emulated which is pointed to
720 * by ctxt->_eip, then increment ctxt->_eip.
722 * Also prefetch the remaining bytes of the instruction without crossing page
723 * boundary if they are not in fetch_cache yet.
725 static int do_insn_fetch_byte(struct x86_emulate_ctxt *ctxt, u8 *dest)
727 struct fetch_cache *fc = &ctxt->fetch;
731 if (ctxt->_eip == fc->end) {
732 unsigned long linear;
733 struct segmented_address addr = { .seg = VCPU_SREG_CS,
735 cur_size = fc->end - fc->start;
736 size = min(15UL - cur_size,
737 PAGE_SIZE - offset_in_page(ctxt->_eip));
738 rc = __linearize(ctxt, addr, size, false, true, &linear);
739 if (unlikely(rc != X86EMUL_CONTINUE))
741 rc = ctxt->ops->fetch(ctxt, linear, fc->data + cur_size,
742 size, &ctxt->exception);
743 if (unlikely(rc != X86EMUL_CONTINUE))
747 *dest = fc->data[ctxt->_eip - fc->start];
749 return X86EMUL_CONTINUE;
752 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
753 void *dest, unsigned size)
757 /* x86 instructions are limited to 15 bytes. */
758 if (unlikely(ctxt->_eip + size - ctxt->eip > 15))
759 return X86EMUL_UNHANDLEABLE;
761 rc = do_insn_fetch_byte(ctxt, dest++);
762 if (rc != X86EMUL_CONTINUE)
765 return X86EMUL_CONTINUE;
768 /* Fetch next part of the instruction being emulated. */
769 #define insn_fetch(_type, _ctxt) \
770 ({ unsigned long _x; \
771 rc = do_insn_fetch(_ctxt, &_x, sizeof(_type)); \
772 if (rc != X86EMUL_CONTINUE) \
777 #define insn_fetch_arr(_arr, _size, _ctxt) \
778 ({ rc = do_insn_fetch(_ctxt, _arr, (_size)); \
779 if (rc != X86EMUL_CONTINUE) \
784 * Given the 'reg' portion of a ModRM byte, and a register block, return a
785 * pointer into the block that addresses the relevant register.
786 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
788 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
792 int highbyte_regs = (ctxt->rex_prefix == 0) && byteop;
794 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
795 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
797 p = reg_rmw(ctxt, modrm_reg);
801 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
802 struct segmented_address addr,
803 u16 *size, unsigned long *address, int op_bytes)
810 rc = segmented_read_std(ctxt, addr, size, 2);
811 if (rc != X86EMUL_CONTINUE)
814 rc = segmented_read_std(ctxt, addr, address, op_bytes);
828 FASTOP1SRC2(mul, mul_ex);
829 FASTOP1SRC2(imul, imul_ex);
830 FASTOP1SRC2EX(div, div_ex);
831 FASTOP1SRC2EX(idiv, idiv_ex);
860 static u8 test_cc(unsigned int condition, unsigned long flags)
863 void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
865 flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
866 asm("push %[flags]; popf; call *%[fastop]"
867 : "=a"(rc) : [fastop]"r"(fop), [flags]"r"(flags));
871 static void fetch_register_operand(struct operand *op)
875 op->val = *(u8 *)op->addr.reg;
878 op->val = *(u16 *)op->addr.reg;
881 op->val = *(u32 *)op->addr.reg;
884 op->val = *(u64 *)op->addr.reg;
889 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
891 ctxt->ops->get_fpu(ctxt);
893 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
894 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
895 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
896 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
897 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
898 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
899 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
900 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
902 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
903 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
904 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
905 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
906 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
907 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
908 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
909 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
913 ctxt->ops->put_fpu(ctxt);
916 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
919 ctxt->ops->get_fpu(ctxt);
921 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
922 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
923 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
924 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
925 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
926 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
927 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
928 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
930 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
931 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
932 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
933 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
934 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
935 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
936 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
937 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
941 ctxt->ops->put_fpu(ctxt);
944 static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
946 ctxt->ops->get_fpu(ctxt);
948 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
949 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
950 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
951 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
952 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
953 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
954 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
955 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
958 ctxt->ops->put_fpu(ctxt);
961 static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
963 ctxt->ops->get_fpu(ctxt);
965 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
966 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
967 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
968 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
969 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
970 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
971 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
972 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
975 ctxt->ops->put_fpu(ctxt);
978 static int em_fninit(struct x86_emulate_ctxt *ctxt)
980 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
981 return emulate_nm(ctxt);
983 ctxt->ops->get_fpu(ctxt);
984 asm volatile("fninit");
985 ctxt->ops->put_fpu(ctxt);
986 return X86EMUL_CONTINUE;
989 static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
993 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
994 return emulate_nm(ctxt);
996 ctxt->ops->get_fpu(ctxt);
997 asm volatile("fnstcw %0": "+m"(fcw));
998 ctxt->ops->put_fpu(ctxt);
1000 /* force 2 byte destination */
1001 ctxt->dst.bytes = 2;
1002 ctxt->dst.val = fcw;
1004 return X86EMUL_CONTINUE;
1007 static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
1011 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1012 return emulate_nm(ctxt);
1014 ctxt->ops->get_fpu(ctxt);
1015 asm volatile("fnstsw %0": "+m"(fsw));
1016 ctxt->ops->put_fpu(ctxt);
1018 /* force 2 byte destination */
1019 ctxt->dst.bytes = 2;
1020 ctxt->dst.val = fsw;
1022 return X86EMUL_CONTINUE;
1025 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1028 unsigned reg = ctxt->modrm_reg;
1030 if (!(ctxt->d & ModRM))
1031 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1033 if (ctxt->d & Sse) {
1037 read_sse_reg(ctxt, &op->vec_val, reg);
1040 if (ctxt->d & Mmx) {
1049 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1050 op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp);
1052 fetch_register_operand(op);
1053 op->orig_val = op->val;
1056 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1058 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1059 ctxt->modrm_seg = VCPU_SREG_SS;
1062 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1066 int index_reg = 0, base_reg = 0, scale;
1067 int rc = X86EMUL_CONTINUE;
1070 if (ctxt->rex_prefix) {
1071 ctxt->modrm_reg = (ctxt->rex_prefix & 4) << 1; /* REX.R */
1072 index_reg = (ctxt->rex_prefix & 2) << 2; /* REX.X */
1073 ctxt->modrm_rm = base_reg = (ctxt->rex_prefix & 1) << 3; /* REG.B */
1076 ctxt->modrm_mod |= (ctxt->modrm & 0xc0) >> 6;
1077 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1078 ctxt->modrm_rm |= (ctxt->modrm & 0x07);
1079 ctxt->modrm_seg = VCPU_SREG_DS;
1081 if (ctxt->modrm_mod == 3 || (ctxt->d & NoMod)) {
1083 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1084 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,
1086 if (ctxt->d & Sse) {
1089 op->addr.xmm = ctxt->modrm_rm;
1090 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
1093 if (ctxt->d & Mmx) {
1096 op->addr.xmm = ctxt->modrm_rm & 7;
1099 fetch_register_operand(op);
1105 if (ctxt->ad_bytes == 2) {
1106 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1107 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1108 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1109 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1111 /* 16-bit ModR/M decode. */
1112 switch (ctxt->modrm_mod) {
1114 if (ctxt->modrm_rm == 6)
1115 modrm_ea += insn_fetch(u16, ctxt);
1118 modrm_ea += insn_fetch(s8, ctxt);
1121 modrm_ea += insn_fetch(u16, ctxt);
1124 switch (ctxt->modrm_rm) {
1126 modrm_ea += bx + si;
1129 modrm_ea += bx + di;
1132 modrm_ea += bp + si;
1135 modrm_ea += bp + di;
1144 if (ctxt->modrm_mod != 0)
1151 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1152 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1153 ctxt->modrm_seg = VCPU_SREG_SS;
1154 modrm_ea = (u16)modrm_ea;
1156 /* 32/64-bit ModR/M decode. */
1157 if ((ctxt->modrm_rm & 7) == 4) {
1158 sib = insn_fetch(u8, ctxt);
1159 index_reg |= (sib >> 3) & 7;
1160 base_reg |= sib & 7;
1163 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1164 modrm_ea += insn_fetch(s32, ctxt);
1166 modrm_ea += reg_read(ctxt, base_reg);
1167 adjust_modrm_seg(ctxt, base_reg);
1170 modrm_ea += reg_read(ctxt, index_reg) << scale;
1171 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1172 if (ctxt->mode == X86EMUL_MODE_PROT64)
1173 ctxt->rip_relative = 1;
1175 base_reg = ctxt->modrm_rm;
1176 modrm_ea += reg_read(ctxt, base_reg);
1177 adjust_modrm_seg(ctxt, base_reg);
1179 switch (ctxt->modrm_mod) {
1181 if (ctxt->modrm_rm == 5)
1182 modrm_ea += insn_fetch(s32, ctxt);
1185 modrm_ea += insn_fetch(s8, ctxt);
1188 modrm_ea += insn_fetch(s32, ctxt);
1192 op->addr.mem.ea = modrm_ea;
1197 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1200 int rc = X86EMUL_CONTINUE;
1203 switch (ctxt->ad_bytes) {
1205 op->addr.mem.ea = insn_fetch(u16, ctxt);
1208 op->addr.mem.ea = insn_fetch(u32, ctxt);
1211 op->addr.mem.ea = insn_fetch(u64, ctxt);
1218 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1222 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1223 mask = ~(ctxt->dst.bytes * 8 - 1);
1225 if (ctxt->src.bytes == 2)
1226 sv = (s16)ctxt->src.val & (s16)mask;
1227 else if (ctxt->src.bytes == 4)
1228 sv = (s32)ctxt->src.val & (s32)mask;
1230 ctxt->dst.addr.mem.ea += (sv >> 3);
1233 /* only subword offset */
1234 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1237 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1238 unsigned long addr, void *dest, unsigned size)
1241 struct read_cache *mc = &ctxt->mem_read;
1243 if (mc->pos < mc->end)
1246 WARN_ON((mc->end + size) >= sizeof(mc->data));
1248 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1250 if (rc != X86EMUL_CONTINUE)
1256 memcpy(dest, mc->data + mc->pos, size);
1258 return X86EMUL_CONTINUE;
1261 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1262 struct segmented_address addr,
1269 rc = linearize(ctxt, addr, size, false, &linear);
1270 if (rc != X86EMUL_CONTINUE)
1272 return read_emulated(ctxt, linear, data, size);
1275 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1276 struct segmented_address addr,
1283 rc = linearize(ctxt, addr, size, true, &linear);
1284 if (rc != X86EMUL_CONTINUE)
1286 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1290 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1291 struct segmented_address addr,
1292 const void *orig_data, const void *data,
1298 rc = linearize(ctxt, addr, size, true, &linear);
1299 if (rc != X86EMUL_CONTINUE)
1301 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1302 size, &ctxt->exception);
1305 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1306 unsigned int size, unsigned short port,
1309 struct read_cache *rc = &ctxt->io_read;
1311 if (rc->pos == rc->end) { /* refill pio read ahead */
1312 unsigned int in_page, n;
1313 unsigned int count = ctxt->rep_prefix ?
1314 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1315 in_page = (ctxt->eflags & EFLG_DF) ?
1316 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1317 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1318 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1322 rc->pos = rc->end = 0;
1323 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1328 if (ctxt->rep_prefix && (ctxt->d & String) &&
1329 !(ctxt->eflags & EFLG_DF)) {
1330 ctxt->dst.data = rc->data + rc->pos;
1331 ctxt->dst.type = OP_MEM_STR;
1332 ctxt->dst.count = (rc->end - rc->pos) / size;
1335 memcpy(dest, rc->data + rc->pos, size);
1341 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1342 u16 index, struct desc_struct *desc)
1347 ctxt->ops->get_idt(ctxt, &dt);
1349 if (dt.size < index * 8 + 7)
1350 return emulate_gp(ctxt, index << 3 | 0x2);
1352 addr = dt.address + index * 8;
1353 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1357 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1358 u16 selector, struct desc_ptr *dt)
1360 const struct x86_emulate_ops *ops = ctxt->ops;
1363 if (selector & 1 << 2) {
1364 struct desc_struct desc;
1367 memset (dt, 0, sizeof *dt);
1368 if (!ops->get_segment(ctxt, &sel, &desc, &base3,
1372 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1373 dt->address = get_desc_base(&desc) | ((u64)base3 << 32);
1375 ops->get_gdt(ctxt, dt);
1378 /* allowed just for 8 bytes segments */
1379 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1380 u16 selector, struct desc_struct *desc,
1384 u16 index = selector >> 3;
1387 get_descriptor_table_ptr(ctxt, selector, &dt);
1389 if (dt.size < index * 8 + 7)
1390 return emulate_gp(ctxt, selector & 0xfffc);
1392 *desc_addr_p = addr = dt.address + index * 8;
1393 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1397 /* allowed just for 8 bytes segments */
1398 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1399 u16 selector, struct desc_struct *desc)
1402 u16 index = selector >> 3;
1405 get_descriptor_table_ptr(ctxt, selector, &dt);
1407 if (dt.size < index * 8 + 7)
1408 return emulate_gp(ctxt, selector & 0xfffc);
1410 addr = dt.address + index * 8;
1411 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1415 /* Does not support long mode */
1416 static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1417 u16 selector, int seg, u8 cpl, bool in_task_switch)
1419 struct desc_struct seg_desc, old_desc;
1421 unsigned err_vec = GP_VECTOR;
1423 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1429 memset(&seg_desc, 0, sizeof seg_desc);
1431 if (ctxt->mode == X86EMUL_MODE_REAL) {
1432 /* set real mode segment descriptor (keep limit etc. for
1434 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1435 set_desc_base(&seg_desc, selector << 4);
1437 } else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {
1438 /* VM86 needs a clean new segment descriptor */
1439 set_desc_base(&seg_desc, selector << 4);
1440 set_desc_limit(&seg_desc, 0xffff);
1450 /* NULL selector is not valid for TR, CS and SS (except for long mode) */
1451 if ((seg == VCPU_SREG_CS
1452 || (seg == VCPU_SREG_SS
1453 && (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl))
1454 || seg == VCPU_SREG_TR)
1458 /* TR should be in GDT only */
1459 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1462 if (null_selector) /* for NULL selector skip all following checks */
1465 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1466 if (ret != X86EMUL_CONTINUE)
1469 err_code = selector & 0xfffc;
1470 err_vec = GP_VECTOR;
1472 /* can't load system descriptor into segment selector */
1473 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1477 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1486 * segment is not a writable data segment or segment
1487 * selector's RPL != CPL or segment selector's RPL != CPL
1489 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1493 if (in_task_switch && rpl != dpl)
1496 if (!(seg_desc.type & 8))
1499 if (seg_desc.type & 4) {
1505 if (rpl > cpl || dpl != cpl)
1508 /* CS(RPL) <- CPL */
1509 selector = (selector & 0xfffc) | cpl;
1512 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1514 old_desc = seg_desc;
1515 seg_desc.type |= 2; /* busy */
1516 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1517 sizeof(seg_desc), &ctxt->exception);
1518 if (ret != X86EMUL_CONTINUE)
1521 case VCPU_SREG_LDTR:
1522 if (seg_desc.s || seg_desc.type != 2)
1525 default: /* DS, ES, FS, or GS */
1527 * segment is not a data or readable code segment or
1528 * ((segment is a data or nonconforming code segment)
1529 * and (both RPL and CPL > DPL))
1531 if ((seg_desc.type & 0xa) == 0x8 ||
1532 (((seg_desc.type & 0xc) != 0xc) &&
1533 (rpl > dpl && cpl > dpl)))
1539 /* mark segment as accessed */
1541 ret = write_segment_descriptor(ctxt, selector, &seg_desc);
1542 if (ret != X86EMUL_CONTINUE)
1544 } else if (ctxt->mode == X86EMUL_MODE_PROT64) {
1545 ret = ctxt->ops->read_std(ctxt, desc_addr+8, &base3,
1546 sizeof(base3), &ctxt->exception);
1547 if (ret != X86EMUL_CONTINUE)
1551 ctxt->ops->set_segment(ctxt, selector, &seg_desc, base3, seg);
1552 return X86EMUL_CONTINUE;
1554 emulate_exception(ctxt, err_vec, err_code, true);
1555 return X86EMUL_PROPAGATE_FAULT;
1558 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1559 u16 selector, int seg)
1561 u8 cpl = ctxt->ops->cpl(ctxt);
1562 return __load_segment_descriptor(ctxt, selector, seg, cpl, false);
1565 static void write_register_operand(struct operand *op)
1567 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1568 switch (op->bytes) {
1570 *(u8 *)op->addr.reg = (u8)op->val;
1573 *(u16 *)op->addr.reg = (u16)op->val;
1576 *op->addr.reg = (u32)op->val;
1577 break; /* 64b: zero-extend */
1579 *op->addr.reg = op->val;
1584 static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)
1590 write_register_operand(op);
1593 if (ctxt->lock_prefix)
1594 rc = segmented_cmpxchg(ctxt,
1600 rc = segmented_write(ctxt,
1604 if (rc != X86EMUL_CONTINUE)
1608 rc = segmented_write(ctxt,
1611 op->bytes * op->count);
1612 if (rc != X86EMUL_CONTINUE)
1616 write_sse_reg(ctxt, &op->vec_val, op->addr.xmm);
1619 write_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
1627 return X86EMUL_CONTINUE;
1630 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1632 struct segmented_address addr;
1634 rsp_increment(ctxt, -bytes);
1635 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1636 addr.seg = VCPU_SREG_SS;
1638 return segmented_write(ctxt, addr, data, bytes);
1641 static int em_push(struct x86_emulate_ctxt *ctxt)
1643 /* Disable writeback. */
1644 ctxt->dst.type = OP_NONE;
1645 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1648 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1649 void *dest, int len)
1652 struct segmented_address addr;
1654 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1655 addr.seg = VCPU_SREG_SS;
1656 rc = segmented_read(ctxt, addr, dest, len);
1657 if (rc != X86EMUL_CONTINUE)
1660 rsp_increment(ctxt, len);
1664 static int em_pop(struct x86_emulate_ctxt *ctxt)
1666 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1669 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1670 void *dest, int len)
1673 unsigned long val, change_mask;
1674 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1675 int cpl = ctxt->ops->cpl(ctxt);
1677 rc = emulate_pop(ctxt, &val, len);
1678 if (rc != X86EMUL_CONTINUE)
1681 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1682 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1684 switch(ctxt->mode) {
1685 case X86EMUL_MODE_PROT64:
1686 case X86EMUL_MODE_PROT32:
1687 case X86EMUL_MODE_PROT16:
1689 change_mask |= EFLG_IOPL;
1691 change_mask |= EFLG_IF;
1693 case X86EMUL_MODE_VM86:
1695 return emulate_gp(ctxt, 0);
1696 change_mask |= EFLG_IF;
1698 default: /* real mode */
1699 change_mask |= (EFLG_IOPL | EFLG_IF);
1703 *(unsigned long *)dest =
1704 (ctxt->eflags & ~change_mask) | (val & change_mask);
1709 static int em_popf(struct x86_emulate_ctxt *ctxt)
1711 ctxt->dst.type = OP_REG;
1712 ctxt->dst.addr.reg = &ctxt->eflags;
1713 ctxt->dst.bytes = ctxt->op_bytes;
1714 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1717 static int em_enter(struct x86_emulate_ctxt *ctxt)
1720 unsigned frame_size = ctxt->src.val;
1721 unsigned nesting_level = ctxt->src2.val & 31;
1725 return X86EMUL_UNHANDLEABLE;
1727 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1728 rc = push(ctxt, &rbp, stack_size(ctxt));
1729 if (rc != X86EMUL_CONTINUE)
1731 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1733 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1734 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1736 return X86EMUL_CONTINUE;
1739 static int em_leave(struct x86_emulate_ctxt *ctxt)
1741 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1743 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1746 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1748 int seg = ctxt->src2.val;
1750 ctxt->src.val = get_segment_selector(ctxt, seg);
1752 return em_push(ctxt);
1755 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1757 int seg = ctxt->src2.val;
1758 unsigned long selector;
1761 rc = emulate_pop(ctxt, &selector, ctxt->op_bytes);
1762 if (rc != X86EMUL_CONTINUE)
1765 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1769 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1771 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
1772 int rc = X86EMUL_CONTINUE;
1773 int reg = VCPU_REGS_RAX;
1775 while (reg <= VCPU_REGS_RDI) {
1776 (reg == VCPU_REGS_RSP) ?
1777 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
1780 if (rc != X86EMUL_CONTINUE)
1789 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1791 ctxt->src.val = (unsigned long)ctxt->eflags;
1792 return em_push(ctxt);
1795 static int em_popa(struct x86_emulate_ctxt *ctxt)
1797 int rc = X86EMUL_CONTINUE;
1798 int reg = VCPU_REGS_RDI;
1800 while (reg >= VCPU_REGS_RAX) {
1801 if (reg == VCPU_REGS_RSP) {
1802 rsp_increment(ctxt, ctxt->op_bytes);
1806 rc = emulate_pop(ctxt, reg_rmw(ctxt, reg), ctxt->op_bytes);
1807 if (rc != X86EMUL_CONTINUE)
1814 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1816 const struct x86_emulate_ops *ops = ctxt->ops;
1823 /* TODO: Add limit checks */
1824 ctxt->src.val = ctxt->eflags;
1826 if (rc != X86EMUL_CONTINUE)
1829 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1831 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1833 if (rc != X86EMUL_CONTINUE)
1836 ctxt->src.val = ctxt->_eip;
1838 if (rc != X86EMUL_CONTINUE)
1841 ops->get_idt(ctxt, &dt);
1843 eip_addr = dt.address + (irq << 2);
1844 cs_addr = dt.address + (irq << 2) + 2;
1846 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1847 if (rc != X86EMUL_CONTINUE)
1850 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1851 if (rc != X86EMUL_CONTINUE)
1854 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1855 if (rc != X86EMUL_CONTINUE)
1863 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1867 invalidate_registers(ctxt);
1868 rc = __emulate_int_real(ctxt, irq);
1869 if (rc == X86EMUL_CONTINUE)
1870 writeback_registers(ctxt);
1874 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
1876 switch(ctxt->mode) {
1877 case X86EMUL_MODE_REAL:
1878 return __emulate_int_real(ctxt, irq);
1879 case X86EMUL_MODE_VM86:
1880 case X86EMUL_MODE_PROT16:
1881 case X86EMUL_MODE_PROT32:
1882 case X86EMUL_MODE_PROT64:
1884 /* Protected mode interrupts unimplemented yet */
1885 return X86EMUL_UNHANDLEABLE;
1889 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
1891 int rc = X86EMUL_CONTINUE;
1892 unsigned long temp_eip = 0;
1893 unsigned long temp_eflags = 0;
1894 unsigned long cs = 0;
1895 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1896 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1897 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1898 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1900 /* TODO: Add stack limit check */
1902 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
1904 if (rc != X86EMUL_CONTINUE)
1907 if (temp_eip & ~0xffff)
1908 return emulate_gp(ctxt, 0);
1910 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1912 if (rc != X86EMUL_CONTINUE)
1915 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
1917 if (rc != X86EMUL_CONTINUE)
1920 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1922 if (rc != X86EMUL_CONTINUE)
1925 ctxt->_eip = temp_eip;
1928 if (ctxt->op_bytes == 4)
1929 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1930 else if (ctxt->op_bytes == 2) {
1931 ctxt->eflags &= ~0xffff;
1932 ctxt->eflags |= temp_eflags;
1935 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1936 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1941 static int em_iret(struct x86_emulate_ctxt *ctxt)
1943 switch(ctxt->mode) {
1944 case X86EMUL_MODE_REAL:
1945 return emulate_iret_real(ctxt);
1946 case X86EMUL_MODE_VM86:
1947 case X86EMUL_MODE_PROT16:
1948 case X86EMUL_MODE_PROT32:
1949 case X86EMUL_MODE_PROT64:
1951 /* iret from protected mode unimplemented yet */
1952 return X86EMUL_UNHANDLEABLE;
1956 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
1961 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1963 rc = load_segment_descriptor(ctxt, sel, VCPU_SREG_CS);
1964 if (rc != X86EMUL_CONTINUE)
1968 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
1969 return X86EMUL_CONTINUE;
1972 static int em_grp45(struct x86_emulate_ctxt *ctxt)
1974 int rc = X86EMUL_CONTINUE;
1976 switch (ctxt->modrm_reg) {
1977 case 2: /* call near abs */ {
1979 old_eip = ctxt->_eip;
1980 ctxt->_eip = ctxt->src.val;
1981 ctxt->src.val = old_eip;
1985 case 4: /* jmp abs */
1986 ctxt->_eip = ctxt->src.val;
1988 case 5: /* jmp far */
1989 rc = em_jmp_far(ctxt);
1998 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
2000 u64 old = ctxt->dst.orig_val64;
2002 if (ctxt->dst.bytes == 16)
2003 return X86EMUL_UNHANDLEABLE;
2005 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
2006 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
2007 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
2008 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2009 ctxt->eflags &= ~EFLG_ZF;
2011 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
2012 (u32) reg_read(ctxt, VCPU_REGS_RBX);
2014 ctxt->eflags |= EFLG_ZF;
2016 return X86EMUL_CONTINUE;
2019 static int em_ret(struct x86_emulate_ctxt *ctxt)
2021 ctxt->dst.type = OP_REG;
2022 ctxt->dst.addr.reg = &ctxt->_eip;
2023 ctxt->dst.bytes = ctxt->op_bytes;
2024 return em_pop(ctxt);
2027 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2032 rc = emulate_pop(ctxt, &ctxt->_eip, ctxt->op_bytes);
2033 if (rc != X86EMUL_CONTINUE)
2035 if (ctxt->op_bytes == 4)
2036 ctxt->_eip = (u32)ctxt->_eip;
2037 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2038 if (rc != X86EMUL_CONTINUE)
2040 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2044 static int em_ret_far_imm(struct x86_emulate_ctxt *ctxt)
2048 rc = em_ret_far(ctxt);
2049 if (rc != X86EMUL_CONTINUE)
2051 rsp_increment(ctxt, ctxt->src.val);
2052 return X86EMUL_CONTINUE;
2055 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2057 /* Save real source value, then compare EAX against destination. */
2058 ctxt->dst.orig_val = ctxt->dst.val;
2059 ctxt->dst.val = reg_read(ctxt, VCPU_REGS_RAX);
2060 ctxt->src.orig_val = ctxt->src.val;
2061 ctxt->src.val = ctxt->dst.orig_val;
2062 fastop(ctxt, em_cmp);
2064 if (ctxt->eflags & EFLG_ZF) {
2065 /* Success: write back to memory. */
2066 ctxt->dst.val = ctxt->src.orig_val;
2068 /* Failure: write the value we saw to EAX. */
2069 ctxt->dst.type = OP_REG;
2070 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2071 ctxt->dst.val = ctxt->dst.orig_val;
2073 return X86EMUL_CONTINUE;
2076 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2078 int seg = ctxt->src2.val;
2082 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2084 rc = load_segment_descriptor(ctxt, sel, seg);
2085 if (rc != X86EMUL_CONTINUE)
2088 ctxt->dst.val = ctxt->src.val;
2093 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2094 struct desc_struct *cs, struct desc_struct *ss)
2096 cs->l = 0; /* will be adjusted later */
2097 set_desc_base(cs, 0); /* flat segment */
2098 cs->g = 1; /* 4kb granularity */
2099 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2100 cs->type = 0x0b; /* Read, Execute, Accessed */
2102 cs->dpl = 0; /* will be adjusted later */
2107 set_desc_base(ss, 0); /* flat segment */
2108 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2109 ss->g = 1; /* 4kb granularity */
2111 ss->type = 0x03; /* Read/Write, Accessed */
2112 ss->d = 1; /* 32bit stack segment */
2119 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2121 u32 eax, ebx, ecx, edx;
2124 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2125 return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2126 && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2127 && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
2130 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2132 const struct x86_emulate_ops *ops = ctxt->ops;
2133 u32 eax, ebx, ecx, edx;
2136 * syscall should always be enabled in longmode - so only become
2137 * vendor specific (cpuid) if other modes are active...
2139 if (ctxt->mode == X86EMUL_MODE_PROT64)
2144 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2146 * Intel ("GenuineIntel")
2147 * remark: Intel CPUs only support "syscall" in 64bit
2148 * longmode. Also an 64bit guest with a
2149 * 32bit compat-app running will #UD !! While this
2150 * behaviour can be fixed (by emulating) into AMD
2151 * response - CPUs of AMD can't behave like Intel.
2153 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2154 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2155 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
2158 /* AMD ("AuthenticAMD") */
2159 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2160 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2161 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
2164 /* AMD ("AMDisbetter!") */
2165 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2166 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2167 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2170 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
2174 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2176 const struct x86_emulate_ops *ops = ctxt->ops;
2177 struct desc_struct cs, ss;
2182 /* syscall is not available in real mode */
2183 if (ctxt->mode == X86EMUL_MODE_REAL ||
2184 ctxt->mode == X86EMUL_MODE_VM86)
2185 return emulate_ud(ctxt);
2187 if (!(em_syscall_is_enabled(ctxt)))
2188 return emulate_ud(ctxt);
2190 ops->get_msr(ctxt, MSR_EFER, &efer);
2191 setup_syscalls_segments(ctxt, &cs, &ss);
2193 if (!(efer & EFER_SCE))
2194 return emulate_ud(ctxt);
2196 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2198 cs_sel = (u16)(msr_data & 0xfffc);
2199 ss_sel = (u16)(msr_data + 8);
2201 if (efer & EFER_LMA) {
2205 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2206 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2208 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2209 if (efer & EFER_LMA) {
2210 #ifdef CONFIG_X86_64
2211 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags & ~EFLG_RF;
2214 ctxt->mode == X86EMUL_MODE_PROT64 ?
2215 MSR_LSTAR : MSR_CSTAR, &msr_data);
2216 ctxt->_eip = msr_data;
2218 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2219 ctxt->eflags &= ~(msr_data | EFLG_RF);
2223 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2224 ctxt->_eip = (u32)msr_data;
2226 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2229 return X86EMUL_CONTINUE;
2232 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2234 const struct x86_emulate_ops *ops = ctxt->ops;
2235 struct desc_struct cs, ss;
2240 ops->get_msr(ctxt, MSR_EFER, &efer);
2241 /* inject #GP if in real mode */
2242 if (ctxt->mode == X86EMUL_MODE_REAL)
2243 return emulate_gp(ctxt, 0);
2246 * Not recognized on AMD in compat mode (but is recognized in legacy
2249 if ((ctxt->mode == X86EMUL_MODE_PROT32) && (efer & EFER_LMA)
2250 && !vendor_intel(ctxt))
2251 return emulate_ud(ctxt);
2253 /* XXX sysenter/sysexit have not been tested in 64bit mode.
2254 * Therefore, we inject an #UD.
2256 if (ctxt->mode == X86EMUL_MODE_PROT64)
2257 return emulate_ud(ctxt);
2259 setup_syscalls_segments(ctxt, &cs, &ss);
2261 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2262 switch (ctxt->mode) {
2263 case X86EMUL_MODE_PROT32:
2264 if ((msr_data & 0xfffc) == 0x0)
2265 return emulate_gp(ctxt, 0);
2267 case X86EMUL_MODE_PROT64:
2268 if (msr_data == 0x0)
2269 return emulate_gp(ctxt, 0);
2275 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2276 cs_sel = (u16)msr_data;
2277 cs_sel &= ~SELECTOR_RPL_MASK;
2278 ss_sel = cs_sel + 8;
2279 ss_sel &= ~SELECTOR_RPL_MASK;
2280 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
2285 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2286 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2288 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2289 ctxt->_eip = msr_data;
2291 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2292 *reg_write(ctxt, VCPU_REGS_RSP) = msr_data;
2294 return X86EMUL_CONTINUE;
2297 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2299 const struct x86_emulate_ops *ops = ctxt->ops;
2300 struct desc_struct cs, ss;
2303 u16 cs_sel = 0, ss_sel = 0;
2305 /* inject #GP if in real mode or Virtual 8086 mode */
2306 if (ctxt->mode == X86EMUL_MODE_REAL ||
2307 ctxt->mode == X86EMUL_MODE_VM86)
2308 return emulate_gp(ctxt, 0);
2310 setup_syscalls_segments(ctxt, &cs, &ss);
2312 if ((ctxt->rex_prefix & 0x8) != 0x0)
2313 usermode = X86EMUL_MODE_PROT64;
2315 usermode = X86EMUL_MODE_PROT32;
2319 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2321 case X86EMUL_MODE_PROT32:
2322 cs_sel = (u16)(msr_data + 16);
2323 if ((msr_data & 0xfffc) == 0x0)
2324 return emulate_gp(ctxt, 0);
2325 ss_sel = (u16)(msr_data + 24);
2327 case X86EMUL_MODE_PROT64:
2328 cs_sel = (u16)(msr_data + 32);
2329 if (msr_data == 0x0)
2330 return emulate_gp(ctxt, 0);
2331 ss_sel = cs_sel + 8;
2336 cs_sel |= SELECTOR_RPL_MASK;
2337 ss_sel |= SELECTOR_RPL_MASK;
2339 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2340 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2342 ctxt->_eip = reg_read(ctxt, VCPU_REGS_RDX);
2343 *reg_write(ctxt, VCPU_REGS_RSP) = reg_read(ctxt, VCPU_REGS_RCX);
2345 return X86EMUL_CONTINUE;
2348 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2351 if (ctxt->mode == X86EMUL_MODE_REAL)
2353 if (ctxt->mode == X86EMUL_MODE_VM86)
2355 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2356 return ctxt->ops->cpl(ctxt) > iopl;
2359 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2362 const struct x86_emulate_ops *ops = ctxt->ops;
2363 struct desc_struct tr_seg;
2366 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2367 unsigned mask = (1 << len) - 1;
2370 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2373 if (desc_limit_scaled(&tr_seg) < 103)
2375 base = get_desc_base(&tr_seg);
2376 #ifdef CONFIG_X86_64
2377 base |= ((u64)base3) << 32;
2379 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2380 if (r != X86EMUL_CONTINUE)
2382 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2384 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2385 if (r != X86EMUL_CONTINUE)
2387 if ((perm >> bit_idx) & mask)
2392 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2398 if (emulator_bad_iopl(ctxt))
2399 if (!emulator_io_port_access_allowed(ctxt, port, len))
2402 ctxt->perm_ok = true;
2407 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2408 struct tss_segment_16 *tss)
2410 tss->ip = ctxt->_eip;
2411 tss->flag = ctxt->eflags;
2412 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
2413 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
2414 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
2415 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
2416 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
2417 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
2418 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
2419 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
2421 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2422 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2423 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2424 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2425 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2428 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2429 struct tss_segment_16 *tss)
2434 ctxt->_eip = tss->ip;
2435 ctxt->eflags = tss->flag | 2;
2436 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
2437 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
2438 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
2439 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
2440 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
2441 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
2442 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
2443 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
2446 * SDM says that segment selectors are loaded before segment
2449 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2450 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2451 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2452 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2453 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2458 * Now load segment descriptors. If fault happens at this stage
2459 * it is handled in a context of new task
2461 ret = __load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR, cpl, true);
2462 if (ret != X86EMUL_CONTINUE)
2464 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl, true);
2465 if (ret != X86EMUL_CONTINUE)
2467 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl, true);
2468 if (ret != X86EMUL_CONTINUE)
2470 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl, true);
2471 if (ret != X86EMUL_CONTINUE)
2473 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl, true);
2474 if (ret != X86EMUL_CONTINUE)
2477 return X86EMUL_CONTINUE;
2480 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2481 u16 tss_selector, u16 old_tss_sel,
2482 ulong old_tss_base, struct desc_struct *new_desc)
2484 const struct x86_emulate_ops *ops = ctxt->ops;
2485 struct tss_segment_16 tss_seg;
2487 u32 new_tss_base = get_desc_base(new_desc);
2489 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2491 if (ret != X86EMUL_CONTINUE)
2492 /* FIXME: need to provide precise fault address */
2495 save_state_to_tss16(ctxt, &tss_seg);
2497 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2499 if (ret != X86EMUL_CONTINUE)
2500 /* FIXME: need to provide precise fault address */
2503 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2505 if (ret != X86EMUL_CONTINUE)
2506 /* FIXME: need to provide precise fault address */
2509 if (old_tss_sel != 0xffff) {
2510 tss_seg.prev_task_link = old_tss_sel;
2512 ret = ops->write_std(ctxt, new_tss_base,
2513 &tss_seg.prev_task_link,
2514 sizeof tss_seg.prev_task_link,
2516 if (ret != X86EMUL_CONTINUE)
2517 /* FIXME: need to provide precise fault address */
2521 return load_state_from_tss16(ctxt, &tss_seg);
2524 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2525 struct tss_segment_32 *tss)
2527 /* CR3 and ldt selector are not saved intentionally */
2528 tss->eip = ctxt->_eip;
2529 tss->eflags = ctxt->eflags;
2530 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
2531 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
2532 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
2533 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
2534 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
2535 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
2536 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
2537 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
2539 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2540 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2541 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2542 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2543 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2544 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2547 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2548 struct tss_segment_32 *tss)
2553 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2554 return emulate_gp(ctxt, 0);
2555 ctxt->_eip = tss->eip;
2556 ctxt->eflags = tss->eflags | 2;
2558 /* General purpose registers */
2559 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
2560 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
2561 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
2562 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
2563 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
2564 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
2565 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
2566 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
2569 * SDM says that segment selectors are loaded before segment
2570 * descriptors. This is important because CPL checks will
2573 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2574 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2575 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2576 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2577 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2578 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2579 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2582 * If we're switching between Protected Mode and VM86, we need to make
2583 * sure to update the mode before loading the segment descriptors so
2584 * that the selectors are interpreted correctly.
2586 if (ctxt->eflags & X86_EFLAGS_VM) {
2587 ctxt->mode = X86EMUL_MODE_VM86;
2590 ctxt->mode = X86EMUL_MODE_PROT32;
2595 * Now load segment descriptors. If fault happenes at this stage
2596 * it is handled in a context of new task
2598 ret = __load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR, cpl, true);
2599 if (ret != X86EMUL_CONTINUE)
2601 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl, true);
2602 if (ret != X86EMUL_CONTINUE)
2604 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl, true);
2605 if (ret != X86EMUL_CONTINUE)
2607 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl, true);
2608 if (ret != X86EMUL_CONTINUE)
2610 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl, true);
2611 if (ret != X86EMUL_CONTINUE)
2613 ret = __load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS, cpl, true);
2614 if (ret != X86EMUL_CONTINUE)
2616 ret = __load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS, cpl, true);
2617 if (ret != X86EMUL_CONTINUE)
2620 return X86EMUL_CONTINUE;
2623 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2624 u16 tss_selector, u16 old_tss_sel,
2625 ulong old_tss_base, struct desc_struct *new_desc)
2627 const struct x86_emulate_ops *ops = ctxt->ops;
2628 struct tss_segment_32 tss_seg;
2630 u32 new_tss_base = get_desc_base(new_desc);
2631 u32 eip_offset = offsetof(struct tss_segment_32, eip);
2632 u32 ldt_sel_offset = offsetof(struct tss_segment_32, ldt_selector);
2634 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2636 if (ret != X86EMUL_CONTINUE)
2637 /* FIXME: need to provide precise fault address */
2640 save_state_to_tss32(ctxt, &tss_seg);
2642 /* Only GP registers and segment selectors are saved */
2643 ret = ops->write_std(ctxt, old_tss_base + eip_offset, &tss_seg.eip,
2644 ldt_sel_offset - eip_offset, &ctxt->exception);
2645 if (ret != X86EMUL_CONTINUE)
2646 /* FIXME: need to provide precise fault address */
2649 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2651 if (ret != X86EMUL_CONTINUE)
2652 /* FIXME: need to provide precise fault address */
2655 if (old_tss_sel != 0xffff) {
2656 tss_seg.prev_task_link = old_tss_sel;
2658 ret = ops->write_std(ctxt, new_tss_base,
2659 &tss_seg.prev_task_link,
2660 sizeof tss_seg.prev_task_link,
2662 if (ret != X86EMUL_CONTINUE)
2663 /* FIXME: need to provide precise fault address */
2667 return load_state_from_tss32(ctxt, &tss_seg);
2670 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2671 u16 tss_selector, int idt_index, int reason,
2672 bool has_error_code, u32 error_code)
2674 const struct x86_emulate_ops *ops = ctxt->ops;
2675 struct desc_struct curr_tss_desc, next_tss_desc;
2677 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2678 ulong old_tss_base =
2679 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2683 /* FIXME: old_tss_base == ~0 ? */
2685 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
2686 if (ret != X86EMUL_CONTINUE)
2688 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
2689 if (ret != X86EMUL_CONTINUE)
2692 /* FIXME: check that next_tss_desc is tss */
2695 * Check privileges. The three cases are task switch caused by...
2697 * 1. jmp/call/int to task gate: Check against DPL of the task gate
2698 * 2. Exception/IRQ/iret: No check is performed
2699 * 3. jmp/call to TSS: Check against DPL of the TSS
2701 if (reason == TASK_SWITCH_GATE) {
2702 if (idt_index != -1) {
2703 /* Software interrupts */
2704 struct desc_struct task_gate_desc;
2707 ret = read_interrupt_descriptor(ctxt, idt_index,
2709 if (ret != X86EMUL_CONTINUE)
2712 dpl = task_gate_desc.dpl;
2713 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2714 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
2716 } else if (reason != TASK_SWITCH_IRET) {
2717 int dpl = next_tss_desc.dpl;
2718 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2719 return emulate_gp(ctxt, tss_selector);
2723 desc_limit = desc_limit_scaled(&next_tss_desc);
2724 if (!next_tss_desc.p ||
2725 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2726 desc_limit < 0x2b)) {
2727 emulate_ts(ctxt, tss_selector & 0xfffc);
2728 return X86EMUL_PROPAGATE_FAULT;
2731 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2732 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2733 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2736 if (reason == TASK_SWITCH_IRET)
2737 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2739 /* set back link to prev task only if NT bit is set in eflags
2740 note that old_tss_sel is not used after this point */
2741 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2742 old_tss_sel = 0xffff;
2744 if (next_tss_desc.type & 8)
2745 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2746 old_tss_base, &next_tss_desc);
2748 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2749 old_tss_base, &next_tss_desc);
2750 if (ret != X86EMUL_CONTINUE)
2753 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2754 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2756 if (reason != TASK_SWITCH_IRET) {
2757 next_tss_desc.type |= (1 << 1); /* set busy flag */
2758 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2761 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2762 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2764 if (has_error_code) {
2765 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2766 ctxt->lock_prefix = 0;
2767 ctxt->src.val = (unsigned long) error_code;
2768 ret = em_push(ctxt);
2774 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2775 u16 tss_selector, int idt_index, int reason,
2776 bool has_error_code, u32 error_code)
2780 invalidate_registers(ctxt);
2781 ctxt->_eip = ctxt->eip;
2782 ctxt->dst.type = OP_NONE;
2784 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
2785 has_error_code, error_code);
2787 if (rc == X86EMUL_CONTINUE) {
2788 ctxt->eip = ctxt->_eip;
2789 writeback_registers(ctxt);
2792 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2795 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
2798 int df = (ctxt->eflags & EFLG_DF) ? -op->count : op->count;
2800 register_address_increment(ctxt, reg_rmw(ctxt, reg), df * op->bytes);
2801 op->addr.mem.ea = register_address(ctxt, reg_read(ctxt, reg));
2804 static int em_das(struct x86_emulate_ctxt *ctxt)
2807 bool af, cf, old_cf;
2809 cf = ctxt->eflags & X86_EFLAGS_CF;
2815 af = ctxt->eflags & X86_EFLAGS_AF;
2816 if ((al & 0x0f) > 9 || af) {
2818 cf = old_cf | (al >= 250);
2823 if (old_al > 0x99 || old_cf) {
2829 /* Set PF, ZF, SF */
2830 ctxt->src.type = OP_IMM;
2832 ctxt->src.bytes = 1;
2833 fastop(ctxt, em_or);
2834 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2836 ctxt->eflags |= X86_EFLAGS_CF;
2838 ctxt->eflags |= X86_EFLAGS_AF;
2839 return X86EMUL_CONTINUE;
2842 static int em_aam(struct x86_emulate_ctxt *ctxt)
2846 if (ctxt->src.val == 0)
2847 return emulate_de(ctxt);
2849 al = ctxt->dst.val & 0xff;
2850 ah = al / ctxt->src.val;
2851 al %= ctxt->src.val;
2853 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);
2855 /* Set PF, ZF, SF */
2856 ctxt->src.type = OP_IMM;
2858 ctxt->src.bytes = 1;
2859 fastop(ctxt, em_or);
2861 return X86EMUL_CONTINUE;
2864 static int em_aad(struct x86_emulate_ctxt *ctxt)
2866 u8 al = ctxt->dst.val & 0xff;
2867 u8 ah = (ctxt->dst.val >> 8) & 0xff;
2869 al = (al + (ah * ctxt->src.val)) & 0xff;
2871 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
2873 /* Set PF, ZF, SF */
2874 ctxt->src.type = OP_IMM;
2876 ctxt->src.bytes = 1;
2877 fastop(ctxt, em_or);
2879 return X86EMUL_CONTINUE;
2882 static int em_call(struct x86_emulate_ctxt *ctxt)
2884 long rel = ctxt->src.val;
2886 ctxt->src.val = (unsigned long)ctxt->_eip;
2888 return em_push(ctxt);
2891 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2897 old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2898 old_eip = ctxt->_eip;
2900 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2901 if (load_segment_descriptor(ctxt, sel, VCPU_SREG_CS))
2902 return X86EMUL_CONTINUE;
2905 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
2907 ctxt->src.val = old_cs;
2909 if (rc != X86EMUL_CONTINUE)
2912 ctxt->src.val = old_eip;
2913 return em_push(ctxt);
2916 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
2920 ctxt->dst.type = OP_REG;
2921 ctxt->dst.addr.reg = &ctxt->_eip;
2922 ctxt->dst.bytes = ctxt->op_bytes;
2923 rc = emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
2924 if (rc != X86EMUL_CONTINUE)
2926 rsp_increment(ctxt, ctxt->src.val);
2927 return X86EMUL_CONTINUE;
2930 static int em_xchg(struct x86_emulate_ctxt *ctxt)
2932 /* Write back the register source. */
2933 ctxt->src.val = ctxt->dst.val;
2934 write_register_operand(&ctxt->src);
2936 /* Write back the memory destination with implicit LOCK prefix. */
2937 ctxt->dst.val = ctxt->src.orig_val;
2938 ctxt->lock_prefix = 1;
2939 return X86EMUL_CONTINUE;
2942 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
2944 ctxt->dst.val = ctxt->src2.val;
2945 return fastop(ctxt, em_imul);
2948 static int em_cwd(struct x86_emulate_ctxt *ctxt)
2950 ctxt->dst.type = OP_REG;
2951 ctxt->dst.bytes = ctxt->src.bytes;
2952 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
2953 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
2955 return X86EMUL_CONTINUE;
2958 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
2962 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
2963 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
2964 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
2965 return X86EMUL_CONTINUE;
2968 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
2972 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
2973 return emulate_gp(ctxt, 0);
2974 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
2975 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
2976 return X86EMUL_CONTINUE;
2979 static int em_mov(struct x86_emulate_ctxt *ctxt)
2981 memcpy(ctxt->dst.valptr, ctxt->src.valptr, ctxt->op_bytes);
2982 return X86EMUL_CONTINUE;
2985 #define FFL(x) bit(X86_FEATURE_##x)
2987 static int em_movbe(struct x86_emulate_ctxt *ctxt)
2989 u32 ebx, ecx, edx, eax = 1;
2993 * Check MOVBE is set in the guest-visible CPUID leaf.
2995 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2996 if (!(ecx & FFL(MOVBE)))
2997 return emulate_ud(ctxt);
2999 switch (ctxt->op_bytes) {
3002 * From MOVBE definition: "...When the operand size is 16 bits,
3003 * the upper word of the destination register remains unchanged
3006 * Both casting ->valptr and ->val to u16 breaks strict aliasing
3007 * rules so we have to do the operation almost per hand.
3009 tmp = (u16)ctxt->src.val;
3010 ctxt->dst.val &= ~0xffffUL;
3011 ctxt->dst.val |= (unsigned long)swab16(tmp);
3014 ctxt->dst.val = swab32((u32)ctxt->src.val);
3017 ctxt->dst.val = swab64(ctxt->src.val);
3020 return X86EMUL_PROPAGATE_FAULT;
3022 return X86EMUL_CONTINUE;
3025 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3027 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
3028 return emulate_gp(ctxt, 0);
3030 /* Disable writeback. */
3031 ctxt->dst.type = OP_NONE;
3032 return X86EMUL_CONTINUE;
3035 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
3039 if (ctxt->mode == X86EMUL_MODE_PROT64)
3040 val = ctxt->src.val & ~0ULL;
3042 val = ctxt->src.val & ~0U;
3044 /* #UD condition is already handled. */
3045 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
3046 return emulate_gp(ctxt, 0);
3048 /* Disable writeback. */
3049 ctxt->dst.type = OP_NONE;
3050 return X86EMUL_CONTINUE;
3053 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3057 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3058 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3059 if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
3060 return emulate_gp(ctxt, 0);
3062 return X86EMUL_CONTINUE;
3065 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3069 if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3070 return emulate_gp(ctxt, 0);
3072 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3073 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3074 return X86EMUL_CONTINUE;
3077 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3079 if (ctxt->modrm_reg > VCPU_SREG_GS)
3080 return emulate_ud(ctxt);
3082 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
3083 return X86EMUL_CONTINUE;
3086 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3088 u16 sel = ctxt->src.val;
3090 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3091 return emulate_ud(ctxt);
3093 if (ctxt->modrm_reg == VCPU_SREG_SS)
3094 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3096 /* Disable writeback. */
3097 ctxt->dst.type = OP_NONE;
3098 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3101 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3103 u16 sel = ctxt->src.val;
3105 /* Disable writeback. */
3106 ctxt->dst.type = OP_NONE;
3107 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3110 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3112 u16 sel = ctxt->src.val;
3114 /* Disable writeback. */
3115 ctxt->dst.type = OP_NONE;
3116 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3119 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3124 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3125 if (rc == X86EMUL_CONTINUE)
3126 ctxt->ops->invlpg(ctxt, linear);
3127 /* Disable writeback. */
3128 ctxt->dst.type = OP_NONE;
3129 return X86EMUL_CONTINUE;
3132 static int em_clts(struct x86_emulate_ctxt *ctxt)
3136 cr0 = ctxt->ops->get_cr(ctxt, 0);
3138 ctxt->ops->set_cr(ctxt, 0, cr0);
3139 return X86EMUL_CONTINUE;
3142 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
3146 if (ctxt->modrm_mod != 3 || ctxt->modrm_rm != 1)
3147 return X86EMUL_UNHANDLEABLE;
3149 rc = ctxt->ops->fix_hypercall(ctxt);
3150 if (rc != X86EMUL_CONTINUE)
3153 /* Let the processor re-execute the fixed hypercall */
3154 ctxt->_eip = ctxt->eip;
3155 /* Disable writeback. */
3156 ctxt->dst.type = OP_NONE;
3157 return X86EMUL_CONTINUE;
3160 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3161 void (*get)(struct x86_emulate_ctxt *ctxt,
3162 struct desc_ptr *ptr))
3164 struct desc_ptr desc_ptr;
3166 if (ctxt->mode == X86EMUL_MODE_PROT64)
3168 get(ctxt, &desc_ptr);
3169 if (ctxt->op_bytes == 2) {
3171 desc_ptr.address &= 0x00ffffff;
3173 /* Disable writeback. */
3174 ctxt->dst.type = OP_NONE;
3175 return segmented_write(ctxt, ctxt->dst.addr.mem,
3176 &desc_ptr, 2 + ctxt->op_bytes);
3179 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3181 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3184 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3186 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3189 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3191 struct desc_ptr desc_ptr;
3194 if (ctxt->mode == X86EMUL_MODE_PROT64)
3196 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3197 &desc_ptr.size, &desc_ptr.address,
3199 if (rc != X86EMUL_CONTINUE)
3201 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3202 /* Disable writeback. */
3203 ctxt->dst.type = OP_NONE;
3204 return X86EMUL_CONTINUE;
3207 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
3211 rc = ctxt->ops->fix_hypercall(ctxt);
3213 /* Disable writeback. */
3214 ctxt->dst.type = OP_NONE;
3218 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3220 struct desc_ptr desc_ptr;
3223 if (ctxt->mode == X86EMUL_MODE_PROT64)
3225 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3226 &desc_ptr.size, &desc_ptr.address,
3228 if (rc != X86EMUL_CONTINUE)
3230 ctxt->ops->set_idt(ctxt, &desc_ptr);
3231 /* Disable writeback. */
3232 ctxt->dst.type = OP_NONE;
3233 return X86EMUL_CONTINUE;
3236 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3238 if (ctxt->dst.type == OP_MEM)
3239 ctxt->dst.bytes = 2;
3240 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3241 return X86EMUL_CONTINUE;
3244 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3246 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3247 | (ctxt->src.val & 0x0f));
3248 ctxt->dst.type = OP_NONE;
3249 return X86EMUL_CONTINUE;
3252 static int em_loop(struct x86_emulate_ctxt *ctxt)
3254 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX), -1);
3255 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3256 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3257 jmp_rel(ctxt, ctxt->src.val);
3259 return X86EMUL_CONTINUE;
3262 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3264 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3265 jmp_rel(ctxt, ctxt->src.val);
3267 return X86EMUL_CONTINUE;
3270 static int em_in(struct x86_emulate_ctxt *ctxt)
3272 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3274 return X86EMUL_IO_NEEDED;
3276 return X86EMUL_CONTINUE;
3279 static int em_out(struct x86_emulate_ctxt *ctxt)
3281 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3283 /* Disable writeback. */
3284 ctxt->dst.type = OP_NONE;
3285 return X86EMUL_CONTINUE;
3288 static int em_cli(struct x86_emulate_ctxt *ctxt)
3290 if (emulator_bad_iopl(ctxt))
3291 return emulate_gp(ctxt, 0);
3293 ctxt->eflags &= ~X86_EFLAGS_IF;
3294 return X86EMUL_CONTINUE;
3297 static int em_sti(struct x86_emulate_ctxt *ctxt)
3299 if (emulator_bad_iopl(ctxt))
3300 return emulate_gp(ctxt, 0);
3302 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3303 ctxt->eflags |= X86_EFLAGS_IF;
3304 return X86EMUL_CONTINUE;
3307 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3309 u32 eax, ebx, ecx, edx;
3311 eax = reg_read(ctxt, VCPU_REGS_RAX);
3312 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3313 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3314 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3315 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3316 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3317 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
3318 return X86EMUL_CONTINUE;
3321 static int em_sahf(struct x86_emulate_ctxt *ctxt)
3325 flags = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF;
3326 flags &= *reg_rmw(ctxt, VCPU_REGS_RAX) >> 8;
3328 ctxt->eflags &= ~0xffUL;
3329 ctxt->eflags |= flags | X86_EFLAGS_FIXED;
3330 return X86EMUL_CONTINUE;
3333 static int em_lahf(struct x86_emulate_ctxt *ctxt)
3335 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
3336 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
3337 return X86EMUL_CONTINUE;
3340 static int em_bswap(struct x86_emulate_ctxt *ctxt)
3342 switch (ctxt->op_bytes) {
3343 #ifdef CONFIG_X86_64
3345 asm("bswap %0" : "+r"(ctxt->dst.val));
3349 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
3352 return X86EMUL_CONTINUE;
3355 static bool valid_cr(int nr)
3367 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
3369 if (!valid_cr(ctxt->modrm_reg))
3370 return emulate_ud(ctxt);
3372 return X86EMUL_CONTINUE;
3375 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
3377 u64 new_val = ctxt->src.val64;
3378 int cr = ctxt->modrm_reg;
3381 static u64 cr_reserved_bits[] = {
3382 0xffffffff00000000ULL,
3383 0, 0, 0, /* CR3 checked later */
3390 return emulate_ud(ctxt);
3392 if (new_val & cr_reserved_bits[cr])
3393 return emulate_gp(ctxt, 0);
3398 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
3399 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
3400 return emulate_gp(ctxt, 0);
3402 cr4 = ctxt->ops->get_cr(ctxt, 4);
3403 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3405 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
3406 !(cr4 & X86_CR4_PAE))
3407 return emulate_gp(ctxt, 0);
3414 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3415 if (efer & EFER_LMA)
3416 rsvd = CR3_L_MODE_RESERVED_BITS;
3419 return emulate_gp(ctxt, 0);
3424 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3426 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
3427 return emulate_gp(ctxt, 0);
3433 return X86EMUL_CONTINUE;
3436 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
3440 ctxt->ops->get_dr(ctxt, 7, &dr7);
3442 /* Check if DR7.Global_Enable is set */
3443 return dr7 & (1 << 13);
3446 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
3448 int dr = ctxt->modrm_reg;
3452 return emulate_ud(ctxt);
3454 cr4 = ctxt->ops->get_cr(ctxt, 4);
3455 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
3456 return emulate_ud(ctxt);
3458 if (check_dr7_gd(ctxt))
3459 return emulate_db(ctxt);
3461 return X86EMUL_CONTINUE;
3464 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
3466 u64 new_val = ctxt->src.val64;
3467 int dr = ctxt->modrm_reg;
3469 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
3470 return emulate_gp(ctxt, 0);
3472 return check_dr_read(ctxt);
3475 static int check_svme(struct x86_emulate_ctxt *ctxt)
3479 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3481 if (!(efer & EFER_SVME))
3482 return emulate_ud(ctxt);
3484 return X86EMUL_CONTINUE;
3487 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
3489 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
3491 /* Valid physical address? */
3492 if (rax & 0xffff000000000000ULL)
3493 return emulate_gp(ctxt, 0);
3495 return check_svme(ctxt);
3498 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
3500 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3502 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
3503 return emulate_ud(ctxt);
3505 return X86EMUL_CONTINUE;
3508 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
3510 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3511 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
3513 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
3514 ctxt->ops->check_pmc(ctxt, rcx))
3515 return emulate_gp(ctxt, 0);
3517 return X86EMUL_CONTINUE;
3520 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
3522 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
3523 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
3524 return emulate_gp(ctxt, 0);
3526 return X86EMUL_CONTINUE;
3529 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
3531 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
3532 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
3533 return emulate_gp(ctxt, 0);
3535 return X86EMUL_CONTINUE;
3538 #define D(_y) { .flags = (_y) }
3539 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
3540 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
3541 .check_perm = (_p) }
3542 #define N D(NotImpl)
3543 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3544 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
3545 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
3546 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
3547 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3548 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
3549 #define II(_f, _e, _i) \
3550 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
3551 #define IIP(_f, _e, _i, _p) \
3552 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
3553 .check_perm = (_p) }
3554 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3556 #define D2bv(_f) D((_f) | ByteOp), D(_f)
3557 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3558 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
3559 #define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
3560 #define I2bvIP(_f, _e, _i, _p) \
3561 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
3563 #define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
3564 F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3565 F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3567 static const struct opcode group7_rm1[] = {
3568 DI(SrcNone | Priv, monitor),
3569 DI(SrcNone | Priv, mwait),
3573 static const struct opcode group7_rm3[] = {
3574 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
3575 II(SrcNone | Prot | EmulateOnUD, em_vmmcall, vmmcall),
3576 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
3577 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
3578 DIP(SrcNone | Prot | Priv, stgi, check_svme),
3579 DIP(SrcNone | Prot | Priv, clgi, check_svme),
3580 DIP(SrcNone | Prot | Priv, skinit, check_svme),
3581 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
3584 static const struct opcode group7_rm7[] = {
3586 DIP(SrcNone, rdtscp, check_rdtsc),
3590 static const struct opcode group1[] = {
3592 F(Lock | PageTable, em_or),
3595 F(Lock | PageTable, em_and),
3601 static const struct opcode group1A[] = {
3602 I(DstMem | SrcNone | Mov | Stack, em_pop), N, N, N, N, N, N, N,
3605 static const struct opcode group2[] = {
3606 F(DstMem | ModRM, em_rol),
3607 F(DstMem | ModRM, em_ror),
3608 F(DstMem | ModRM, em_rcl),
3609 F(DstMem | ModRM, em_rcr),
3610 F(DstMem | ModRM, em_shl),
3611 F(DstMem | ModRM, em_shr),
3612 F(DstMem | ModRM, em_shl),
3613 F(DstMem | ModRM, em_sar),
3616 static const struct opcode group3[] = {
3617 F(DstMem | SrcImm | NoWrite, em_test),
3618 F(DstMem | SrcImm | NoWrite, em_test),
3619 F(DstMem | SrcNone | Lock, em_not),
3620 F(DstMem | SrcNone | Lock, em_neg),
3621 F(DstXacc | Src2Mem, em_mul_ex),
3622 F(DstXacc | Src2Mem, em_imul_ex),
3623 F(DstXacc | Src2Mem, em_div_ex),
3624 F(DstXacc | Src2Mem, em_idiv_ex),
3627 static const struct opcode group4[] = {
3628 F(ByteOp | DstMem | SrcNone | Lock, em_inc),
3629 F(ByteOp | DstMem | SrcNone | Lock, em_dec),
3633 static const struct opcode group5[] = {
3634 F(DstMem | SrcNone | Lock, em_inc),
3635 F(DstMem | SrcNone | Lock, em_dec),
3636 I(SrcMem | Stack, em_grp45),
3637 I(SrcMemFAddr | ImplicitOps | Stack, em_call_far),
3638 I(SrcMem | Stack, em_grp45),
3639 I(SrcMemFAddr | ImplicitOps, em_grp45),
3640 I(SrcMem | Stack, em_grp45), D(Undefined),
3643 static const struct opcode group6[] = {
3646 II(Prot | Priv | SrcMem16, em_lldt, lldt),
3647 II(Prot | Priv | SrcMem16, em_ltr, ltr),
3651 static const struct group_dual group7 = { {
3652 II(Mov | DstMem, em_sgdt, sgdt),
3653 II(Mov | DstMem, em_sidt, sidt),
3654 II(SrcMem | Priv, em_lgdt, lgdt),
3655 II(SrcMem | Priv, em_lidt, lidt),
3656 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3657 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3658 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3660 I(SrcNone | Priv | EmulateOnUD, em_vmcall),
3662 N, EXT(0, group7_rm3),
3663 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3664 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3668 static const struct opcode group8[] = {
3670 F(DstMem | SrcImmByte | NoWrite, em_bt),
3671 F(DstMem | SrcImmByte | Lock | PageTable, em_bts),
3672 F(DstMem | SrcImmByte | Lock, em_btr),
3673 F(DstMem | SrcImmByte | Lock | PageTable, em_btc),
3676 static const struct group_dual group9 = { {
3677 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
3679 N, N, N, N, N, N, N, N,
3682 static const struct opcode group11[] = {
3683 I(DstMem | SrcImm | Mov | PageTable, em_mov),
3687 static const struct gprefix pfx_0f_6f_0f_7f = {
3688 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
3691 static const struct gprefix pfx_vmovntpx = {
3692 I(0, em_mov), N, N, N,
3695 static const struct gprefix pfx_0f_28_0f_29 = {
3696 I(Aligned, em_mov), I(Aligned, em_mov), N, N,
3699 static const struct escape escape_d9 = { {
3700 N, N, N, N, N, N, N, I(DstMem, em_fnstcw),
3703 N, N, N, N, N, N, N, N,
3705 N, N, N, N, N, N, N, N,
3707 N, N, N, N, N, N, N, N,
3709 N, N, N, N, N, N, N, N,
3711 N, N, N, N, N, N, N, N,
3713 N, N, N, N, N, N, N, N,
3715 N, N, N, N, N, N, N, N,
3717 N, N, N, N, N, N, N, N,
3720 static const struct escape escape_db = { {
3721 N, N, N, N, N, N, N, N,
3724 N, N, N, N, N, N, N, N,
3726 N, N, N, N, N, N, N, N,
3728 N, N, N, N, N, N, N, N,
3730 N, N, N, N, N, N, N, N,
3732 N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
3734 N, N, N, N, N, N, N, N,
3736 N, N, N, N, N, N, N, N,
3738 N, N, N, N, N, N, N, N,
3741 static const struct escape escape_dd = { {
3742 N, N, N, N, N, N, N, I(DstMem, em_fnstsw),
3745 N, N, N, N, N, N, N, N,
3747 N, N, N, N, N, N, N, N,
3749 N, N, N, N, N, N, N, N,
3751 N, N, N, N, N, N, N, N,
3753 N, N, N, N, N, N, N, N,
3755 N, N, N, N, N, N, N, N,
3757 N, N, N, N, N, N, N, N,
3759 N, N, N, N, N, N, N, N,
3762 static const struct opcode opcode_table[256] = {
3764 F6ALU(Lock, em_add),
3765 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
3766 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
3768 F6ALU(Lock | PageTable, em_or),
3769 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
3772 F6ALU(Lock, em_adc),
3773 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
3774 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
3776 F6ALU(Lock, em_sbb),
3777 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
3778 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
3780 F6ALU(Lock | PageTable, em_and), N, N,
3782 F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3784 F6ALU(Lock, em_xor), N, N,
3786 F6ALU(NoWrite, em_cmp), N, N,
3788 X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
3790 X8(I(SrcReg | Stack, em_push)),
3792 X8(I(DstReg | Stack, em_pop)),
3794 I(ImplicitOps | Stack | No64, em_pusha),
3795 I(ImplicitOps | Stack | No64, em_popa),
3796 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
3799 I(SrcImm | Mov | Stack, em_push),
3800 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3801 I(SrcImmByte | Mov | Stack, em_push),
3802 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3803 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
3804 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
3808 G(ByteOp | DstMem | SrcImm, group1),
3809 G(DstMem | SrcImm, group1),
3810 G(ByteOp | DstMem | SrcImm | No64, group1),
3811 G(DstMem | SrcImmByte, group1),
3812 F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
3813 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
3815 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
3816 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3817 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
3818 D(ModRM | SrcMem | NoAccess | DstReg),
3819 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
3822 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3824 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3825 I(SrcImmFAddr | No64, em_call_far), N,
3826 II(ImplicitOps | Stack, em_pushf, pushf),
3827 II(ImplicitOps | Stack, em_popf, popf),
3828 I(ImplicitOps, em_sahf), I(ImplicitOps, em_lahf),
3830 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3831 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
3832 I2bv(SrcSI | DstDI | Mov | String, em_mov),
3833 F2bv(SrcSI | DstDI | String | NoWrite, em_cmp),
3835 F2bv(DstAcc | SrcImm | NoWrite, em_test),
3836 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
3837 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3838 F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp),
3840 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3842 X8(I(DstReg | SrcImm64 | Mov, em_mov)),
3844 G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
3845 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3846 I(ImplicitOps | Stack, em_ret),
3847 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
3848 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
3849 G(ByteOp, group11), G(0, group11),
3851 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
3852 I(ImplicitOps | Stack | SrcImmU16, em_ret_far_imm),
3853 I(ImplicitOps | Stack, em_ret_far),
3854 D(ImplicitOps), DI(SrcImmByte, intn),
3855 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
3857 G(Src2One | ByteOp, group2), G(Src2One, group2),
3858 G(Src2CL | ByteOp, group2), G(Src2CL, group2),
3859 I(DstAcc | SrcImmUByte | No64, em_aam),
3860 I(DstAcc | SrcImmUByte | No64, em_aad),
3861 F(DstAcc | ByteOp | No64, em_salc),
3862 I(DstAcc | SrcXLat | ByteOp, em_mov),
3864 N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
3866 X3(I(SrcImmByte, em_loop)),
3867 I(SrcImmByte, em_jcxz),
3868 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
3869 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
3871 I(SrcImm | Stack, em_call), D(SrcImm | ImplicitOps),
3872 I(SrcImmFAddr | No64, em_jmp_far), D(SrcImmByte | ImplicitOps),
3873 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
3874 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
3876 N, DI(ImplicitOps, icebp), N, N,
3877 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
3878 G(ByteOp, group3), G(0, group3),
3880 D(ImplicitOps), D(ImplicitOps),
3881 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
3882 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
3885 static const struct opcode twobyte_table[256] = {
3887 G(0, group6), GD(0, &group7), N, N,
3888 N, I(ImplicitOps | EmulateOnUD, em_syscall),
3889 II(ImplicitOps | Priv, em_clts, clts), N,
3890 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3891 N, D(ImplicitOps | ModRM), N, N,
3893 N, N, N, N, N, N, N, N,
3894 D(ImplicitOps | ModRM), N, N, N, N, N, N, D(ImplicitOps | ModRM),
3896 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, cr_read, check_cr_read),
3897 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, dr_read, check_dr_read),
3898 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_cr_write, cr_write,
3900 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_dr_write, dr_write,
3903 GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_28_0f_29),
3904 GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_28_0f_29),
3905 N, GP(ModRM | DstMem | SrcReg | Sse | Mov | Aligned, &pfx_vmovntpx),
3908 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
3909 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
3910 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
3911 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
3912 I(ImplicitOps | EmulateOnUD, em_sysenter),
3913 I(ImplicitOps | Priv | EmulateOnUD, em_sysexit),
3915 N, N, N, N, N, N, N, N,
3917 X16(D(DstReg | SrcMem | ModRM | Mov)),
3919 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3924 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3929 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3933 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3935 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
3936 II(ImplicitOps, em_cpuid, cpuid),
3937 F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
3938 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
3939 F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
3941 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
3942 DI(ImplicitOps, rsm),
3943 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
3944 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
3945 F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
3946 D(ModRM), F(DstReg | SrcMem | ModRM, em_imul),
3948 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_cmpxchg),
3949 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
3950 F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
3951 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
3952 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
3953 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3957 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
3958 F(DstReg | SrcMem | ModRM, em_bsf), F(DstReg | SrcMem | ModRM, em_bsr),
3959 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3961 F2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd),
3962 N, D(DstMem | SrcReg | ModRM | Mov),
3963 N, N, N, GD(0, &group9),
3965 X8(I(DstReg, em_bswap)),
3967 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3969 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3971 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
3974 static const struct gprefix three_byte_0f_38_f0 = {
3975 I(DstReg | SrcMem | Mov, em_movbe), N, N, N
3978 static const struct gprefix three_byte_0f_38_f1 = {
3979 I(DstMem | SrcReg | Mov, em_movbe), N, N, N
3983 * Insns below are selected by the prefix which indexed by the third opcode
3986 static const struct opcode opcode_map_0f_38[256] = {
3988 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
3990 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
3992 GP(EmulateOnUD | ModRM | Prefix, &three_byte_0f_38_f0),
3993 GP(EmulateOnUD | ModRM | Prefix, &three_byte_0f_38_f1),
4012 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
4016 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4022 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
4023 unsigned size, bool sign_extension)
4025 int rc = X86EMUL_CONTINUE;
4029 op->addr.mem.ea = ctxt->_eip;
4030 /* NB. Immediates are sign-extended as necessary. */
4031 switch (op->bytes) {
4033 op->val = insn_fetch(s8, ctxt);
4036 op->val = insn_fetch(s16, ctxt);
4039 op->val = insn_fetch(s32, ctxt);
4042 op->val = insn_fetch(s64, ctxt);
4045 if (!sign_extension) {
4046 switch (op->bytes) {
4054 op->val &= 0xffffffff;
4062 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
4065 int rc = X86EMUL_CONTINUE;
4069 decode_register_operand(ctxt, op);
4072 rc = decode_imm(ctxt, op, 1, false);
4075 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4079 if ((ctxt->d & BitOp) && op == &ctxt->dst)
4080 fetch_bit_operand(ctxt);
4081 op->orig_val = op->val;
4084 ctxt->memop.bytes = (ctxt->op_bytes == 8) ? 16 : 8;
4088 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4089 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4090 fetch_register_operand(op);
4091 op->orig_val = op->val;
4095 op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes;
4096 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4097 fetch_register_operand(op);
4098 op->orig_val = op->val;
4101 if (ctxt->d & ByteOp) {
4106 op->bytes = ctxt->op_bytes;
4107 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4108 fetch_register_operand(op);
4109 op->orig_val = op->val;
4113 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4115 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RDI));
4116 op->addr.mem.seg = VCPU_SREG_ES;
4123 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4124 fetch_register_operand(op);
4128 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
4131 rc = decode_imm(ctxt, op, 1, true);
4138 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
4141 rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
4144 ctxt->memop.bytes = 1;
4145 if (ctxt->memop.type == OP_REG) {
4146 ctxt->memop.addr.reg = decode_register(ctxt,
4147 ctxt->modrm_rm, true);
4148 fetch_register_operand(&ctxt->memop);
4152 ctxt->memop.bytes = 2;
4155 ctxt->memop.bytes = 4;
4158 rc = decode_imm(ctxt, op, 2, false);
4161 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
4165 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4167 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RSI));
4168 op->addr.mem.seg = seg_override(ctxt);
4174 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4176 register_address(ctxt,
4177 reg_read(ctxt, VCPU_REGS_RBX) +
4178 (reg_read(ctxt, VCPU_REGS_RAX) & 0xff));
4179 op->addr.mem.seg = seg_override(ctxt);
4184 op->addr.mem.ea = ctxt->_eip;
4185 op->bytes = ctxt->op_bytes + 2;
4186 insn_fetch_arr(op->valptr, op->bytes, ctxt);
4189 ctxt->memop.bytes = ctxt->op_bytes + 2;
4192 op->val = VCPU_SREG_ES;
4195 op->val = VCPU_SREG_CS;
4198 op->val = VCPU_SREG_SS;
4201 op->val = VCPU_SREG_DS;
4204 op->val = VCPU_SREG_FS;
4207 op->val = VCPU_SREG_GS;
4210 /* Special instructions do their own operand decoding. */
4212 op->type = OP_NONE; /* Disable writeback. */
4220 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
4222 int rc = X86EMUL_CONTINUE;
4223 int mode = ctxt->mode;
4224 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
4225 bool op_prefix = false;
4226 struct opcode opcode;
4228 ctxt->memop.type = OP_NONE;
4229 ctxt->memopp = NULL;
4230 ctxt->_eip = ctxt->eip;
4231 ctxt->fetch.start = ctxt->_eip;
4232 ctxt->fetch.end = ctxt->fetch.start + insn_len;
4233 ctxt->opcode_len = 1;
4235 memcpy(ctxt->fetch.data, insn, insn_len);
4238 case X86EMUL_MODE_REAL:
4239 case X86EMUL_MODE_VM86:
4240 case X86EMUL_MODE_PROT16:
4241 def_op_bytes = def_ad_bytes = 2;
4243 case X86EMUL_MODE_PROT32:
4244 def_op_bytes = def_ad_bytes = 4;
4246 #ifdef CONFIG_X86_64
4247 case X86EMUL_MODE_PROT64:
4253 return EMULATION_FAILED;
4256 ctxt->op_bytes = def_op_bytes;
4257 ctxt->ad_bytes = def_ad_bytes;
4259 /* Legacy prefixes. */
4261 switch (ctxt->b = insn_fetch(u8, ctxt)) {
4262 case 0x66: /* operand-size override */
4264 /* switch between 2/4 bytes */
4265 ctxt->op_bytes = def_op_bytes ^ 6;
4267 case 0x67: /* address-size override */
4268 if (mode == X86EMUL_MODE_PROT64)
4269 /* switch between 4/8 bytes */
4270 ctxt->ad_bytes = def_ad_bytes ^ 12;
4272 /* switch between 2/4 bytes */
4273 ctxt->ad_bytes = def_ad_bytes ^ 6;
4275 case 0x26: /* ES override */
4276 case 0x2e: /* CS override */
4277 case 0x36: /* SS override */
4278 case 0x3e: /* DS override */
4279 set_seg_override(ctxt, (ctxt->b >> 3) & 3);
4281 case 0x64: /* FS override */
4282 case 0x65: /* GS override */
4283 set_seg_override(ctxt, ctxt->b & 7);
4285 case 0x40 ... 0x4f: /* REX */
4286 if (mode != X86EMUL_MODE_PROT64)
4288 ctxt->rex_prefix = ctxt->b;
4290 case 0xf0: /* LOCK */
4291 ctxt->lock_prefix = 1;
4293 case 0xf2: /* REPNE/REPNZ */
4294 case 0xf3: /* REP/REPE/REPZ */
4295 ctxt->rep_prefix = ctxt->b;
4301 /* Any legacy prefix after a REX prefix nullifies its effect. */
4303 ctxt->rex_prefix = 0;
4309 if (ctxt->rex_prefix & 8)
4310 ctxt->op_bytes = 8; /* REX.W */
4312 /* Opcode byte(s). */
4313 opcode = opcode_table[ctxt->b];
4314 /* Two-byte opcode? */
4315 if (ctxt->b == 0x0f) {
4316 ctxt->opcode_len = 2;
4317 ctxt->b = insn_fetch(u8, ctxt);
4318 opcode = twobyte_table[ctxt->b];
4320 /* 0F_38 opcode map */
4321 if (ctxt->b == 0x38) {
4322 ctxt->opcode_len = 3;
4323 ctxt->b = insn_fetch(u8, ctxt);
4324 opcode = opcode_map_0f_38[ctxt->b];
4327 ctxt->d = opcode.flags;
4329 if (ctxt->d & ModRM)
4330 ctxt->modrm = insn_fetch(u8, ctxt);
4332 /* vex-prefix instructions are not implemented */
4333 if (ctxt->opcode_len == 1 && (ctxt->b == 0xc5 || ctxt->b == 0xc4) &&
4334 (mode == X86EMUL_MODE_PROT64 ||
4335 (mode >= X86EMUL_MODE_PROT16 && (ctxt->modrm & 0x80)))) {
4339 while (ctxt->d & GroupMask) {
4340 switch (ctxt->d & GroupMask) {
4342 goffset = (ctxt->modrm >> 3) & 7;
4343 opcode = opcode.u.group[goffset];
4346 goffset = (ctxt->modrm >> 3) & 7;
4347 if ((ctxt->modrm >> 6) == 3)
4348 opcode = opcode.u.gdual->mod3[goffset];
4350 opcode = opcode.u.gdual->mod012[goffset];
4353 goffset = ctxt->modrm & 7;
4354 opcode = opcode.u.group[goffset];
4357 if (ctxt->rep_prefix && op_prefix)
4358 return EMULATION_FAILED;
4359 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
4360 switch (simd_prefix) {
4361 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
4362 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
4363 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
4364 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
4368 if (ctxt->modrm > 0xbf)
4369 opcode = opcode.u.esc->high[ctxt->modrm - 0xc0];
4371 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
4374 return EMULATION_FAILED;
4377 ctxt->d &= ~(u64)GroupMask;
4378 ctxt->d |= opcode.flags;
4381 ctxt->execute = opcode.u.execute;
4382 ctxt->check_perm = opcode.check_perm;
4383 ctxt->intercept = opcode.intercept;
4386 if (ctxt->d == 0 || (ctxt->d & NotImpl))
4387 return EMULATION_FAILED;
4389 if (!(ctxt->d & EmulateOnUD) && ctxt->ud)
4390 return EMULATION_FAILED;
4392 if (mode == X86EMUL_MODE_PROT64 && (ctxt->d & Stack))
4395 if (ctxt->d & Op3264) {
4396 if (mode == X86EMUL_MODE_PROT64)
4403 ctxt->op_bytes = 16;
4404 else if (ctxt->d & Mmx)
4407 /* ModRM and SIB bytes. */
4408 if (ctxt->d & ModRM) {
4409 rc = decode_modrm(ctxt, &ctxt->memop);
4410 if (!ctxt->has_seg_override)
4411 set_seg_override(ctxt, ctxt->modrm_seg);
4412 } else if (ctxt->d & MemAbs)
4413 rc = decode_abs(ctxt, &ctxt->memop);
4414 if (rc != X86EMUL_CONTINUE)
4417 if (!ctxt->has_seg_override)
4418 set_seg_override(ctxt, VCPU_SREG_DS);
4420 ctxt->memop.addr.mem.seg = seg_override(ctxt);
4422 if (ctxt->memop.type == OP_MEM && ctxt->ad_bytes != 8)
4423 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
4426 * Decode and fetch the source operand: register, memory
4429 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
4430 if (rc != X86EMUL_CONTINUE)
4434 * Decode and fetch the second source operand: register, memory
4437 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
4438 if (rc != X86EMUL_CONTINUE)
4441 /* Decode and fetch the destination operand: register or memory. */
4442 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
4445 if (ctxt->memopp && ctxt->memopp->type == OP_MEM && ctxt->rip_relative)
4446 ctxt->memopp->addr.mem.ea += ctxt->_eip;
4448 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
4451 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
4453 return ctxt->d & PageTable;
4456 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
4458 /* The second termination condition only applies for REPE
4459 * and REPNE. Test if the repeat string operation prefix is
4460 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
4461 * corresponding termination condition according to:
4462 * - if REPE/REPZ and ZF = 0 then done
4463 * - if REPNE/REPNZ and ZF = 1 then done
4465 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
4466 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
4467 && (((ctxt->rep_prefix == REPE_PREFIX) &&
4468 ((ctxt->eflags & EFLG_ZF) == 0))
4469 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
4470 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
4476 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
4480 ctxt->ops->get_fpu(ctxt);
4481 asm volatile("1: fwait \n\t"
4483 ".pushsection .fixup,\"ax\" \n\t"
4485 "movb $1, %[fault] \n\t"
4488 _ASM_EXTABLE(1b, 3b)
4489 : [fault]"+qm"(fault));
4490 ctxt->ops->put_fpu(ctxt);
4492 if (unlikely(fault))
4493 return emulate_exception(ctxt, MF_VECTOR, 0, false);
4495 return X86EMUL_CONTINUE;
4498 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
4501 if (op->type == OP_MM)
4502 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
4505 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))
4507 ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
4508 if (!(ctxt->d & ByteOp))
4509 fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
4510 asm("push %[flags]; popf; call *%[fastop]; pushf; pop %[flags]\n"
4511 : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),
4513 : "c"(ctxt->src2.val));
4514 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
4515 if (!fop) /* exception is returned in fop variable */
4516 return emulate_de(ctxt);
4517 return X86EMUL_CONTINUE;
4520 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
4522 const struct x86_emulate_ops *ops = ctxt->ops;
4523 int rc = X86EMUL_CONTINUE;
4524 int saved_dst_type = ctxt->dst.type;
4526 ctxt->mem_read.pos = 0;
4528 if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||
4529 (ctxt->d & Undefined)) {
4530 rc = emulate_ud(ctxt);
4534 /* LOCK prefix is allowed only with some instructions */
4535 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
4536 rc = emulate_ud(ctxt);
4540 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
4541 rc = emulate_ud(ctxt);
4545 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
4546 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
4547 rc = emulate_ud(ctxt);
4551 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
4552 rc = emulate_nm(ctxt);
4556 if (ctxt->d & Mmx) {
4557 rc = flush_pending_x87_faults(ctxt);
4558 if (rc != X86EMUL_CONTINUE)
4561 * Now that we know the fpu is exception safe, we can fetch
4564 fetch_possible_mmx_operand(ctxt, &ctxt->src);
4565 fetch_possible_mmx_operand(ctxt, &ctxt->src2);
4566 if (!(ctxt->d & Mov))
4567 fetch_possible_mmx_operand(ctxt, &ctxt->dst);
4570 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4571 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4572 X86_ICPT_PRE_EXCEPT);
4573 if (rc != X86EMUL_CONTINUE)
4577 /* Privileged instruction can be executed only in CPL=0 */
4578 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
4579 rc = emulate_gp(ctxt, 0);
4583 /* Instruction can only be executed in protected mode */
4584 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
4585 rc = emulate_ud(ctxt);
4589 /* Do instruction specific permission checks */
4590 if (ctxt->check_perm) {
4591 rc = ctxt->check_perm(ctxt);
4592 if (rc != X86EMUL_CONTINUE)
4596 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4597 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4598 X86_ICPT_POST_EXCEPT);
4599 if (rc != X86EMUL_CONTINUE)
4603 if (ctxt->rep_prefix && (ctxt->d & String)) {
4604 /* All REP prefixes have the same first termination condition */
4605 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
4606 ctxt->eip = ctxt->_eip;
4611 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
4612 rc = segmented_read(ctxt, ctxt->src.addr.mem,
4613 ctxt->src.valptr, ctxt->src.bytes);
4614 if (rc != X86EMUL_CONTINUE)
4616 ctxt->src.orig_val64 = ctxt->src.val64;
4619 if (ctxt->src2.type == OP_MEM) {
4620 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
4621 &ctxt->src2.val, ctxt->src2.bytes);
4622 if (rc != X86EMUL_CONTINUE)
4626 if ((ctxt->d & DstMask) == ImplicitOps)
4630 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
4631 /* optimisation - avoid slow emulated read if Mov */
4632 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
4633 &ctxt->dst.val, ctxt->dst.bytes);
4634 if (rc != X86EMUL_CONTINUE)
4637 ctxt->dst.orig_val = ctxt->dst.val;
4641 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4642 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4643 X86_ICPT_POST_MEMACCESS);
4644 if (rc != X86EMUL_CONTINUE)
4648 if (ctxt->execute) {
4649 if (ctxt->d & Fastop) {
4650 void (*fop)(struct fastop *) = (void *)ctxt->execute;
4651 rc = fastop(ctxt, fop);
4652 if (rc != X86EMUL_CONTINUE)
4656 rc = ctxt->execute(ctxt);
4657 if (rc != X86EMUL_CONTINUE)
4662 if (ctxt->opcode_len == 2)
4664 else if (ctxt->opcode_len == 3)
4665 goto threebyte_insn;
4668 case 0x63: /* movsxd */
4669 if (ctxt->mode != X86EMUL_MODE_PROT64)
4670 goto cannot_emulate;
4671 ctxt->dst.val = (s32) ctxt->src.val;
4673 case 0x70 ... 0x7f: /* jcc (short) */
4674 if (test_cc(ctxt->b, ctxt->eflags))
4675 jmp_rel(ctxt, ctxt->src.val);
4677 case 0x8d: /* lea r16/r32, m */
4678 ctxt->dst.val = ctxt->src.addr.mem.ea;
4680 case 0x90 ... 0x97: /* nop / xchg reg, rax */
4681 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
4685 case 0x98: /* cbw/cwde/cdqe */
4686 switch (ctxt->op_bytes) {
4687 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
4688 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
4689 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
4692 case 0xcc: /* int3 */
4693 rc = emulate_int(ctxt, 3);
4695 case 0xcd: /* int n */
4696 rc = emulate_int(ctxt, ctxt->src.val);
4698 case 0xce: /* into */
4699 if (ctxt->eflags & EFLG_OF)
4700 rc = emulate_int(ctxt, 4);
4702 case 0xe9: /* jmp rel */
4703 case 0xeb: /* jmp rel short */
4704 jmp_rel(ctxt, ctxt->src.val);
4705 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4707 case 0xf4: /* hlt */
4708 ctxt->ops->halt(ctxt);
4710 case 0xf5: /* cmc */
4711 /* complement carry flag from eflags reg */
4712 ctxt->eflags ^= EFLG_CF;
4714 case 0xf8: /* clc */
4715 ctxt->eflags &= ~EFLG_CF;
4717 case 0xf9: /* stc */
4718 ctxt->eflags |= EFLG_CF;
4720 case 0xfc: /* cld */
4721 ctxt->eflags &= ~EFLG_DF;
4723 case 0xfd: /* std */
4724 ctxt->eflags |= EFLG_DF;
4727 goto cannot_emulate;
4730 if (rc != X86EMUL_CONTINUE)
4734 if (!(ctxt->d & NoWrite)) {
4735 rc = writeback(ctxt, &ctxt->dst);
4736 if (rc != X86EMUL_CONTINUE)
4739 if (ctxt->d & SrcWrite) {
4740 BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR);
4741 rc = writeback(ctxt, &ctxt->src);
4742 if (rc != X86EMUL_CONTINUE)
4747 * restore dst type in case the decoding will be reused
4748 * (happens for string instruction )
4750 ctxt->dst.type = saved_dst_type;
4752 if ((ctxt->d & SrcMask) == SrcSI)
4753 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
4755 if ((ctxt->d & DstMask) == DstDI)
4756 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
4758 if (ctxt->rep_prefix && (ctxt->d & String)) {
4760 struct read_cache *r = &ctxt->io_read;
4761 if ((ctxt->d & SrcMask) == SrcSI)
4762 count = ctxt->src.count;
4764 count = ctxt->dst.count;
4765 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX),
4768 if (!string_insn_completed(ctxt)) {
4770 * Re-enter guest when pio read ahead buffer is empty
4771 * or, if it is not used, after each 1024 iteration.
4773 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
4774 (r->end == 0 || r->end != r->pos)) {
4776 * Reset read cache. Usually happens before
4777 * decode, but since instruction is restarted
4778 * we have to do it here.
4780 ctxt->mem_read.end = 0;
4781 writeback_registers(ctxt);
4782 return EMULATION_RESTART;
4784 goto done; /* skip rip writeback */
4788 ctxt->eip = ctxt->_eip;
4791 if (rc == X86EMUL_PROPAGATE_FAULT)
4792 ctxt->have_exception = true;
4793 if (rc == X86EMUL_INTERCEPTED)
4794 return EMULATION_INTERCEPTED;
4796 if (rc == X86EMUL_CONTINUE)
4797 writeback_registers(ctxt);
4799 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4803 case 0x09: /* wbinvd */
4804 (ctxt->ops->wbinvd)(ctxt);
4806 case 0x08: /* invd */
4807 case 0x0d: /* GrpP (prefetch) */
4808 case 0x18: /* Grp16 (prefetch/nop) */
4809 case 0x1f: /* nop */
4811 case 0x20: /* mov cr, reg */
4812 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
4814 case 0x21: /* mov from dr to reg */
4815 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
4817 case 0x40 ... 0x4f: /* cmov */
4818 ctxt->dst.val = ctxt->dst.orig_val = ctxt->src.val;
4819 if (!test_cc(ctxt->b, ctxt->eflags))
4820 ctxt->dst.type = OP_NONE; /* no writeback */
4822 case 0x80 ... 0x8f: /* jnz rel, etc*/
4823 if (test_cc(ctxt->b, ctxt->eflags))
4824 jmp_rel(ctxt, ctxt->src.val);
4826 case 0x90 ... 0x9f: /* setcc r/m8 */
4827 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
4829 case 0xae: /* clflush */
4831 case 0xb6 ... 0xb7: /* movzx */
4832 ctxt->dst.bytes = ctxt->op_bytes;
4833 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
4834 : (u16) ctxt->src.val;
4836 case 0xbe ... 0xbf: /* movsx */
4837 ctxt->dst.bytes = ctxt->op_bytes;
4838 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
4839 (s16) ctxt->src.val;
4841 case 0xc3: /* movnti */
4842 ctxt->dst.bytes = ctxt->op_bytes;
4843 ctxt->dst.val = (ctxt->op_bytes == 8) ? (u64) ctxt->src.val :
4844 (u32) ctxt->src.val;
4847 goto cannot_emulate;
4852 if (rc != X86EMUL_CONTINUE)
4858 return EMULATION_FAILED;
4861 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
4863 invalidate_registers(ctxt);
4866 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
4868 writeback_registers(ctxt);