1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
35 #define OpImplicit 1ull /* No generic decode */
36 #define OpReg 2ull /* Register */
37 #define OpMem 3ull /* Memory */
38 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
39 #define OpDI 5ull /* ES:DI/EDI/RDI */
40 #define OpMem64 6ull /* Memory, 64-bit */
41 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
42 #define OpDX 8ull /* DX register */
43 #define OpCL 9ull /* CL register (for shifts) */
44 #define OpImmByte 10ull /* 8-bit sign extended immediate */
45 #define OpOne 11ull /* Implied 1 */
46 #define OpImm 12ull /* Sign extended immediate */
48 #define OpBits 4 /* Width of operand field */
49 #define OpMask ((1ull << OpBits) - 1)
52 * Opcode effective-address decode tables.
53 * Note that we only emulate instructions that have at least one memory
54 * operand (excluding implicit stack references). We assume that stack
55 * references and instruction fetches will never occur in special memory
56 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
60 /* Operand sizes: 8-bit operands or specified/overridden size. */
61 #define ByteOp (1<<0) /* 8-bit operands. */
62 /* Destination operand type. */
64 #define ImplicitOps (OpImplicit << DstShift)
65 #define DstReg (OpReg << DstShift)
66 #define DstMem (OpMem << DstShift)
67 #define DstAcc (OpAcc << DstShift)
68 #define DstDI (OpDI << DstShift)
69 #define DstMem64 (OpMem64 << DstShift)
70 #define DstImmUByte (OpImmUByte << DstShift)
71 #define DstDX (OpDX << DstShift)
72 #define DstMask (OpMask << DstShift)
73 /* Source operand type. */
74 #define SrcNone (0<<5) /* No source operand. */
75 #define SrcReg (1<<5) /* Register operand. */
76 #define SrcMem (2<<5) /* Memory operand. */
77 #define SrcMem16 (3<<5) /* Memory operand (16-bit). */
78 #define SrcMem32 (4<<5) /* Memory operand (32-bit). */
79 #define SrcImm (5<<5) /* Immediate operand. */
80 #define SrcImmByte (6<<5) /* 8-bit sign-extended immediate operand. */
81 #define SrcOne (7<<5) /* Implied '1' */
82 #define SrcImmUByte (8<<5) /* 8-bit unsigned immediate operand. */
83 #define SrcImmU (9<<5) /* Immediate operand, unsigned */
84 #define SrcSI (0xa<<5) /* Source is in the DS:RSI */
85 #define SrcImmFAddr (0xb<<5) /* Source is immediate far address */
86 #define SrcMemFAddr (0xc<<5) /* Source is far address in memory */
87 #define SrcAcc (0xd<<5) /* Source Accumulator */
88 #define SrcImmU16 (0xe<<5) /* Immediate operand, unsigned, 16 bits */
89 #define SrcDX (0xf<<5) /* Source is in DX register */
90 #define SrcMask (0xf<<5)
92 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
93 #define String (1<<13) /* String instruction (rep capable) */
94 #define Stack (1<<14) /* Stack instruction (push/pop) */
95 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
96 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
97 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
98 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
99 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
100 #define Sse (1<<18) /* SSE Vector instruction */
101 /* Generic ModRM decode. */
102 #define ModRM (1<<19)
103 /* Destination is only written; never read. */
106 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
107 #define VendorSpecific (1<<22) /* Vendor specific instruction */
108 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
109 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
110 #define Undefined (1<<25) /* No Such Instruction */
111 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
112 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
114 /* Source 2 operand type */
115 #define Src2Shift (29)
116 #define Src2None (OpNone << Src2Shift)
117 #define Src2CL (OpCL << Src2Shift)
118 #define Src2ImmByte (OpImmByte << Src2Shift)
119 #define Src2One (OpOne << Src2Shift)
120 #define Src2Imm (OpImm << Src2Shift)
121 #define Src2Mask (OpMask << Src2Shift)
123 #define X2(x...) x, x
124 #define X3(x...) X2(x), x
125 #define X4(x...) X2(x), X2(x)
126 #define X5(x...) X4(x), x
127 #define X6(x...) X4(x), X2(x)
128 #define X7(x...) X4(x), X3(x)
129 #define X8(x...) X4(x), X4(x)
130 #define X16(x...) X8(x), X8(x)
136 int (*execute)(struct x86_emulate_ctxt *ctxt);
137 struct opcode *group;
138 struct group_dual *gdual;
139 struct gprefix *gprefix;
141 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
145 struct opcode mod012[8];
146 struct opcode mod3[8];
150 struct opcode pfx_no;
151 struct opcode pfx_66;
152 struct opcode pfx_f2;
153 struct opcode pfx_f3;
156 /* EFLAGS bit definitions. */
157 #define EFLG_ID (1<<21)
158 #define EFLG_VIP (1<<20)
159 #define EFLG_VIF (1<<19)
160 #define EFLG_AC (1<<18)
161 #define EFLG_VM (1<<17)
162 #define EFLG_RF (1<<16)
163 #define EFLG_IOPL (3<<12)
164 #define EFLG_NT (1<<14)
165 #define EFLG_OF (1<<11)
166 #define EFLG_DF (1<<10)
167 #define EFLG_IF (1<<9)
168 #define EFLG_TF (1<<8)
169 #define EFLG_SF (1<<7)
170 #define EFLG_ZF (1<<6)
171 #define EFLG_AF (1<<4)
172 #define EFLG_PF (1<<2)
173 #define EFLG_CF (1<<0)
175 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
176 #define EFLG_RESERVED_ONE_MASK 2
179 * Instruction emulation:
180 * Most instructions are emulated directly via a fragment of inline assembly
181 * code. This allows us to save/restore EFLAGS and thus very easily pick up
182 * any modified flags.
185 #if defined(CONFIG_X86_64)
186 #define _LO32 "k" /* force 32-bit operand */
187 #define _STK "%%rsp" /* stack pointer */
188 #elif defined(__i386__)
189 #define _LO32 "" /* force 32-bit operand */
190 #define _STK "%%esp" /* stack pointer */
194 * These EFLAGS bits are restored from saved value during emulation, and
195 * any changes are written back to the saved value after emulation.
197 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
199 /* Before executing instruction: restore necessary bits in EFLAGS. */
200 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
201 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
202 "movl %"_sav",%"_LO32 _tmp"; " \
205 "movl %"_msk",%"_LO32 _tmp"; " \
206 "andl %"_LO32 _tmp",("_STK"); " \
208 "notl %"_LO32 _tmp"; " \
209 "andl %"_LO32 _tmp",("_STK"); " \
210 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
212 "orl %"_LO32 _tmp",("_STK"); " \
216 /* After executing instruction: write-back necessary bits in EFLAGS. */
217 #define _POST_EFLAGS(_sav, _msk, _tmp) \
218 /* _sav |= EFLAGS & _msk; */ \
221 "andl %"_msk",%"_LO32 _tmp"; " \
222 "orl %"_LO32 _tmp",%"_sav"; "
230 #define ____emulate_2op(ctxt, _op, _x, _y, _suffix, _dsttype) \
232 __asm__ __volatile__ ( \
233 _PRE_EFLAGS("0", "4", "2") \
234 _op _suffix " %"_x"3,%1; " \
235 _POST_EFLAGS("0", "4", "2") \
236 : "=m" ((ctxt)->eflags), \
237 "+q" (*(_dsttype*)&(ctxt)->dst.val), \
239 : _y ((ctxt)->src.val), "i" (EFLAGS_MASK)); \
243 /* Raw emulation: instruction has two explicit operands. */
244 #define __emulate_2op_nobyte(ctxt,_op,_wx,_wy,_lx,_ly,_qx,_qy) \
246 unsigned long _tmp; \
248 switch ((ctxt)->dst.bytes) { \
250 ____emulate_2op(ctxt,_op,_wx,_wy,"w",u16); \
253 ____emulate_2op(ctxt,_op,_lx,_ly,"l",u32); \
256 ON64(____emulate_2op(ctxt,_op,_qx,_qy,"q",u64)); \
261 #define __emulate_2op(ctxt,_op,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
263 unsigned long _tmp; \
264 switch ((ctxt)->dst.bytes) { \
266 ____emulate_2op(ctxt,_op,_bx,_by,"b",u8); \
269 __emulate_2op_nobyte(ctxt, _op, \
270 _wx, _wy, _lx, _ly, _qx, _qy); \
275 /* Source operand is byte-sized and may be restricted to just %cl. */
276 #define emulate_2op_SrcB(ctxt, _op) \
277 __emulate_2op(ctxt, _op, "b", "c", "b", "c", "b", "c", "b", "c")
279 /* Source operand is byte, word, long or quad sized. */
280 #define emulate_2op_SrcV(ctxt, _op) \
281 __emulate_2op(ctxt, _op, "b", "q", "w", "r", _LO32, "r", "", "r")
283 /* Source operand is word, long or quad sized. */
284 #define emulate_2op_SrcV_nobyte(ctxt, _op) \
285 __emulate_2op_nobyte(ctxt, _op, "w", "r", _LO32, "r", "", "r")
287 /* Instruction has three operands and one operand is stored in ECX register */
288 #define __emulate_2op_cl(ctxt, _op, _suffix, _type) \
290 unsigned long _tmp; \
291 _type _clv = (ctxt)->src2.val; \
292 _type _srcv = (ctxt)->src.val; \
293 _type _dstv = (ctxt)->dst.val; \
295 __asm__ __volatile__ ( \
296 _PRE_EFLAGS("0", "5", "2") \
297 _op _suffix " %4,%1 \n" \
298 _POST_EFLAGS("0", "5", "2") \
299 : "=m" ((ctxt)->eflags), "+r" (_dstv), "=&r" (_tmp) \
300 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
303 (ctxt)->src2.val = (unsigned long) _clv; \
304 (ctxt)->src2.val = (unsigned long) _srcv; \
305 (ctxt)->dst.val = (unsigned long) _dstv; \
308 #define emulate_2op_cl(ctxt, _op) \
310 switch ((ctxt)->dst.bytes) { \
312 __emulate_2op_cl(ctxt, _op, "w", u16); \
315 __emulate_2op_cl(ctxt, _op, "l", u32); \
318 ON64(__emulate_2op_cl(ctxt, _op, "q", ulong)); \
323 #define __emulate_1op(ctxt, _op, _suffix) \
325 unsigned long _tmp; \
327 __asm__ __volatile__ ( \
328 _PRE_EFLAGS("0", "3", "2") \
329 _op _suffix " %1; " \
330 _POST_EFLAGS("0", "3", "2") \
331 : "=m" ((ctxt)->eflags), "+m" ((ctxt)->dst.val), \
333 : "i" (EFLAGS_MASK)); \
336 /* Instruction has only one explicit operand (no source operand). */
337 #define emulate_1op(ctxt, _op) \
339 switch ((ctxt)->dst.bytes) { \
340 case 1: __emulate_1op(ctxt, _op, "b"); break; \
341 case 2: __emulate_1op(ctxt, _op, "w"); break; \
342 case 4: __emulate_1op(ctxt, _op, "l"); break; \
343 case 8: ON64(__emulate_1op(ctxt, _op, "q")); break; \
347 #define __emulate_1op_rax_rdx(ctxt, _op, _suffix, _ex) \
349 unsigned long _tmp; \
350 ulong *rax = &(ctxt)->regs[VCPU_REGS_RAX]; \
351 ulong *rdx = &(ctxt)->regs[VCPU_REGS_RDX]; \
353 __asm__ __volatile__ ( \
354 _PRE_EFLAGS("0", "5", "1") \
356 _op _suffix " %6; " \
358 _POST_EFLAGS("0", "5", "1") \
359 ".pushsection .fixup,\"ax\" \n\t" \
360 "3: movb $1, %4 \n\t" \
363 _ASM_EXTABLE(1b, 3b) \
364 : "=m" ((ctxt)->eflags), "=&r" (_tmp), \
365 "+a" (*rax), "+d" (*rdx), "+qm"(_ex) \
366 : "i" (EFLAGS_MASK), "m" ((ctxt)->src.val), \
367 "a" (*rax), "d" (*rdx)); \
370 /* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
371 #define emulate_1op_rax_rdx(ctxt, _op, _ex) \
373 switch((ctxt)->src.bytes) { \
375 __emulate_1op_rax_rdx(ctxt, _op, "b", _ex); \
378 __emulate_1op_rax_rdx(ctxt, _op, "w", _ex); \
381 __emulate_1op_rax_rdx(ctxt, _op, "l", _ex); \
384 __emulate_1op_rax_rdx(ctxt, _op, "q", _ex)); \
389 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
390 enum x86_intercept intercept,
391 enum x86_intercept_stage stage)
393 struct x86_instruction_info info = {
394 .intercept = intercept,
395 .rep_prefix = ctxt->rep_prefix,
396 .modrm_mod = ctxt->modrm_mod,
397 .modrm_reg = ctxt->modrm_reg,
398 .modrm_rm = ctxt->modrm_rm,
399 .src_val = ctxt->src.val64,
400 .src_bytes = ctxt->src.bytes,
401 .dst_bytes = ctxt->dst.bytes,
402 .ad_bytes = ctxt->ad_bytes,
403 .next_rip = ctxt->eip,
406 return ctxt->ops->intercept(ctxt, &info, stage);
409 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
411 return (1UL << (ctxt->ad_bytes << 3)) - 1;
414 /* Access/update address held in a register, based on addressing mode. */
415 static inline unsigned long
416 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
418 if (ctxt->ad_bytes == sizeof(unsigned long))
421 return reg & ad_mask(ctxt);
424 static inline unsigned long
425 register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
427 return address_mask(ctxt, reg);
431 register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
433 if (ctxt->ad_bytes == sizeof(unsigned long))
436 *reg = (*reg & ~ad_mask(ctxt)) | ((*reg + inc) & ad_mask(ctxt));
439 static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
441 register_address_increment(ctxt, &ctxt->_eip, rel);
444 static u32 desc_limit_scaled(struct desc_struct *desc)
446 u32 limit = get_desc_limit(desc);
448 return desc->g ? (limit << 12) | 0xfff : limit;
451 static void set_seg_override(struct x86_emulate_ctxt *ctxt, int seg)
453 ctxt->has_seg_override = true;
454 ctxt->seg_override = seg;
457 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
459 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
462 return ctxt->ops->get_cached_segment_base(ctxt, seg);
465 static unsigned seg_override(struct x86_emulate_ctxt *ctxt)
467 if (!ctxt->has_seg_override)
470 return ctxt->seg_override;
473 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
474 u32 error, bool valid)
476 ctxt->exception.vector = vec;
477 ctxt->exception.error_code = error;
478 ctxt->exception.error_code_valid = valid;
479 return X86EMUL_PROPAGATE_FAULT;
482 static int emulate_db(struct x86_emulate_ctxt *ctxt)
484 return emulate_exception(ctxt, DB_VECTOR, 0, false);
487 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
489 return emulate_exception(ctxt, GP_VECTOR, err, true);
492 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
494 return emulate_exception(ctxt, SS_VECTOR, err, true);
497 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
499 return emulate_exception(ctxt, UD_VECTOR, 0, false);
502 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
504 return emulate_exception(ctxt, TS_VECTOR, err, true);
507 static int emulate_de(struct x86_emulate_ctxt *ctxt)
509 return emulate_exception(ctxt, DE_VECTOR, 0, false);
512 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
514 return emulate_exception(ctxt, NM_VECTOR, 0, false);
517 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
520 struct desc_struct desc;
522 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
526 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
531 struct desc_struct desc;
533 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
534 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
537 static int __linearize(struct x86_emulate_ctxt *ctxt,
538 struct segmented_address addr,
539 unsigned size, bool write, bool fetch,
542 struct desc_struct desc;
549 la = seg_base(ctxt, addr.seg) + addr.ea;
550 switch (ctxt->mode) {
551 case X86EMUL_MODE_REAL:
553 case X86EMUL_MODE_PROT64:
554 if (((signed long)la << 16) >> 16 != la)
555 return emulate_gp(ctxt, 0);
558 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
562 /* code segment or read-only data segment */
563 if (((desc.type & 8) || !(desc.type & 2)) && write)
565 /* unreadable code segment */
566 if (!fetch && (desc.type & 8) && !(desc.type & 2))
568 lim = desc_limit_scaled(&desc);
569 if ((desc.type & 8) || !(desc.type & 4)) {
570 /* expand-up segment */
571 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
574 /* exapand-down segment */
575 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
577 lim = desc.d ? 0xffffffff : 0xffff;
578 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
581 cpl = ctxt->ops->cpl(ctxt);
584 if (!(desc.type & 8)) {
588 } else if ((desc.type & 8) && !(desc.type & 4)) {
589 /* nonconforming code segment */
592 } else if ((desc.type & 8) && (desc.type & 4)) {
593 /* conforming code segment */
599 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
602 return X86EMUL_CONTINUE;
604 if (addr.seg == VCPU_SREG_SS)
605 return emulate_ss(ctxt, addr.seg);
607 return emulate_gp(ctxt, addr.seg);
610 static int linearize(struct x86_emulate_ctxt *ctxt,
611 struct segmented_address addr,
612 unsigned size, bool write,
615 return __linearize(ctxt, addr, size, write, false, linear);
619 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
620 struct segmented_address addr,
627 rc = linearize(ctxt, addr, size, false, &linear);
628 if (rc != X86EMUL_CONTINUE)
630 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
634 * Fetch the next byte of the instruction being emulated which is pointed to
635 * by ctxt->_eip, then increment ctxt->_eip.
637 * Also prefetch the remaining bytes of the instruction without crossing page
638 * boundary if they are not in fetch_cache yet.
640 static int do_insn_fetch_byte(struct x86_emulate_ctxt *ctxt, u8 *dest)
642 struct fetch_cache *fc = &ctxt->fetch;
646 if (ctxt->_eip == fc->end) {
647 unsigned long linear;
648 struct segmented_address addr = { .seg = VCPU_SREG_CS,
650 cur_size = fc->end - fc->start;
651 size = min(15UL - cur_size,
652 PAGE_SIZE - offset_in_page(ctxt->_eip));
653 rc = __linearize(ctxt, addr, size, false, true, &linear);
654 if (unlikely(rc != X86EMUL_CONTINUE))
656 rc = ctxt->ops->fetch(ctxt, linear, fc->data + cur_size,
657 size, &ctxt->exception);
658 if (unlikely(rc != X86EMUL_CONTINUE))
662 *dest = fc->data[ctxt->_eip - fc->start];
664 return X86EMUL_CONTINUE;
667 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
668 void *dest, unsigned size)
672 /* x86 instructions are limited to 15 bytes. */
673 if (unlikely(ctxt->_eip + size - ctxt->eip > 15))
674 return X86EMUL_UNHANDLEABLE;
676 rc = do_insn_fetch_byte(ctxt, dest++);
677 if (rc != X86EMUL_CONTINUE)
680 return X86EMUL_CONTINUE;
683 /* Fetch next part of the instruction being emulated. */
684 #define insn_fetch(_type, _ctxt) \
685 ({ unsigned long _x; \
686 rc = do_insn_fetch(_ctxt, &_x, sizeof(_type)); \
687 if (rc != X86EMUL_CONTINUE) \
692 #define insn_fetch_arr(_arr, _size, _ctxt) \
693 ({ rc = do_insn_fetch(_ctxt, _arr, (_size)); \
694 if (rc != X86EMUL_CONTINUE) \
699 * Given the 'reg' portion of a ModRM byte, and a register block, return a
700 * pointer into the block that addresses the relevant register.
701 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
703 static void *decode_register(u8 modrm_reg, unsigned long *regs,
708 p = ®s[modrm_reg];
709 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
710 p = (unsigned char *)®s[modrm_reg & 3] + 1;
714 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
715 struct segmented_address addr,
716 u16 *size, unsigned long *address, int op_bytes)
723 rc = segmented_read_std(ctxt, addr, size, 2);
724 if (rc != X86EMUL_CONTINUE)
727 rc = segmented_read_std(ctxt, addr, address, op_bytes);
731 static int test_cc(unsigned int condition, unsigned int flags)
735 switch ((condition & 15) >> 1) {
737 rc |= (flags & EFLG_OF);
739 case 1: /* b/c/nae */
740 rc |= (flags & EFLG_CF);
743 rc |= (flags & EFLG_ZF);
746 rc |= (flags & (EFLG_CF|EFLG_ZF));
749 rc |= (flags & EFLG_SF);
752 rc |= (flags & EFLG_PF);
755 rc |= (flags & EFLG_ZF);
758 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
762 /* Odd condition identifiers (lsb == 1) have inverted sense. */
763 return (!!rc ^ (condition & 1));
766 static void fetch_register_operand(struct operand *op)
770 op->val = *(u8 *)op->addr.reg;
773 op->val = *(u16 *)op->addr.reg;
776 op->val = *(u32 *)op->addr.reg;
779 op->val = *(u64 *)op->addr.reg;
784 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
786 ctxt->ops->get_fpu(ctxt);
788 case 0: asm("movdqu %%xmm0, %0" : "=m"(*data)); break;
789 case 1: asm("movdqu %%xmm1, %0" : "=m"(*data)); break;
790 case 2: asm("movdqu %%xmm2, %0" : "=m"(*data)); break;
791 case 3: asm("movdqu %%xmm3, %0" : "=m"(*data)); break;
792 case 4: asm("movdqu %%xmm4, %0" : "=m"(*data)); break;
793 case 5: asm("movdqu %%xmm5, %0" : "=m"(*data)); break;
794 case 6: asm("movdqu %%xmm6, %0" : "=m"(*data)); break;
795 case 7: asm("movdqu %%xmm7, %0" : "=m"(*data)); break;
797 case 8: asm("movdqu %%xmm8, %0" : "=m"(*data)); break;
798 case 9: asm("movdqu %%xmm9, %0" : "=m"(*data)); break;
799 case 10: asm("movdqu %%xmm10, %0" : "=m"(*data)); break;
800 case 11: asm("movdqu %%xmm11, %0" : "=m"(*data)); break;
801 case 12: asm("movdqu %%xmm12, %0" : "=m"(*data)); break;
802 case 13: asm("movdqu %%xmm13, %0" : "=m"(*data)); break;
803 case 14: asm("movdqu %%xmm14, %0" : "=m"(*data)); break;
804 case 15: asm("movdqu %%xmm15, %0" : "=m"(*data)); break;
808 ctxt->ops->put_fpu(ctxt);
811 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
814 ctxt->ops->get_fpu(ctxt);
816 case 0: asm("movdqu %0, %%xmm0" : : "m"(*data)); break;
817 case 1: asm("movdqu %0, %%xmm1" : : "m"(*data)); break;
818 case 2: asm("movdqu %0, %%xmm2" : : "m"(*data)); break;
819 case 3: asm("movdqu %0, %%xmm3" : : "m"(*data)); break;
820 case 4: asm("movdqu %0, %%xmm4" : : "m"(*data)); break;
821 case 5: asm("movdqu %0, %%xmm5" : : "m"(*data)); break;
822 case 6: asm("movdqu %0, %%xmm6" : : "m"(*data)); break;
823 case 7: asm("movdqu %0, %%xmm7" : : "m"(*data)); break;
825 case 8: asm("movdqu %0, %%xmm8" : : "m"(*data)); break;
826 case 9: asm("movdqu %0, %%xmm9" : : "m"(*data)); break;
827 case 10: asm("movdqu %0, %%xmm10" : : "m"(*data)); break;
828 case 11: asm("movdqu %0, %%xmm11" : : "m"(*data)); break;
829 case 12: asm("movdqu %0, %%xmm12" : : "m"(*data)); break;
830 case 13: asm("movdqu %0, %%xmm13" : : "m"(*data)); break;
831 case 14: asm("movdqu %0, %%xmm14" : : "m"(*data)); break;
832 case 15: asm("movdqu %0, %%xmm15" : : "m"(*data)); break;
836 ctxt->ops->put_fpu(ctxt);
839 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
843 unsigned reg = ctxt->modrm_reg;
844 int highbyte_regs = ctxt->rex_prefix == 0;
846 if (!(ctxt->d & ModRM))
847 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
853 read_sse_reg(ctxt, &op->vec_val, reg);
858 if ((ctxt->d & ByteOp) && !inhibit_bytereg) {
859 op->addr.reg = decode_register(reg, ctxt->regs, highbyte_regs);
862 op->addr.reg = decode_register(reg, ctxt->regs, 0);
863 op->bytes = ctxt->op_bytes;
865 fetch_register_operand(op);
866 op->orig_val = op->val;
869 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
873 int index_reg = 0, base_reg = 0, scale;
874 int rc = X86EMUL_CONTINUE;
877 if (ctxt->rex_prefix) {
878 ctxt->modrm_reg = (ctxt->rex_prefix & 4) << 1; /* REX.R */
879 index_reg = (ctxt->rex_prefix & 2) << 2; /* REX.X */
880 ctxt->modrm_rm = base_reg = (ctxt->rex_prefix & 1) << 3; /* REG.B */
883 ctxt->modrm = insn_fetch(u8, ctxt);
884 ctxt->modrm_mod |= (ctxt->modrm & 0xc0) >> 6;
885 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
886 ctxt->modrm_rm |= (ctxt->modrm & 0x07);
887 ctxt->modrm_seg = VCPU_SREG_DS;
889 if (ctxt->modrm_mod == 3) {
891 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
892 op->addr.reg = decode_register(ctxt->modrm_rm,
893 ctxt->regs, ctxt->d & ByteOp);
897 op->addr.xmm = ctxt->modrm_rm;
898 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
901 fetch_register_operand(op);
907 if (ctxt->ad_bytes == 2) {
908 unsigned bx = ctxt->regs[VCPU_REGS_RBX];
909 unsigned bp = ctxt->regs[VCPU_REGS_RBP];
910 unsigned si = ctxt->regs[VCPU_REGS_RSI];
911 unsigned di = ctxt->regs[VCPU_REGS_RDI];
913 /* 16-bit ModR/M decode. */
914 switch (ctxt->modrm_mod) {
916 if (ctxt->modrm_rm == 6)
917 modrm_ea += insn_fetch(u16, ctxt);
920 modrm_ea += insn_fetch(s8, ctxt);
923 modrm_ea += insn_fetch(u16, ctxt);
926 switch (ctxt->modrm_rm) {
946 if (ctxt->modrm_mod != 0)
953 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
954 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
955 ctxt->modrm_seg = VCPU_SREG_SS;
956 modrm_ea = (u16)modrm_ea;
958 /* 32/64-bit ModR/M decode. */
959 if ((ctxt->modrm_rm & 7) == 4) {
960 sib = insn_fetch(u8, ctxt);
961 index_reg |= (sib >> 3) & 7;
965 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
966 modrm_ea += insn_fetch(s32, ctxt);
968 modrm_ea += ctxt->regs[base_reg];
970 modrm_ea += ctxt->regs[index_reg] << scale;
971 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
972 if (ctxt->mode == X86EMUL_MODE_PROT64)
973 ctxt->rip_relative = 1;
975 modrm_ea += ctxt->regs[ctxt->modrm_rm];
976 switch (ctxt->modrm_mod) {
978 if (ctxt->modrm_rm == 5)
979 modrm_ea += insn_fetch(s32, ctxt);
982 modrm_ea += insn_fetch(s8, ctxt);
985 modrm_ea += insn_fetch(s32, ctxt);
989 op->addr.mem.ea = modrm_ea;
994 static int decode_abs(struct x86_emulate_ctxt *ctxt,
997 int rc = X86EMUL_CONTINUE;
1000 switch (ctxt->ad_bytes) {
1002 op->addr.mem.ea = insn_fetch(u16, ctxt);
1005 op->addr.mem.ea = insn_fetch(u32, ctxt);
1008 op->addr.mem.ea = insn_fetch(u64, ctxt);
1015 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1019 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1020 mask = ~(ctxt->dst.bytes * 8 - 1);
1022 if (ctxt->src.bytes == 2)
1023 sv = (s16)ctxt->src.val & (s16)mask;
1024 else if (ctxt->src.bytes == 4)
1025 sv = (s32)ctxt->src.val & (s32)mask;
1027 ctxt->dst.addr.mem.ea += (sv >> 3);
1030 /* only subword offset */
1031 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1034 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1035 unsigned long addr, void *dest, unsigned size)
1038 struct read_cache *mc = &ctxt->mem_read;
1041 int n = min(size, 8u);
1043 if (mc->pos < mc->end)
1046 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, n,
1048 if (rc != X86EMUL_CONTINUE)
1053 memcpy(dest, mc->data + mc->pos, n);
1058 return X86EMUL_CONTINUE;
1061 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1062 struct segmented_address addr,
1069 rc = linearize(ctxt, addr, size, false, &linear);
1070 if (rc != X86EMUL_CONTINUE)
1072 return read_emulated(ctxt, linear, data, size);
1075 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1076 struct segmented_address addr,
1083 rc = linearize(ctxt, addr, size, true, &linear);
1084 if (rc != X86EMUL_CONTINUE)
1086 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1090 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1091 struct segmented_address addr,
1092 const void *orig_data, const void *data,
1098 rc = linearize(ctxt, addr, size, true, &linear);
1099 if (rc != X86EMUL_CONTINUE)
1101 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1102 size, &ctxt->exception);
1105 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1106 unsigned int size, unsigned short port,
1109 struct read_cache *rc = &ctxt->io_read;
1111 if (rc->pos == rc->end) { /* refill pio read ahead */
1112 unsigned int in_page, n;
1113 unsigned int count = ctxt->rep_prefix ?
1114 address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) : 1;
1115 in_page = (ctxt->eflags & EFLG_DF) ?
1116 offset_in_page(ctxt->regs[VCPU_REGS_RDI]) :
1117 PAGE_SIZE - offset_in_page(ctxt->regs[VCPU_REGS_RDI]);
1118 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1122 rc->pos = rc->end = 0;
1123 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1128 memcpy(dest, rc->data + rc->pos, size);
1133 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1134 u16 selector, struct desc_ptr *dt)
1136 struct x86_emulate_ops *ops = ctxt->ops;
1138 if (selector & 1 << 2) {
1139 struct desc_struct desc;
1142 memset (dt, 0, sizeof *dt);
1143 if (!ops->get_segment(ctxt, &sel, &desc, NULL, VCPU_SREG_LDTR))
1146 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1147 dt->address = get_desc_base(&desc);
1149 ops->get_gdt(ctxt, dt);
1152 /* allowed just for 8 bytes segments */
1153 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1154 u16 selector, struct desc_struct *desc)
1157 u16 index = selector >> 3;
1160 get_descriptor_table_ptr(ctxt, selector, &dt);
1162 if (dt.size < index * 8 + 7)
1163 return emulate_gp(ctxt, selector & 0xfffc);
1165 addr = dt.address + index * 8;
1166 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1170 /* allowed just for 8 bytes segments */
1171 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1172 u16 selector, struct desc_struct *desc)
1175 u16 index = selector >> 3;
1178 get_descriptor_table_ptr(ctxt, selector, &dt);
1180 if (dt.size < index * 8 + 7)
1181 return emulate_gp(ctxt, selector & 0xfffc);
1183 addr = dt.address + index * 8;
1184 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1188 /* Does not support long mode */
1189 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1190 u16 selector, int seg)
1192 struct desc_struct seg_desc;
1194 unsigned err_vec = GP_VECTOR;
1196 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1199 memset(&seg_desc, 0, sizeof seg_desc);
1201 if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
1202 || ctxt->mode == X86EMUL_MODE_REAL) {
1203 /* set real mode segment descriptor */
1204 set_desc_base(&seg_desc, selector << 4);
1205 set_desc_limit(&seg_desc, 0xffff);
1212 /* NULL selector is not valid for TR, CS and SS */
1213 if ((seg == VCPU_SREG_CS || seg == VCPU_SREG_SS || seg == VCPU_SREG_TR)
1217 /* TR should be in GDT only */
1218 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1221 if (null_selector) /* for NULL selector skip all following checks */
1224 ret = read_segment_descriptor(ctxt, selector, &seg_desc);
1225 if (ret != X86EMUL_CONTINUE)
1228 err_code = selector & 0xfffc;
1229 err_vec = GP_VECTOR;
1231 /* can't load system descriptor into segment selecor */
1232 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1236 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1242 cpl = ctxt->ops->cpl(ctxt);
1247 * segment is not a writable data segment or segment
1248 * selector's RPL != CPL or segment selector's RPL != CPL
1250 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1254 if (!(seg_desc.type & 8))
1257 if (seg_desc.type & 4) {
1263 if (rpl > cpl || dpl != cpl)
1266 /* CS(RPL) <- CPL */
1267 selector = (selector & 0xfffc) | cpl;
1270 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1273 case VCPU_SREG_LDTR:
1274 if (seg_desc.s || seg_desc.type != 2)
1277 default: /* DS, ES, FS, or GS */
1279 * segment is not a data or readable code segment or
1280 * ((segment is a data or nonconforming code segment)
1281 * and (both RPL and CPL > DPL))
1283 if ((seg_desc.type & 0xa) == 0x8 ||
1284 (((seg_desc.type & 0xc) != 0xc) &&
1285 (rpl > dpl && cpl > dpl)))
1291 /* mark segment as accessed */
1293 ret = write_segment_descriptor(ctxt, selector, &seg_desc);
1294 if (ret != X86EMUL_CONTINUE)
1298 ctxt->ops->set_segment(ctxt, selector, &seg_desc, 0, seg);
1299 return X86EMUL_CONTINUE;
1301 emulate_exception(ctxt, err_vec, err_code, true);
1302 return X86EMUL_PROPAGATE_FAULT;
1305 static void write_register_operand(struct operand *op)
1307 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1308 switch (op->bytes) {
1310 *(u8 *)op->addr.reg = (u8)op->val;
1313 *(u16 *)op->addr.reg = (u16)op->val;
1316 *op->addr.reg = (u32)op->val;
1317 break; /* 64b: zero-extend */
1319 *op->addr.reg = op->val;
1324 static int writeback(struct x86_emulate_ctxt *ctxt)
1328 switch (ctxt->dst.type) {
1330 write_register_operand(&ctxt->dst);
1333 if (ctxt->lock_prefix)
1334 rc = segmented_cmpxchg(ctxt,
1336 &ctxt->dst.orig_val,
1340 rc = segmented_write(ctxt,
1344 if (rc != X86EMUL_CONTINUE)
1348 write_sse_reg(ctxt, &ctxt->dst.vec_val, ctxt->dst.addr.xmm);
1356 return X86EMUL_CONTINUE;
1359 static int em_push(struct x86_emulate_ctxt *ctxt)
1361 struct segmented_address addr;
1363 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], -ctxt->op_bytes);
1364 addr.ea = register_address(ctxt, ctxt->regs[VCPU_REGS_RSP]);
1365 addr.seg = VCPU_SREG_SS;
1367 /* Disable writeback. */
1368 ctxt->dst.type = OP_NONE;
1369 return segmented_write(ctxt, addr, &ctxt->src.val, ctxt->op_bytes);
1372 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1373 void *dest, int len)
1376 struct segmented_address addr;
1378 addr.ea = register_address(ctxt, ctxt->regs[VCPU_REGS_RSP]);
1379 addr.seg = VCPU_SREG_SS;
1380 rc = segmented_read(ctxt, addr, dest, len);
1381 if (rc != X86EMUL_CONTINUE)
1384 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], len);
1388 static int em_pop(struct x86_emulate_ctxt *ctxt)
1390 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1393 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1394 void *dest, int len)
1397 unsigned long val, change_mask;
1398 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1399 int cpl = ctxt->ops->cpl(ctxt);
1401 rc = emulate_pop(ctxt, &val, len);
1402 if (rc != X86EMUL_CONTINUE)
1405 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1406 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1408 switch(ctxt->mode) {
1409 case X86EMUL_MODE_PROT64:
1410 case X86EMUL_MODE_PROT32:
1411 case X86EMUL_MODE_PROT16:
1413 change_mask |= EFLG_IOPL;
1415 change_mask |= EFLG_IF;
1417 case X86EMUL_MODE_VM86:
1419 return emulate_gp(ctxt, 0);
1420 change_mask |= EFLG_IF;
1422 default: /* real mode */
1423 change_mask |= (EFLG_IOPL | EFLG_IF);
1427 *(unsigned long *)dest =
1428 (ctxt->eflags & ~change_mask) | (val & change_mask);
1433 static int em_popf(struct x86_emulate_ctxt *ctxt)
1435 ctxt->dst.type = OP_REG;
1436 ctxt->dst.addr.reg = &ctxt->eflags;
1437 ctxt->dst.bytes = ctxt->op_bytes;
1438 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1441 static int emulate_push_sreg(struct x86_emulate_ctxt *ctxt, int seg)
1443 ctxt->src.val = get_segment_selector(ctxt, seg);
1445 return em_push(ctxt);
1448 static int emulate_pop_sreg(struct x86_emulate_ctxt *ctxt, int seg)
1450 unsigned long selector;
1453 rc = emulate_pop(ctxt, &selector, ctxt->op_bytes);
1454 if (rc != X86EMUL_CONTINUE)
1457 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1461 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1463 unsigned long old_esp = ctxt->regs[VCPU_REGS_RSP];
1464 int rc = X86EMUL_CONTINUE;
1465 int reg = VCPU_REGS_RAX;
1467 while (reg <= VCPU_REGS_RDI) {
1468 (reg == VCPU_REGS_RSP) ?
1469 (ctxt->src.val = old_esp) : (ctxt->src.val = ctxt->regs[reg]);
1472 if (rc != X86EMUL_CONTINUE)
1481 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1483 ctxt->src.val = (unsigned long)ctxt->eflags;
1484 return em_push(ctxt);
1487 static int em_popa(struct x86_emulate_ctxt *ctxt)
1489 int rc = X86EMUL_CONTINUE;
1490 int reg = VCPU_REGS_RDI;
1492 while (reg >= VCPU_REGS_RAX) {
1493 if (reg == VCPU_REGS_RSP) {
1494 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP],
1499 rc = emulate_pop(ctxt, &ctxt->regs[reg], ctxt->op_bytes);
1500 if (rc != X86EMUL_CONTINUE)
1507 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1509 struct x86_emulate_ops *ops = ctxt->ops;
1516 /* TODO: Add limit checks */
1517 ctxt->src.val = ctxt->eflags;
1519 if (rc != X86EMUL_CONTINUE)
1522 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1524 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1526 if (rc != X86EMUL_CONTINUE)
1529 ctxt->src.val = ctxt->_eip;
1531 if (rc != X86EMUL_CONTINUE)
1534 ops->get_idt(ctxt, &dt);
1536 eip_addr = dt.address + (irq << 2);
1537 cs_addr = dt.address + (irq << 2) + 2;
1539 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1540 if (rc != X86EMUL_CONTINUE)
1543 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1544 if (rc != X86EMUL_CONTINUE)
1547 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1548 if (rc != X86EMUL_CONTINUE)
1556 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
1558 switch(ctxt->mode) {
1559 case X86EMUL_MODE_REAL:
1560 return emulate_int_real(ctxt, irq);
1561 case X86EMUL_MODE_VM86:
1562 case X86EMUL_MODE_PROT16:
1563 case X86EMUL_MODE_PROT32:
1564 case X86EMUL_MODE_PROT64:
1566 /* Protected mode interrupts unimplemented yet */
1567 return X86EMUL_UNHANDLEABLE;
1571 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
1573 int rc = X86EMUL_CONTINUE;
1574 unsigned long temp_eip = 0;
1575 unsigned long temp_eflags = 0;
1576 unsigned long cs = 0;
1577 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1578 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1579 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1580 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1582 /* TODO: Add stack limit check */
1584 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
1586 if (rc != X86EMUL_CONTINUE)
1589 if (temp_eip & ~0xffff)
1590 return emulate_gp(ctxt, 0);
1592 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1594 if (rc != X86EMUL_CONTINUE)
1597 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
1599 if (rc != X86EMUL_CONTINUE)
1602 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1604 if (rc != X86EMUL_CONTINUE)
1607 ctxt->_eip = temp_eip;
1610 if (ctxt->op_bytes == 4)
1611 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1612 else if (ctxt->op_bytes == 2) {
1613 ctxt->eflags &= ~0xffff;
1614 ctxt->eflags |= temp_eflags;
1617 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1618 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1623 static int em_iret(struct x86_emulate_ctxt *ctxt)
1625 switch(ctxt->mode) {
1626 case X86EMUL_MODE_REAL:
1627 return emulate_iret_real(ctxt);
1628 case X86EMUL_MODE_VM86:
1629 case X86EMUL_MODE_PROT16:
1630 case X86EMUL_MODE_PROT32:
1631 case X86EMUL_MODE_PROT64:
1633 /* iret from protected mode unimplemented yet */
1634 return X86EMUL_UNHANDLEABLE;
1638 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
1643 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1645 rc = load_segment_descriptor(ctxt, sel, VCPU_SREG_CS);
1646 if (rc != X86EMUL_CONTINUE)
1650 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
1651 return X86EMUL_CONTINUE;
1654 static int em_grp1a(struct x86_emulate_ctxt *ctxt)
1656 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->dst.bytes);
1659 static int em_grp2(struct x86_emulate_ctxt *ctxt)
1661 switch (ctxt->modrm_reg) {
1663 emulate_2op_SrcB(ctxt, "rol");
1666 emulate_2op_SrcB(ctxt, "ror");
1669 emulate_2op_SrcB(ctxt, "rcl");
1672 emulate_2op_SrcB(ctxt, "rcr");
1674 case 4: /* sal/shl */
1675 case 6: /* sal/shl */
1676 emulate_2op_SrcB(ctxt, "sal");
1679 emulate_2op_SrcB(ctxt, "shr");
1682 emulate_2op_SrcB(ctxt, "sar");
1685 return X86EMUL_CONTINUE;
1688 static int em_not(struct x86_emulate_ctxt *ctxt)
1690 ctxt->dst.val = ~ctxt->dst.val;
1691 return X86EMUL_CONTINUE;
1694 static int em_neg(struct x86_emulate_ctxt *ctxt)
1696 emulate_1op(ctxt, "neg");
1697 return X86EMUL_CONTINUE;
1700 static int em_mul_ex(struct x86_emulate_ctxt *ctxt)
1704 emulate_1op_rax_rdx(ctxt, "mul", ex);
1705 return X86EMUL_CONTINUE;
1708 static int em_imul_ex(struct x86_emulate_ctxt *ctxt)
1712 emulate_1op_rax_rdx(ctxt, "imul", ex);
1713 return X86EMUL_CONTINUE;
1716 static int em_div_ex(struct x86_emulate_ctxt *ctxt)
1720 emulate_1op_rax_rdx(ctxt, "div", de);
1722 return emulate_de(ctxt);
1723 return X86EMUL_CONTINUE;
1726 static int em_idiv_ex(struct x86_emulate_ctxt *ctxt)
1730 emulate_1op_rax_rdx(ctxt, "idiv", de);
1732 return emulate_de(ctxt);
1733 return X86EMUL_CONTINUE;
1736 static int em_grp45(struct x86_emulate_ctxt *ctxt)
1738 int rc = X86EMUL_CONTINUE;
1740 switch (ctxt->modrm_reg) {
1742 emulate_1op(ctxt, "inc");
1745 emulate_1op(ctxt, "dec");
1747 case 2: /* call near abs */ {
1749 old_eip = ctxt->_eip;
1750 ctxt->_eip = ctxt->src.val;
1751 ctxt->src.val = old_eip;
1755 case 4: /* jmp abs */
1756 ctxt->_eip = ctxt->src.val;
1758 case 5: /* jmp far */
1759 rc = em_jmp_far(ctxt);
1768 static int em_grp9(struct x86_emulate_ctxt *ctxt)
1770 u64 old = ctxt->dst.orig_val64;
1772 if (((u32) (old >> 0) != (u32) ctxt->regs[VCPU_REGS_RAX]) ||
1773 ((u32) (old >> 32) != (u32) ctxt->regs[VCPU_REGS_RDX])) {
1774 ctxt->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
1775 ctxt->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1776 ctxt->eflags &= ~EFLG_ZF;
1778 ctxt->dst.val64 = ((u64)ctxt->regs[VCPU_REGS_RCX] << 32) |
1779 (u32) ctxt->regs[VCPU_REGS_RBX];
1781 ctxt->eflags |= EFLG_ZF;
1783 return X86EMUL_CONTINUE;
1786 static int em_ret(struct x86_emulate_ctxt *ctxt)
1788 ctxt->dst.type = OP_REG;
1789 ctxt->dst.addr.reg = &ctxt->_eip;
1790 ctxt->dst.bytes = ctxt->op_bytes;
1791 return em_pop(ctxt);
1794 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
1799 rc = emulate_pop(ctxt, &ctxt->_eip, ctxt->op_bytes);
1800 if (rc != X86EMUL_CONTINUE)
1802 if (ctxt->op_bytes == 4)
1803 ctxt->_eip = (u32)ctxt->_eip;
1804 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1805 if (rc != X86EMUL_CONTINUE)
1807 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1811 static int emulate_load_segment(struct x86_emulate_ctxt *ctxt, int seg)
1816 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1818 rc = load_segment_descriptor(ctxt, sel, seg);
1819 if (rc != X86EMUL_CONTINUE)
1822 ctxt->dst.val = ctxt->src.val;
1827 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1828 struct desc_struct *cs, struct desc_struct *ss)
1832 memset(cs, 0, sizeof(struct desc_struct));
1833 ctxt->ops->get_segment(ctxt, &selector, cs, NULL, VCPU_SREG_CS);
1834 memset(ss, 0, sizeof(struct desc_struct));
1836 cs->l = 0; /* will be adjusted later */
1837 set_desc_base(cs, 0); /* flat segment */
1838 cs->g = 1; /* 4kb granularity */
1839 set_desc_limit(cs, 0xfffff); /* 4GB limit */
1840 cs->type = 0x0b; /* Read, Execute, Accessed */
1842 cs->dpl = 0; /* will be adjusted later */
1846 set_desc_base(ss, 0); /* flat segment */
1847 set_desc_limit(ss, 0xfffff); /* 4GB limit */
1848 ss->g = 1; /* 4kb granularity */
1850 ss->type = 0x03; /* Read/Write, Accessed */
1851 ss->d = 1; /* 32bit stack segment */
1856 static int em_syscall(struct x86_emulate_ctxt *ctxt)
1858 struct x86_emulate_ops *ops = ctxt->ops;
1859 struct desc_struct cs, ss;
1864 /* syscall is not available in real mode */
1865 if (ctxt->mode == X86EMUL_MODE_REAL ||
1866 ctxt->mode == X86EMUL_MODE_VM86)
1867 return emulate_ud(ctxt);
1869 ops->get_msr(ctxt, MSR_EFER, &efer);
1870 setup_syscalls_segments(ctxt, &cs, &ss);
1872 ops->get_msr(ctxt, MSR_STAR, &msr_data);
1874 cs_sel = (u16)(msr_data & 0xfffc);
1875 ss_sel = (u16)(msr_data + 8);
1877 if (efer & EFER_LMA) {
1881 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
1882 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
1884 ctxt->regs[VCPU_REGS_RCX] = ctxt->_eip;
1885 if (efer & EFER_LMA) {
1886 #ifdef CONFIG_X86_64
1887 ctxt->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
1890 ctxt->mode == X86EMUL_MODE_PROT64 ?
1891 MSR_LSTAR : MSR_CSTAR, &msr_data);
1892 ctxt->_eip = msr_data;
1894 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
1895 ctxt->eflags &= ~(msr_data | EFLG_RF);
1899 ops->get_msr(ctxt, MSR_STAR, &msr_data);
1900 ctxt->_eip = (u32)msr_data;
1902 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1905 return X86EMUL_CONTINUE;
1908 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
1910 struct x86_emulate_ops *ops = ctxt->ops;
1911 struct desc_struct cs, ss;
1916 ops->get_msr(ctxt, MSR_EFER, &efer);
1917 /* inject #GP if in real mode */
1918 if (ctxt->mode == X86EMUL_MODE_REAL)
1919 return emulate_gp(ctxt, 0);
1921 /* XXX sysenter/sysexit have not been tested in 64bit mode.
1922 * Therefore, we inject an #UD.
1924 if (ctxt->mode == X86EMUL_MODE_PROT64)
1925 return emulate_ud(ctxt);
1927 setup_syscalls_segments(ctxt, &cs, &ss);
1929 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
1930 switch (ctxt->mode) {
1931 case X86EMUL_MODE_PROT32:
1932 if ((msr_data & 0xfffc) == 0x0)
1933 return emulate_gp(ctxt, 0);
1935 case X86EMUL_MODE_PROT64:
1936 if (msr_data == 0x0)
1937 return emulate_gp(ctxt, 0);
1941 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1942 cs_sel = (u16)msr_data;
1943 cs_sel &= ~SELECTOR_RPL_MASK;
1944 ss_sel = cs_sel + 8;
1945 ss_sel &= ~SELECTOR_RPL_MASK;
1946 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
1951 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
1952 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
1954 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
1955 ctxt->_eip = msr_data;
1957 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
1958 ctxt->regs[VCPU_REGS_RSP] = msr_data;
1960 return X86EMUL_CONTINUE;
1963 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
1965 struct x86_emulate_ops *ops = ctxt->ops;
1966 struct desc_struct cs, ss;
1969 u16 cs_sel = 0, ss_sel = 0;
1971 /* inject #GP if in real mode or Virtual 8086 mode */
1972 if (ctxt->mode == X86EMUL_MODE_REAL ||
1973 ctxt->mode == X86EMUL_MODE_VM86)
1974 return emulate_gp(ctxt, 0);
1976 setup_syscalls_segments(ctxt, &cs, &ss);
1978 if ((ctxt->rex_prefix & 0x8) != 0x0)
1979 usermode = X86EMUL_MODE_PROT64;
1981 usermode = X86EMUL_MODE_PROT32;
1985 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
1987 case X86EMUL_MODE_PROT32:
1988 cs_sel = (u16)(msr_data + 16);
1989 if ((msr_data & 0xfffc) == 0x0)
1990 return emulate_gp(ctxt, 0);
1991 ss_sel = (u16)(msr_data + 24);
1993 case X86EMUL_MODE_PROT64:
1994 cs_sel = (u16)(msr_data + 32);
1995 if (msr_data == 0x0)
1996 return emulate_gp(ctxt, 0);
1997 ss_sel = cs_sel + 8;
2002 cs_sel |= SELECTOR_RPL_MASK;
2003 ss_sel |= SELECTOR_RPL_MASK;
2005 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2006 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2008 ctxt->_eip = ctxt->regs[VCPU_REGS_RDX];
2009 ctxt->regs[VCPU_REGS_RSP] = ctxt->regs[VCPU_REGS_RCX];
2011 return X86EMUL_CONTINUE;
2014 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2017 if (ctxt->mode == X86EMUL_MODE_REAL)
2019 if (ctxt->mode == X86EMUL_MODE_VM86)
2021 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2022 return ctxt->ops->cpl(ctxt) > iopl;
2025 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2028 struct x86_emulate_ops *ops = ctxt->ops;
2029 struct desc_struct tr_seg;
2032 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2033 unsigned mask = (1 << len) - 1;
2036 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2039 if (desc_limit_scaled(&tr_seg) < 103)
2041 base = get_desc_base(&tr_seg);
2042 #ifdef CONFIG_X86_64
2043 base |= ((u64)base3) << 32;
2045 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2046 if (r != X86EMUL_CONTINUE)
2048 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2050 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2051 if (r != X86EMUL_CONTINUE)
2053 if ((perm >> bit_idx) & mask)
2058 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2064 if (emulator_bad_iopl(ctxt))
2065 if (!emulator_io_port_access_allowed(ctxt, port, len))
2068 ctxt->perm_ok = true;
2073 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2074 struct tss_segment_16 *tss)
2076 tss->ip = ctxt->_eip;
2077 tss->flag = ctxt->eflags;
2078 tss->ax = ctxt->regs[VCPU_REGS_RAX];
2079 tss->cx = ctxt->regs[VCPU_REGS_RCX];
2080 tss->dx = ctxt->regs[VCPU_REGS_RDX];
2081 tss->bx = ctxt->regs[VCPU_REGS_RBX];
2082 tss->sp = ctxt->regs[VCPU_REGS_RSP];
2083 tss->bp = ctxt->regs[VCPU_REGS_RBP];
2084 tss->si = ctxt->regs[VCPU_REGS_RSI];
2085 tss->di = ctxt->regs[VCPU_REGS_RDI];
2087 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2088 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2089 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2090 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2091 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2094 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2095 struct tss_segment_16 *tss)
2099 ctxt->_eip = tss->ip;
2100 ctxt->eflags = tss->flag | 2;
2101 ctxt->regs[VCPU_REGS_RAX] = tss->ax;
2102 ctxt->regs[VCPU_REGS_RCX] = tss->cx;
2103 ctxt->regs[VCPU_REGS_RDX] = tss->dx;
2104 ctxt->regs[VCPU_REGS_RBX] = tss->bx;
2105 ctxt->regs[VCPU_REGS_RSP] = tss->sp;
2106 ctxt->regs[VCPU_REGS_RBP] = tss->bp;
2107 ctxt->regs[VCPU_REGS_RSI] = tss->si;
2108 ctxt->regs[VCPU_REGS_RDI] = tss->di;
2111 * SDM says that segment selectors are loaded before segment
2114 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2115 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2116 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2117 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2118 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2121 * Now load segment descriptors. If fault happenes at this stage
2122 * it is handled in a context of new task
2124 ret = load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR);
2125 if (ret != X86EMUL_CONTINUE)
2127 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2128 if (ret != X86EMUL_CONTINUE)
2130 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2131 if (ret != X86EMUL_CONTINUE)
2133 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2134 if (ret != X86EMUL_CONTINUE)
2136 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2137 if (ret != X86EMUL_CONTINUE)
2140 return X86EMUL_CONTINUE;
2143 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2144 u16 tss_selector, u16 old_tss_sel,
2145 ulong old_tss_base, struct desc_struct *new_desc)
2147 struct x86_emulate_ops *ops = ctxt->ops;
2148 struct tss_segment_16 tss_seg;
2150 u32 new_tss_base = get_desc_base(new_desc);
2152 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2154 if (ret != X86EMUL_CONTINUE)
2155 /* FIXME: need to provide precise fault address */
2158 save_state_to_tss16(ctxt, &tss_seg);
2160 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2162 if (ret != X86EMUL_CONTINUE)
2163 /* FIXME: need to provide precise fault address */
2166 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2168 if (ret != X86EMUL_CONTINUE)
2169 /* FIXME: need to provide precise fault address */
2172 if (old_tss_sel != 0xffff) {
2173 tss_seg.prev_task_link = old_tss_sel;
2175 ret = ops->write_std(ctxt, new_tss_base,
2176 &tss_seg.prev_task_link,
2177 sizeof tss_seg.prev_task_link,
2179 if (ret != X86EMUL_CONTINUE)
2180 /* FIXME: need to provide precise fault address */
2184 return load_state_from_tss16(ctxt, &tss_seg);
2187 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2188 struct tss_segment_32 *tss)
2190 tss->cr3 = ctxt->ops->get_cr(ctxt, 3);
2191 tss->eip = ctxt->_eip;
2192 tss->eflags = ctxt->eflags;
2193 tss->eax = ctxt->regs[VCPU_REGS_RAX];
2194 tss->ecx = ctxt->regs[VCPU_REGS_RCX];
2195 tss->edx = ctxt->regs[VCPU_REGS_RDX];
2196 tss->ebx = ctxt->regs[VCPU_REGS_RBX];
2197 tss->esp = ctxt->regs[VCPU_REGS_RSP];
2198 tss->ebp = ctxt->regs[VCPU_REGS_RBP];
2199 tss->esi = ctxt->regs[VCPU_REGS_RSI];
2200 tss->edi = ctxt->regs[VCPU_REGS_RDI];
2202 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2203 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2204 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2205 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2206 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2207 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2208 tss->ldt_selector = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2211 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2212 struct tss_segment_32 *tss)
2216 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2217 return emulate_gp(ctxt, 0);
2218 ctxt->_eip = tss->eip;
2219 ctxt->eflags = tss->eflags | 2;
2220 ctxt->regs[VCPU_REGS_RAX] = tss->eax;
2221 ctxt->regs[VCPU_REGS_RCX] = tss->ecx;
2222 ctxt->regs[VCPU_REGS_RDX] = tss->edx;
2223 ctxt->regs[VCPU_REGS_RBX] = tss->ebx;
2224 ctxt->regs[VCPU_REGS_RSP] = tss->esp;
2225 ctxt->regs[VCPU_REGS_RBP] = tss->ebp;
2226 ctxt->regs[VCPU_REGS_RSI] = tss->esi;
2227 ctxt->regs[VCPU_REGS_RDI] = tss->edi;
2230 * SDM says that segment selectors are loaded before segment
2233 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2234 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2235 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2236 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2237 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2238 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2239 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2242 * Now load segment descriptors. If fault happenes at this stage
2243 * it is handled in a context of new task
2245 ret = load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2246 if (ret != X86EMUL_CONTINUE)
2248 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2249 if (ret != X86EMUL_CONTINUE)
2251 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2252 if (ret != X86EMUL_CONTINUE)
2254 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2255 if (ret != X86EMUL_CONTINUE)
2257 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2258 if (ret != X86EMUL_CONTINUE)
2260 ret = load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS);
2261 if (ret != X86EMUL_CONTINUE)
2263 ret = load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS);
2264 if (ret != X86EMUL_CONTINUE)
2267 return X86EMUL_CONTINUE;
2270 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2271 u16 tss_selector, u16 old_tss_sel,
2272 ulong old_tss_base, struct desc_struct *new_desc)
2274 struct x86_emulate_ops *ops = ctxt->ops;
2275 struct tss_segment_32 tss_seg;
2277 u32 new_tss_base = get_desc_base(new_desc);
2279 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2281 if (ret != X86EMUL_CONTINUE)
2282 /* FIXME: need to provide precise fault address */
2285 save_state_to_tss32(ctxt, &tss_seg);
2287 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2289 if (ret != X86EMUL_CONTINUE)
2290 /* FIXME: need to provide precise fault address */
2293 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2295 if (ret != X86EMUL_CONTINUE)
2296 /* FIXME: need to provide precise fault address */
2299 if (old_tss_sel != 0xffff) {
2300 tss_seg.prev_task_link = old_tss_sel;
2302 ret = ops->write_std(ctxt, new_tss_base,
2303 &tss_seg.prev_task_link,
2304 sizeof tss_seg.prev_task_link,
2306 if (ret != X86EMUL_CONTINUE)
2307 /* FIXME: need to provide precise fault address */
2311 return load_state_from_tss32(ctxt, &tss_seg);
2314 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2315 u16 tss_selector, int reason,
2316 bool has_error_code, u32 error_code)
2318 struct x86_emulate_ops *ops = ctxt->ops;
2319 struct desc_struct curr_tss_desc, next_tss_desc;
2321 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2322 ulong old_tss_base =
2323 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2326 /* FIXME: old_tss_base == ~0 ? */
2328 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2329 if (ret != X86EMUL_CONTINUE)
2331 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2332 if (ret != X86EMUL_CONTINUE)
2335 /* FIXME: check that next_tss_desc is tss */
2337 if (reason != TASK_SWITCH_IRET) {
2338 if ((tss_selector & 3) > next_tss_desc.dpl ||
2339 ops->cpl(ctxt) > next_tss_desc.dpl)
2340 return emulate_gp(ctxt, 0);
2343 desc_limit = desc_limit_scaled(&next_tss_desc);
2344 if (!next_tss_desc.p ||
2345 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2346 desc_limit < 0x2b)) {
2347 emulate_ts(ctxt, tss_selector & 0xfffc);
2348 return X86EMUL_PROPAGATE_FAULT;
2351 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2352 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2353 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2356 if (reason == TASK_SWITCH_IRET)
2357 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2359 /* set back link to prev task only if NT bit is set in eflags
2360 note that old_tss_sel is not used afetr this point */
2361 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2362 old_tss_sel = 0xffff;
2364 if (next_tss_desc.type & 8)
2365 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2366 old_tss_base, &next_tss_desc);
2368 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2369 old_tss_base, &next_tss_desc);
2370 if (ret != X86EMUL_CONTINUE)
2373 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2374 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2376 if (reason != TASK_SWITCH_IRET) {
2377 next_tss_desc.type |= (1 << 1); /* set busy flag */
2378 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2381 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2382 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2384 if (has_error_code) {
2385 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2386 ctxt->lock_prefix = 0;
2387 ctxt->src.val = (unsigned long) error_code;
2388 ret = em_push(ctxt);
2394 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2395 u16 tss_selector, int reason,
2396 bool has_error_code, u32 error_code)
2400 ctxt->_eip = ctxt->eip;
2401 ctxt->dst.type = OP_NONE;
2403 rc = emulator_do_task_switch(ctxt, tss_selector, reason,
2404 has_error_code, error_code);
2406 if (rc == X86EMUL_CONTINUE)
2407 ctxt->eip = ctxt->_eip;
2409 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2412 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned seg,
2413 int reg, struct operand *op)
2415 int df = (ctxt->eflags & EFLG_DF) ? -1 : 1;
2417 register_address_increment(ctxt, &ctxt->regs[reg], df * op->bytes);
2418 op->addr.mem.ea = register_address(ctxt, ctxt->regs[reg]);
2419 op->addr.mem.seg = seg;
2422 static int em_das(struct x86_emulate_ctxt *ctxt)
2425 bool af, cf, old_cf;
2427 cf = ctxt->eflags & X86_EFLAGS_CF;
2433 af = ctxt->eflags & X86_EFLAGS_AF;
2434 if ((al & 0x0f) > 9 || af) {
2436 cf = old_cf | (al >= 250);
2441 if (old_al > 0x99 || old_cf) {
2447 /* Set PF, ZF, SF */
2448 ctxt->src.type = OP_IMM;
2450 ctxt->src.bytes = 1;
2451 emulate_2op_SrcV(ctxt, "or");
2452 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2454 ctxt->eflags |= X86_EFLAGS_CF;
2456 ctxt->eflags |= X86_EFLAGS_AF;
2457 return X86EMUL_CONTINUE;
2460 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2466 old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2467 old_eip = ctxt->_eip;
2469 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2470 if (load_segment_descriptor(ctxt, sel, VCPU_SREG_CS))
2471 return X86EMUL_CONTINUE;
2474 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
2476 ctxt->src.val = old_cs;
2478 if (rc != X86EMUL_CONTINUE)
2481 ctxt->src.val = old_eip;
2482 return em_push(ctxt);
2485 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
2489 ctxt->dst.type = OP_REG;
2490 ctxt->dst.addr.reg = &ctxt->_eip;
2491 ctxt->dst.bytes = ctxt->op_bytes;
2492 rc = emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
2493 if (rc != X86EMUL_CONTINUE)
2495 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], ctxt->src.val);
2496 return X86EMUL_CONTINUE;
2499 static int em_add(struct x86_emulate_ctxt *ctxt)
2501 emulate_2op_SrcV(ctxt, "add");
2502 return X86EMUL_CONTINUE;
2505 static int em_or(struct x86_emulate_ctxt *ctxt)
2507 emulate_2op_SrcV(ctxt, "or");
2508 return X86EMUL_CONTINUE;
2511 static int em_adc(struct x86_emulate_ctxt *ctxt)
2513 emulate_2op_SrcV(ctxt, "adc");
2514 return X86EMUL_CONTINUE;
2517 static int em_sbb(struct x86_emulate_ctxt *ctxt)
2519 emulate_2op_SrcV(ctxt, "sbb");
2520 return X86EMUL_CONTINUE;
2523 static int em_and(struct x86_emulate_ctxt *ctxt)
2525 emulate_2op_SrcV(ctxt, "and");
2526 return X86EMUL_CONTINUE;
2529 static int em_sub(struct x86_emulate_ctxt *ctxt)
2531 emulate_2op_SrcV(ctxt, "sub");
2532 return X86EMUL_CONTINUE;
2535 static int em_xor(struct x86_emulate_ctxt *ctxt)
2537 emulate_2op_SrcV(ctxt, "xor");
2538 return X86EMUL_CONTINUE;
2541 static int em_cmp(struct x86_emulate_ctxt *ctxt)
2543 emulate_2op_SrcV(ctxt, "cmp");
2544 /* Disable writeback. */
2545 ctxt->dst.type = OP_NONE;
2546 return X86EMUL_CONTINUE;
2549 static int em_test(struct x86_emulate_ctxt *ctxt)
2551 emulate_2op_SrcV(ctxt, "test");
2552 /* Disable writeback. */
2553 ctxt->dst.type = OP_NONE;
2554 return X86EMUL_CONTINUE;
2557 static int em_xchg(struct x86_emulate_ctxt *ctxt)
2559 /* Write back the register source. */
2560 ctxt->src.val = ctxt->dst.val;
2561 write_register_operand(&ctxt->src);
2563 /* Write back the memory destination with implicit LOCK prefix. */
2564 ctxt->dst.val = ctxt->src.orig_val;
2565 ctxt->lock_prefix = 1;
2566 return X86EMUL_CONTINUE;
2569 static int em_imul(struct x86_emulate_ctxt *ctxt)
2571 emulate_2op_SrcV_nobyte(ctxt, "imul");
2572 return X86EMUL_CONTINUE;
2575 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
2577 ctxt->dst.val = ctxt->src2.val;
2578 return em_imul(ctxt);
2581 static int em_cwd(struct x86_emulate_ctxt *ctxt)
2583 ctxt->dst.type = OP_REG;
2584 ctxt->dst.bytes = ctxt->src.bytes;
2585 ctxt->dst.addr.reg = &ctxt->regs[VCPU_REGS_RDX];
2586 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
2588 return X86EMUL_CONTINUE;
2591 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
2595 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
2596 ctxt->regs[VCPU_REGS_RAX] = (u32)tsc;
2597 ctxt->regs[VCPU_REGS_RDX] = tsc >> 32;
2598 return X86EMUL_CONTINUE;
2601 static int em_mov(struct x86_emulate_ctxt *ctxt)
2603 ctxt->dst.val = ctxt->src.val;
2604 return X86EMUL_CONTINUE;
2607 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
2609 if (ctxt->modrm_reg > VCPU_SREG_GS)
2610 return emulate_ud(ctxt);
2612 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
2613 return X86EMUL_CONTINUE;
2616 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
2618 u16 sel = ctxt->src.val;
2620 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
2621 return emulate_ud(ctxt);
2623 if (ctxt->modrm_reg == VCPU_SREG_SS)
2624 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
2626 /* Disable writeback. */
2627 ctxt->dst.type = OP_NONE;
2628 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
2631 static int em_movdqu(struct x86_emulate_ctxt *ctxt)
2633 memcpy(&ctxt->dst.vec_val, &ctxt->src.vec_val, ctxt->op_bytes);
2634 return X86EMUL_CONTINUE;
2637 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
2642 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
2643 if (rc == X86EMUL_CONTINUE)
2644 ctxt->ops->invlpg(ctxt, linear);
2645 /* Disable writeback. */
2646 ctxt->dst.type = OP_NONE;
2647 return X86EMUL_CONTINUE;
2650 static int em_clts(struct x86_emulate_ctxt *ctxt)
2654 cr0 = ctxt->ops->get_cr(ctxt, 0);
2656 ctxt->ops->set_cr(ctxt, 0, cr0);
2657 return X86EMUL_CONTINUE;
2660 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
2664 if (ctxt->modrm_mod != 3 || ctxt->modrm_rm != 1)
2665 return X86EMUL_UNHANDLEABLE;
2667 rc = ctxt->ops->fix_hypercall(ctxt);
2668 if (rc != X86EMUL_CONTINUE)
2671 /* Let the processor re-execute the fixed hypercall */
2672 ctxt->_eip = ctxt->eip;
2673 /* Disable writeback. */
2674 ctxt->dst.type = OP_NONE;
2675 return X86EMUL_CONTINUE;
2678 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
2680 struct desc_ptr desc_ptr;
2683 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
2684 &desc_ptr.size, &desc_ptr.address,
2686 if (rc != X86EMUL_CONTINUE)
2688 ctxt->ops->set_gdt(ctxt, &desc_ptr);
2689 /* Disable writeback. */
2690 ctxt->dst.type = OP_NONE;
2691 return X86EMUL_CONTINUE;
2694 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
2698 rc = ctxt->ops->fix_hypercall(ctxt);
2700 /* Disable writeback. */
2701 ctxt->dst.type = OP_NONE;
2705 static int em_lidt(struct x86_emulate_ctxt *ctxt)
2707 struct desc_ptr desc_ptr;
2710 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
2711 &desc_ptr.size, &desc_ptr.address,
2713 if (rc != X86EMUL_CONTINUE)
2715 ctxt->ops->set_idt(ctxt, &desc_ptr);
2716 /* Disable writeback. */
2717 ctxt->dst.type = OP_NONE;
2718 return X86EMUL_CONTINUE;
2721 static int em_smsw(struct x86_emulate_ctxt *ctxt)
2723 ctxt->dst.bytes = 2;
2724 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
2725 return X86EMUL_CONTINUE;
2728 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
2730 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
2731 | (ctxt->src.val & 0x0f));
2732 ctxt->dst.type = OP_NONE;
2733 return X86EMUL_CONTINUE;
2736 static int em_loop(struct x86_emulate_ctxt *ctxt)
2738 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RCX], -1);
2739 if ((address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) != 0) &&
2740 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
2741 jmp_rel(ctxt, ctxt->src.val);
2743 return X86EMUL_CONTINUE;
2746 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
2748 if (address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) == 0)
2749 jmp_rel(ctxt, ctxt->src.val);
2751 return X86EMUL_CONTINUE;
2754 static int em_cli(struct x86_emulate_ctxt *ctxt)
2756 if (emulator_bad_iopl(ctxt))
2757 return emulate_gp(ctxt, 0);
2759 ctxt->eflags &= ~X86_EFLAGS_IF;
2760 return X86EMUL_CONTINUE;
2763 static int em_sti(struct x86_emulate_ctxt *ctxt)
2765 if (emulator_bad_iopl(ctxt))
2766 return emulate_gp(ctxt, 0);
2768 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
2769 ctxt->eflags |= X86_EFLAGS_IF;
2770 return X86EMUL_CONTINUE;
2773 static bool valid_cr(int nr)
2785 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
2787 if (!valid_cr(ctxt->modrm_reg))
2788 return emulate_ud(ctxt);
2790 return X86EMUL_CONTINUE;
2793 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
2795 u64 new_val = ctxt->src.val64;
2796 int cr = ctxt->modrm_reg;
2799 static u64 cr_reserved_bits[] = {
2800 0xffffffff00000000ULL,
2801 0, 0, 0, /* CR3 checked later */
2808 return emulate_ud(ctxt);
2810 if (new_val & cr_reserved_bits[cr])
2811 return emulate_gp(ctxt, 0);
2816 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
2817 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
2818 return emulate_gp(ctxt, 0);
2820 cr4 = ctxt->ops->get_cr(ctxt, 4);
2821 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2823 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
2824 !(cr4 & X86_CR4_PAE))
2825 return emulate_gp(ctxt, 0);
2832 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2833 if (efer & EFER_LMA)
2834 rsvd = CR3_L_MODE_RESERVED_BITS;
2835 else if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PAE)
2836 rsvd = CR3_PAE_RESERVED_BITS;
2837 else if (ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PG)
2838 rsvd = CR3_NONPAE_RESERVED_BITS;
2841 return emulate_gp(ctxt, 0);
2848 cr4 = ctxt->ops->get_cr(ctxt, 4);
2849 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2851 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
2852 return emulate_gp(ctxt, 0);
2858 return X86EMUL_CONTINUE;
2861 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
2865 ctxt->ops->get_dr(ctxt, 7, &dr7);
2867 /* Check if DR7.Global_Enable is set */
2868 return dr7 & (1 << 13);
2871 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
2873 int dr = ctxt->modrm_reg;
2877 return emulate_ud(ctxt);
2879 cr4 = ctxt->ops->get_cr(ctxt, 4);
2880 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
2881 return emulate_ud(ctxt);
2883 if (check_dr7_gd(ctxt))
2884 return emulate_db(ctxt);
2886 return X86EMUL_CONTINUE;
2889 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
2891 u64 new_val = ctxt->src.val64;
2892 int dr = ctxt->modrm_reg;
2894 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
2895 return emulate_gp(ctxt, 0);
2897 return check_dr_read(ctxt);
2900 static int check_svme(struct x86_emulate_ctxt *ctxt)
2904 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2906 if (!(efer & EFER_SVME))
2907 return emulate_ud(ctxt);
2909 return X86EMUL_CONTINUE;
2912 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
2914 u64 rax = ctxt->regs[VCPU_REGS_RAX];
2916 /* Valid physical address? */
2917 if (rax & 0xffff000000000000ULL)
2918 return emulate_gp(ctxt, 0);
2920 return check_svme(ctxt);
2923 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
2925 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
2927 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
2928 return emulate_ud(ctxt);
2930 return X86EMUL_CONTINUE;
2933 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
2935 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
2936 u64 rcx = ctxt->regs[VCPU_REGS_RCX];
2938 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
2940 return emulate_gp(ctxt, 0);
2942 return X86EMUL_CONTINUE;
2945 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
2947 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
2948 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
2949 return emulate_gp(ctxt, 0);
2951 return X86EMUL_CONTINUE;
2954 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
2956 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
2957 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
2958 return emulate_gp(ctxt, 0);
2960 return X86EMUL_CONTINUE;
2963 #define D(_y) { .flags = (_y) }
2964 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
2965 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
2966 .check_perm = (_p) }
2968 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
2969 #define G(_f, _g) { .flags = ((_f) | Group), .u.group = (_g) }
2970 #define GD(_f, _g) { .flags = ((_f) | GroupDual), .u.gdual = (_g) }
2971 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
2972 #define II(_f, _e, _i) \
2973 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
2974 #define IIP(_f, _e, _i, _p) \
2975 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
2976 .check_perm = (_p) }
2977 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
2979 #define D2bv(_f) D((_f) | ByteOp), D(_f)
2980 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
2981 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
2983 #define I6ALU(_f, _e) I2bv((_f) | DstMem | SrcReg | ModRM, _e), \
2984 I2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
2985 I2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
2987 static struct opcode group7_rm1[] = {
2988 DI(SrcNone | ModRM | Priv, monitor),
2989 DI(SrcNone | ModRM | Priv, mwait),
2993 static struct opcode group7_rm3[] = {
2994 DIP(SrcNone | ModRM | Prot | Priv, vmrun, check_svme_pa),
2995 II(SrcNone | ModRM | Prot | VendorSpecific, em_vmmcall, vmmcall),
2996 DIP(SrcNone | ModRM | Prot | Priv, vmload, check_svme_pa),
2997 DIP(SrcNone | ModRM | Prot | Priv, vmsave, check_svme_pa),
2998 DIP(SrcNone | ModRM | Prot | Priv, stgi, check_svme),
2999 DIP(SrcNone | ModRM | Prot | Priv, clgi, check_svme),
3000 DIP(SrcNone | ModRM | Prot | Priv, skinit, check_svme),
3001 DIP(SrcNone | ModRM | Prot | Priv, invlpga, check_svme),
3004 static struct opcode group7_rm7[] = {
3006 DIP(SrcNone | ModRM, rdtscp, check_rdtsc),
3010 static struct opcode group1[] = {
3021 static struct opcode group1A[] = {
3022 D(DstMem | SrcNone | ModRM | Mov | Stack), N, N, N, N, N, N, N,
3025 static struct opcode group3[] = {
3026 I(DstMem | SrcImm | ModRM, em_test),
3027 I(DstMem | SrcImm | ModRM, em_test),
3028 I(DstMem | SrcNone | ModRM | Lock, em_not),
3029 I(DstMem | SrcNone | ModRM | Lock, em_neg),
3030 I(SrcMem | ModRM, em_mul_ex),
3031 I(SrcMem | ModRM, em_imul_ex),
3032 I(SrcMem | ModRM, em_div_ex),
3033 I(SrcMem | ModRM, em_idiv_ex),
3036 static struct opcode group4[] = {
3037 D(ByteOp | DstMem | SrcNone | ModRM | Lock), D(ByteOp | DstMem | SrcNone | ModRM | Lock),
3041 static struct opcode group5[] = {
3042 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
3043 D(SrcMem | ModRM | Stack),
3044 I(SrcMemFAddr | ModRM | ImplicitOps | Stack, em_call_far),
3045 D(SrcMem | ModRM | Stack), D(SrcMemFAddr | ModRM | ImplicitOps),
3046 D(SrcMem | ModRM | Stack), N,
3049 static struct opcode group6[] = {
3050 DI(ModRM | Prot, sldt),
3051 DI(ModRM | Prot, str),
3052 DI(ModRM | Prot | Priv, lldt),
3053 DI(ModRM | Prot | Priv, ltr),
3057 static struct group_dual group7 = { {
3058 DI(ModRM | Mov | DstMem | Priv, sgdt),
3059 DI(ModRM | Mov | DstMem | Priv, sidt),
3060 II(ModRM | SrcMem | Priv, em_lgdt, lgdt),
3061 II(ModRM | SrcMem | Priv, em_lidt, lidt),
3062 II(SrcNone | ModRM | DstMem | Mov, em_smsw, smsw), N,
3063 II(SrcMem16 | ModRM | Mov | Priv, em_lmsw, lmsw),
3064 II(SrcMem | ModRM | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3066 I(SrcNone | ModRM | Priv | VendorSpecific, em_vmcall),
3068 N, EXT(0, group7_rm3),
3069 II(SrcNone | ModRM | DstMem | Mov, em_smsw, smsw), N,
3070 II(SrcMem16 | ModRM | Mov | Priv, em_lmsw, lmsw), EXT(0, group7_rm7),
3073 static struct opcode group8[] = {
3075 D(DstMem | SrcImmByte | ModRM), D(DstMem | SrcImmByte | ModRM | Lock),
3076 D(DstMem | SrcImmByte | ModRM | Lock), D(DstMem | SrcImmByte | ModRM | Lock),
3079 static struct group_dual group9 = { {
3080 N, D(DstMem64 | ModRM | Lock), N, N, N, N, N, N,
3082 N, N, N, N, N, N, N, N,
3085 static struct opcode group11[] = {
3086 I(DstMem | SrcImm | ModRM | Mov, em_mov), X7(D(Undefined)),
3089 static struct gprefix pfx_0f_6f_0f_7f = {
3090 N, N, N, I(Sse, em_movdqu),
3093 static struct opcode opcode_table[256] = {
3095 I6ALU(Lock, em_add),
3096 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
3099 D(ImplicitOps | Stack | No64), N,
3101 I6ALU(Lock, em_adc),
3102 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
3104 I6ALU(Lock, em_sbb),
3105 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
3107 I6ALU(Lock, em_and), N, N,
3109 I6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3111 I6ALU(Lock, em_xor), N, N,
3113 I6ALU(0, em_cmp), N, N,
3117 X8(I(SrcReg | Stack, em_push)),
3119 X8(I(DstReg | Stack, em_pop)),
3121 I(ImplicitOps | Stack | No64, em_pusha),
3122 I(ImplicitOps | Stack | No64, em_popa),
3123 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
3126 I(SrcImm | Mov | Stack, em_push),
3127 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3128 I(SrcImmByte | Mov | Stack, em_push),
3129 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3130 D2bvIP(DstDI | SrcDX | Mov | String, ins, check_perm_in), /* insb, insw/insd */
3131 D2bvIP(SrcSI | DstDX | String, outs, check_perm_out), /* outsb, outsw/outsd */
3135 G(ByteOp | DstMem | SrcImm | ModRM | Group, group1),
3136 G(DstMem | SrcImm | ModRM | Group, group1),
3137 G(ByteOp | DstMem | SrcImm | ModRM | No64 | Group, group1),
3138 G(DstMem | SrcImmByte | ModRM | Group, group1),
3139 I2bv(DstMem | SrcReg | ModRM, em_test),
3140 I2bv(DstMem | SrcReg | ModRM | Lock, em_xchg),
3142 I2bv(DstMem | SrcReg | ModRM | Mov, em_mov),
3143 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3144 I(DstMem | SrcNone | ModRM | Mov, em_mov_rm_sreg),
3145 D(ModRM | SrcMem | NoAccess | DstReg),
3146 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
3149 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3151 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3152 I(SrcImmFAddr | No64, em_call_far), N,
3153 II(ImplicitOps | Stack, em_pushf, pushf),
3154 II(ImplicitOps | Stack, em_popf, popf), N, N,
3156 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3157 I2bv(DstMem | SrcAcc | Mov | MemAbs, em_mov),
3158 I2bv(SrcSI | DstDI | Mov | String, em_mov),
3159 I2bv(SrcSI | DstDI | String, em_cmp),
3161 I2bv(DstAcc | SrcImm, em_test),
3162 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
3163 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3164 I2bv(SrcAcc | DstDI | String, em_cmp),
3166 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3168 X8(I(DstReg | SrcImm | Mov, em_mov)),
3170 D2bv(DstMem | SrcImmByte | ModRM),
3171 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3172 I(ImplicitOps | Stack, em_ret),
3173 D(DstReg | SrcMemFAddr | ModRM | No64), D(DstReg | SrcMemFAddr | ModRM | No64),
3174 G(ByteOp, group11), G(0, group11),
3176 N, N, N, I(ImplicitOps | Stack, em_ret_far),
3177 D(ImplicitOps), DI(SrcImmByte, intn),
3178 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
3180 D2bv(DstMem | SrcOne | ModRM), D2bv(DstMem | ModRM),
3183 N, N, N, N, N, N, N, N,
3185 X3(I(SrcImmByte, em_loop)),
3186 I(SrcImmByte, em_jcxz),
3187 D2bvIP(SrcImmUByte | DstAcc, in, check_perm_in),
3188 D2bvIP(SrcAcc | DstImmUByte, out, check_perm_out),
3190 D(SrcImm | Stack), D(SrcImm | ImplicitOps),
3191 I(SrcImmFAddr | No64, em_jmp_far), D(SrcImmByte | ImplicitOps),
3192 D2bvIP(SrcDX | DstAcc, in, check_perm_in),
3193 D2bvIP(SrcAcc | DstDX, out, check_perm_out),
3195 N, DI(ImplicitOps, icebp), N, N,
3196 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
3197 G(ByteOp, group3), G(0, group3),
3199 D(ImplicitOps), D(ImplicitOps),
3200 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
3201 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
3204 static struct opcode twobyte_table[256] = {
3206 G(0, group6), GD(0, &group7), N, N,
3207 N, I(ImplicitOps | VendorSpecific, em_syscall),
3208 II(ImplicitOps | Priv, em_clts, clts), N,
3209 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3210 N, D(ImplicitOps | ModRM), N, N,
3212 N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
3214 DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
3215 DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
3216 DIP(ModRM | SrcMem | Priv | Op3264, cr_write, check_cr_write),
3217 DIP(ModRM | SrcMem | Priv | Op3264, dr_write, check_dr_write),
3219 N, N, N, N, N, N, N, N,
3221 DI(ImplicitOps | Priv, wrmsr),
3222 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
3223 DI(ImplicitOps | Priv, rdmsr),
3224 DIP(ImplicitOps | Priv, rdpmc, check_rdpmc),
3225 I(ImplicitOps | VendorSpecific, em_sysenter),
3226 I(ImplicitOps | Priv | VendorSpecific, em_sysexit),
3228 N, N, N, N, N, N, N, N,
3230 X16(D(DstReg | SrcMem | ModRM | Mov)),
3232 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3237 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3242 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3246 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3248 D(ImplicitOps | Stack), D(ImplicitOps | Stack),
3249 DI(ImplicitOps, cpuid), D(DstMem | SrcReg | ModRM | BitOp),
3250 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3251 D(DstMem | SrcReg | Src2CL | ModRM), N, N,
3253 D(ImplicitOps | Stack), D(ImplicitOps | Stack),
3254 DI(ImplicitOps, rsm), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3255 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3256 D(DstMem | SrcReg | Src2CL | ModRM),
3257 D(ModRM), I(DstReg | SrcMem | ModRM, em_imul),
3259 D2bv(DstMem | SrcReg | ModRM | Lock),
3260 D(DstReg | SrcMemFAddr | ModRM), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3261 D(DstReg | SrcMemFAddr | ModRM), D(DstReg | SrcMemFAddr | ModRM),
3262 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3265 G(BitOp, group8), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3266 D(DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
3267 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3269 D2bv(DstMem | SrcReg | ModRM | Lock),
3270 N, D(DstMem | SrcReg | ModRM | Mov),
3271 N, N, N, GD(0, &group9),
3272 N, N, N, N, N, N, N, N,
3274 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3276 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3278 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
3294 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
3298 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3304 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
3305 unsigned size, bool sign_extension)
3307 int rc = X86EMUL_CONTINUE;
3311 op->addr.mem.ea = ctxt->_eip;
3312 /* NB. Immediates are sign-extended as necessary. */
3313 switch (op->bytes) {
3315 op->val = insn_fetch(s8, ctxt);
3318 op->val = insn_fetch(s16, ctxt);
3321 op->val = insn_fetch(s32, ctxt);
3324 if (!sign_extension) {
3325 switch (op->bytes) {
3333 op->val &= 0xffffffff;
3341 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
3344 int rc = X86EMUL_CONTINUE;
3348 decode_register_operand(ctxt, op,
3349 ctxt->twobyte && (ctxt->b == 0xb6 || ctxt->b == 0xb7));
3352 rc = decode_imm(ctxt, op, 1, false);
3361 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3362 if (ctxt->d & BitOp)
3363 fetch_bit_operand(ctxt);
3364 op->orig_val = op->val;
3368 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3369 op->addr.reg = &ctxt->regs[VCPU_REGS_RAX];
3370 fetch_register_operand(op);
3371 op->orig_val = op->val;
3375 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3377 register_address(ctxt, ctxt->regs[VCPU_REGS_RDI]);
3378 op->addr.mem.seg = VCPU_SREG_ES;
3384 op->addr.reg = &ctxt->regs[VCPU_REGS_RDX];
3385 fetch_register_operand(op);
3389 op->val = ctxt->regs[VCPU_REGS_RCX] & 0xff;
3392 rc = decode_imm(ctxt, op, 1, true);
3399 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
3402 /* Special instructions do their own operand decoding. */
3404 op->type = OP_NONE; /* Disable writeback. */
3412 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
3414 int rc = X86EMUL_CONTINUE;
3415 int mode = ctxt->mode;
3416 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
3417 bool op_prefix = false;
3418 struct opcode opcode;
3420 ctxt->memop.type = OP_NONE;
3421 ctxt->memopp = NULL;
3422 ctxt->_eip = ctxt->eip;
3423 ctxt->fetch.start = ctxt->_eip;
3424 ctxt->fetch.end = ctxt->fetch.start + insn_len;
3426 memcpy(ctxt->fetch.data, insn, insn_len);
3429 case X86EMUL_MODE_REAL:
3430 case X86EMUL_MODE_VM86:
3431 case X86EMUL_MODE_PROT16:
3432 def_op_bytes = def_ad_bytes = 2;
3434 case X86EMUL_MODE_PROT32:
3435 def_op_bytes = def_ad_bytes = 4;
3437 #ifdef CONFIG_X86_64
3438 case X86EMUL_MODE_PROT64:
3444 return EMULATION_FAILED;
3447 ctxt->op_bytes = def_op_bytes;
3448 ctxt->ad_bytes = def_ad_bytes;
3450 /* Legacy prefixes. */
3452 switch (ctxt->b = insn_fetch(u8, ctxt)) {
3453 case 0x66: /* operand-size override */
3455 /* switch between 2/4 bytes */
3456 ctxt->op_bytes = def_op_bytes ^ 6;
3458 case 0x67: /* address-size override */
3459 if (mode == X86EMUL_MODE_PROT64)
3460 /* switch between 4/8 bytes */
3461 ctxt->ad_bytes = def_ad_bytes ^ 12;
3463 /* switch between 2/4 bytes */
3464 ctxt->ad_bytes = def_ad_bytes ^ 6;
3466 case 0x26: /* ES override */
3467 case 0x2e: /* CS override */
3468 case 0x36: /* SS override */
3469 case 0x3e: /* DS override */
3470 set_seg_override(ctxt, (ctxt->b >> 3) & 3);
3472 case 0x64: /* FS override */
3473 case 0x65: /* GS override */
3474 set_seg_override(ctxt, ctxt->b & 7);
3476 case 0x40 ... 0x4f: /* REX */
3477 if (mode != X86EMUL_MODE_PROT64)
3479 ctxt->rex_prefix = ctxt->b;
3481 case 0xf0: /* LOCK */
3482 ctxt->lock_prefix = 1;
3484 case 0xf2: /* REPNE/REPNZ */
3485 case 0xf3: /* REP/REPE/REPZ */
3486 ctxt->rep_prefix = ctxt->b;
3492 /* Any legacy prefix after a REX prefix nullifies its effect. */
3494 ctxt->rex_prefix = 0;
3500 if (ctxt->rex_prefix & 8)
3501 ctxt->op_bytes = 8; /* REX.W */
3503 /* Opcode byte(s). */
3504 opcode = opcode_table[ctxt->b];
3505 /* Two-byte opcode? */
3506 if (ctxt->b == 0x0f) {
3508 ctxt->b = insn_fetch(u8, ctxt);
3509 opcode = twobyte_table[ctxt->b];
3511 ctxt->d = opcode.flags;
3513 while (ctxt->d & GroupMask) {
3514 switch (ctxt->d & GroupMask) {
3516 ctxt->modrm = insn_fetch(u8, ctxt);
3518 goffset = (ctxt->modrm >> 3) & 7;
3519 opcode = opcode.u.group[goffset];
3522 ctxt->modrm = insn_fetch(u8, ctxt);
3524 goffset = (ctxt->modrm >> 3) & 7;
3525 if ((ctxt->modrm >> 6) == 3)
3526 opcode = opcode.u.gdual->mod3[goffset];
3528 opcode = opcode.u.gdual->mod012[goffset];
3531 goffset = ctxt->modrm & 7;
3532 opcode = opcode.u.group[goffset];
3535 if (ctxt->rep_prefix && op_prefix)
3536 return EMULATION_FAILED;
3537 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
3538 switch (simd_prefix) {
3539 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
3540 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
3541 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
3542 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
3546 return EMULATION_FAILED;
3549 ctxt->d &= ~(u64)GroupMask;
3550 ctxt->d |= opcode.flags;
3553 ctxt->execute = opcode.u.execute;
3554 ctxt->check_perm = opcode.check_perm;
3555 ctxt->intercept = opcode.intercept;
3558 if (ctxt->d == 0 || (ctxt->d & Undefined))
3559 return EMULATION_FAILED;
3561 if (!(ctxt->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
3562 return EMULATION_FAILED;
3564 if (mode == X86EMUL_MODE_PROT64 && (ctxt->d & Stack))
3567 if (ctxt->d & Op3264) {
3568 if (mode == X86EMUL_MODE_PROT64)
3575 ctxt->op_bytes = 16;
3577 /* ModRM and SIB bytes. */
3578 if (ctxt->d & ModRM) {
3579 rc = decode_modrm(ctxt, &ctxt->memop);
3580 if (!ctxt->has_seg_override)
3581 set_seg_override(ctxt, ctxt->modrm_seg);
3582 } else if (ctxt->d & MemAbs)
3583 rc = decode_abs(ctxt, &ctxt->memop);
3584 if (rc != X86EMUL_CONTINUE)
3587 if (!ctxt->has_seg_override)
3588 set_seg_override(ctxt, VCPU_SREG_DS);
3590 ctxt->memop.addr.mem.seg = seg_override(ctxt);
3592 if (ctxt->memop.type == OP_MEM && ctxt->ad_bytes != 8)
3593 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
3596 * Decode and fetch the source operand: register, memory
3599 switch (ctxt->d & SrcMask) {
3603 decode_register_operand(ctxt, &ctxt->src, 0);
3606 ctxt->memop.bytes = 2;
3609 ctxt->memop.bytes = 4;
3612 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3614 ctxt->src = ctxt->memop;
3615 ctxt->memopp = &ctxt->src;
3618 rc = decode_imm(ctxt, &ctxt->src, 2, false);
3621 rc = decode_imm(ctxt, &ctxt->src, imm_size(ctxt), true);
3624 rc = decode_imm(ctxt, &ctxt->src, imm_size(ctxt), false);
3627 rc = decode_imm(ctxt, &ctxt->src, 1, true);
3630 rc = decode_imm(ctxt, &ctxt->src, 1, false);
3633 ctxt->src.type = OP_REG;
3634 ctxt->src.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3635 ctxt->src.addr.reg = &ctxt->regs[VCPU_REGS_RAX];
3636 fetch_register_operand(&ctxt->src);
3639 ctxt->src.bytes = 1;
3643 ctxt->src.type = OP_MEM;
3644 ctxt->src.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3645 ctxt->src.addr.mem.ea =
3646 register_address(ctxt, ctxt->regs[VCPU_REGS_RSI]);
3647 ctxt->src.addr.mem.seg = seg_override(ctxt);
3651 ctxt->src.type = OP_IMM;
3652 ctxt->src.addr.mem.ea = ctxt->_eip;
3653 ctxt->src.bytes = ctxt->op_bytes + 2;
3654 insn_fetch_arr(ctxt->src.valptr, ctxt->src.bytes, ctxt);
3657 ctxt->memop.bytes = ctxt->op_bytes + 2;
3661 ctxt->src.type = OP_REG;
3662 ctxt->src.bytes = 2;
3663 ctxt->src.addr.reg = &ctxt->regs[VCPU_REGS_RDX];
3664 fetch_register_operand(&ctxt->src);
3668 if (rc != X86EMUL_CONTINUE)
3672 * Decode and fetch the second source operand: register, memory
3675 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
3676 if (rc != X86EMUL_CONTINUE)
3679 /* Decode and fetch the destination operand: register or memory. */
3680 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
3683 if (ctxt->memopp && ctxt->memopp->type == OP_MEM && ctxt->rip_relative)
3684 ctxt->memopp->addr.mem.ea += ctxt->_eip;
3686 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
3689 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
3691 /* The second termination condition only applies for REPE
3692 * and REPNE. Test if the repeat string operation prefix is
3693 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
3694 * corresponding termination condition according to:
3695 * - if REPE/REPZ and ZF = 0 then done
3696 * - if REPNE/REPNZ and ZF = 1 then done
3698 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
3699 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
3700 && (((ctxt->rep_prefix == REPE_PREFIX) &&
3701 ((ctxt->eflags & EFLG_ZF) == 0))
3702 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
3703 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
3709 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
3711 struct x86_emulate_ops *ops = ctxt->ops;
3713 int rc = X86EMUL_CONTINUE;
3714 int saved_dst_type = ctxt->dst.type;
3716 ctxt->mem_read.pos = 0;
3718 if (ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) {
3719 rc = emulate_ud(ctxt);
3723 /* LOCK prefix is allowed only with some instructions */
3724 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
3725 rc = emulate_ud(ctxt);
3729 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
3730 rc = emulate_ud(ctxt);
3735 && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)
3736 || !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
3737 rc = emulate_ud(ctxt);
3741 if ((ctxt->d & Sse) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
3742 rc = emulate_nm(ctxt);
3746 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
3747 rc = emulator_check_intercept(ctxt, ctxt->intercept,
3748 X86_ICPT_PRE_EXCEPT);
3749 if (rc != X86EMUL_CONTINUE)
3753 /* Privileged instruction can be executed only in CPL=0 */
3754 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
3755 rc = emulate_gp(ctxt, 0);
3759 /* Instruction can only be executed in protected mode */
3760 if ((ctxt->d & Prot) && !(ctxt->mode & X86EMUL_MODE_PROT)) {
3761 rc = emulate_ud(ctxt);
3765 /* Do instruction specific permission checks */
3766 if (ctxt->check_perm) {
3767 rc = ctxt->check_perm(ctxt);
3768 if (rc != X86EMUL_CONTINUE)
3772 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
3773 rc = emulator_check_intercept(ctxt, ctxt->intercept,
3774 X86_ICPT_POST_EXCEPT);
3775 if (rc != X86EMUL_CONTINUE)
3779 if (ctxt->rep_prefix && (ctxt->d & String)) {
3780 /* All REP prefixes have the same first termination condition */
3781 if (address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) == 0) {
3782 ctxt->eip = ctxt->_eip;
3787 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
3788 rc = segmented_read(ctxt, ctxt->src.addr.mem,
3789 ctxt->src.valptr, ctxt->src.bytes);
3790 if (rc != X86EMUL_CONTINUE)
3792 ctxt->src.orig_val64 = ctxt->src.val64;
3795 if (ctxt->src2.type == OP_MEM) {
3796 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
3797 &ctxt->src2.val, ctxt->src2.bytes);
3798 if (rc != X86EMUL_CONTINUE)
3802 if ((ctxt->d & DstMask) == ImplicitOps)
3806 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
3807 /* optimisation - avoid slow emulated read if Mov */
3808 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
3809 &ctxt->dst.val, ctxt->dst.bytes);
3810 if (rc != X86EMUL_CONTINUE)
3813 ctxt->dst.orig_val = ctxt->dst.val;
3817 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
3818 rc = emulator_check_intercept(ctxt, ctxt->intercept,
3819 X86_ICPT_POST_MEMACCESS);
3820 if (rc != X86EMUL_CONTINUE)
3824 if (ctxt->execute) {
3825 rc = ctxt->execute(ctxt);
3826 if (rc != X86EMUL_CONTINUE)
3835 case 0x06: /* push es */
3836 rc = emulate_push_sreg(ctxt, VCPU_SREG_ES);
3838 case 0x07: /* pop es */
3839 rc = emulate_pop_sreg(ctxt, VCPU_SREG_ES);
3841 case 0x0e: /* push cs */
3842 rc = emulate_push_sreg(ctxt, VCPU_SREG_CS);
3844 case 0x16: /* push ss */
3845 rc = emulate_push_sreg(ctxt, VCPU_SREG_SS);
3847 case 0x17: /* pop ss */
3848 rc = emulate_pop_sreg(ctxt, VCPU_SREG_SS);
3850 case 0x1e: /* push ds */
3851 rc = emulate_push_sreg(ctxt, VCPU_SREG_DS);
3853 case 0x1f: /* pop ds */
3854 rc = emulate_pop_sreg(ctxt, VCPU_SREG_DS);
3856 case 0x40 ... 0x47: /* inc r16/r32 */
3857 emulate_1op(ctxt, "inc");
3859 case 0x48 ... 0x4f: /* dec r16/r32 */
3860 emulate_1op(ctxt, "dec");
3862 case 0x63: /* movsxd */
3863 if (ctxt->mode != X86EMUL_MODE_PROT64)
3864 goto cannot_emulate;
3865 ctxt->dst.val = (s32) ctxt->src.val;
3867 case 0x6c: /* insb */
3868 case 0x6d: /* insw/insd */
3869 ctxt->src.val = ctxt->regs[VCPU_REGS_RDX];
3871 case 0x6e: /* outsb */
3872 case 0x6f: /* outsw/outsd */
3873 ctxt->dst.val = ctxt->regs[VCPU_REGS_RDX];
3876 case 0x70 ... 0x7f: /* jcc (short) */
3877 if (test_cc(ctxt->b, ctxt->eflags))
3878 jmp_rel(ctxt, ctxt->src.val);
3880 case 0x8d: /* lea r16/r32, m */
3881 ctxt->dst.val = ctxt->src.addr.mem.ea;
3883 case 0x8f: /* pop (sole member of Grp1a) */
3884 rc = em_grp1a(ctxt);
3886 case 0x90 ... 0x97: /* nop / xchg reg, rax */
3887 if (ctxt->dst.addr.reg == &ctxt->regs[VCPU_REGS_RAX])
3891 case 0x98: /* cbw/cwde/cdqe */
3892 switch (ctxt->op_bytes) {
3893 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
3894 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
3895 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
3901 case 0xc4: /* les */
3902 rc = emulate_load_segment(ctxt, VCPU_SREG_ES);
3904 case 0xc5: /* lds */
3905 rc = emulate_load_segment(ctxt, VCPU_SREG_DS);
3907 case 0xcc: /* int3 */
3908 rc = emulate_int(ctxt, 3);
3910 case 0xcd: /* int n */
3911 rc = emulate_int(ctxt, ctxt->src.val);
3913 case 0xce: /* into */
3914 if (ctxt->eflags & EFLG_OF)
3915 rc = emulate_int(ctxt, 4);
3917 case 0xd0 ... 0xd1: /* Grp2 */
3920 case 0xd2 ... 0xd3: /* Grp2 */
3921 ctxt->src.val = ctxt->regs[VCPU_REGS_RCX];
3924 case 0xe4: /* inb */
3927 case 0xe6: /* outb */
3928 case 0xe7: /* out */
3930 case 0xe8: /* call (near) */ {
3931 long int rel = ctxt->src.val;
3932 ctxt->src.val = (unsigned long) ctxt->_eip;
3937 case 0xe9: /* jmp rel */
3938 case 0xeb: /* jmp rel short */
3939 jmp_rel(ctxt, ctxt->src.val);
3940 ctxt->dst.type = OP_NONE; /* Disable writeback. */
3942 case 0xec: /* in al,dx */
3943 case 0xed: /* in (e/r)ax,dx */
3945 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3947 goto done; /* IO is needed */
3949 case 0xee: /* out dx,al */
3950 case 0xef: /* out dx,(e/r)ax */
3952 ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3954 ctxt->dst.type = OP_NONE; /* Disable writeback. */
3956 case 0xf4: /* hlt */
3957 ctxt->ops->halt(ctxt);
3959 case 0xf5: /* cmc */
3960 /* complement carry flag from eflags reg */
3961 ctxt->eflags ^= EFLG_CF;
3963 case 0xf8: /* clc */
3964 ctxt->eflags &= ~EFLG_CF;
3966 case 0xf9: /* stc */
3967 ctxt->eflags |= EFLG_CF;
3969 case 0xfc: /* cld */
3970 ctxt->eflags &= ~EFLG_DF;
3972 case 0xfd: /* std */
3973 ctxt->eflags |= EFLG_DF;
3975 case 0xfe: /* Grp4 */
3976 rc = em_grp45(ctxt);
3978 case 0xff: /* Grp5 */
3979 rc = em_grp45(ctxt);
3982 goto cannot_emulate;
3985 if (rc != X86EMUL_CONTINUE)
3989 rc = writeback(ctxt);
3990 if (rc != X86EMUL_CONTINUE)
3994 * restore dst type in case the decoding will be reused
3995 * (happens for string instruction )
3997 ctxt->dst.type = saved_dst_type;
3999 if ((ctxt->d & SrcMask) == SrcSI)
4000 string_addr_inc(ctxt, seg_override(ctxt),
4001 VCPU_REGS_RSI, &ctxt->src);
4003 if ((ctxt->d & DstMask) == DstDI)
4004 string_addr_inc(ctxt, VCPU_SREG_ES, VCPU_REGS_RDI,
4007 if (ctxt->rep_prefix && (ctxt->d & String)) {
4008 struct read_cache *r = &ctxt->io_read;
4009 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RCX], -1);
4011 if (!string_insn_completed(ctxt)) {
4013 * Re-enter guest when pio read ahead buffer is empty
4014 * or, if it is not used, after each 1024 iteration.
4016 if ((r->end != 0 || ctxt->regs[VCPU_REGS_RCX] & 0x3ff) &&
4017 (r->end == 0 || r->end != r->pos)) {
4019 * Reset read cache. Usually happens before
4020 * decode, but since instruction is restarted
4021 * we have to do it here.
4023 ctxt->mem_read.end = 0;
4024 return EMULATION_RESTART;
4026 goto done; /* skip rip writeback */
4030 ctxt->eip = ctxt->_eip;
4033 if (rc == X86EMUL_PROPAGATE_FAULT)
4034 ctxt->have_exception = true;
4035 if (rc == X86EMUL_INTERCEPTED)
4036 return EMULATION_INTERCEPTED;
4038 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4042 case 0x09: /* wbinvd */
4043 (ctxt->ops->wbinvd)(ctxt);
4045 case 0x08: /* invd */
4046 case 0x0d: /* GrpP (prefetch) */
4047 case 0x18: /* Grp16 (prefetch/nop) */
4049 case 0x20: /* mov cr, reg */
4050 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
4052 case 0x21: /* mov from dr to reg */
4053 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
4055 case 0x22: /* mov reg, cr */
4056 if (ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val)) {
4057 emulate_gp(ctxt, 0);
4058 rc = X86EMUL_PROPAGATE_FAULT;
4061 ctxt->dst.type = OP_NONE;
4063 case 0x23: /* mov from reg to dr */
4064 if (ops->set_dr(ctxt, ctxt->modrm_reg, ctxt->src.val &
4065 ((ctxt->mode == X86EMUL_MODE_PROT64) ?
4066 ~0ULL : ~0U)) < 0) {
4067 /* #UD condition is already handled by the code above */
4068 emulate_gp(ctxt, 0);
4069 rc = X86EMUL_PROPAGATE_FAULT;
4073 ctxt->dst.type = OP_NONE; /* no writeback */
4077 msr_data = (u32)ctxt->regs[VCPU_REGS_RAX]
4078 | ((u64)ctxt->regs[VCPU_REGS_RDX] << 32);
4079 if (ops->set_msr(ctxt, ctxt->regs[VCPU_REGS_RCX], msr_data)) {
4080 emulate_gp(ctxt, 0);
4081 rc = X86EMUL_PROPAGATE_FAULT;
4084 rc = X86EMUL_CONTINUE;
4088 if (ops->get_msr(ctxt, ctxt->regs[VCPU_REGS_RCX], &msr_data)) {
4089 emulate_gp(ctxt, 0);
4090 rc = X86EMUL_PROPAGATE_FAULT;
4093 ctxt->regs[VCPU_REGS_RAX] = (u32)msr_data;
4094 ctxt->regs[VCPU_REGS_RDX] = msr_data >> 32;
4096 rc = X86EMUL_CONTINUE;
4098 case 0x40 ... 0x4f: /* cmov */
4099 ctxt->dst.val = ctxt->dst.orig_val = ctxt->src.val;
4100 if (!test_cc(ctxt->b, ctxt->eflags))
4101 ctxt->dst.type = OP_NONE; /* no writeback */
4103 case 0x80 ... 0x8f: /* jnz rel, etc*/
4104 if (test_cc(ctxt->b, ctxt->eflags))
4105 jmp_rel(ctxt, ctxt->src.val);
4107 case 0x90 ... 0x9f: /* setcc r/m8 */
4108 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
4110 case 0xa0: /* push fs */
4111 rc = emulate_push_sreg(ctxt, VCPU_SREG_FS);
4113 case 0xa1: /* pop fs */
4114 rc = emulate_pop_sreg(ctxt, VCPU_SREG_FS);
4118 ctxt->dst.type = OP_NONE;
4119 /* only subword offset */
4120 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
4121 emulate_2op_SrcV_nobyte(ctxt, "bt");
4123 case 0xa4: /* shld imm8, r, r/m */
4124 case 0xa5: /* shld cl, r, r/m */
4125 emulate_2op_cl(ctxt, "shld");
4127 case 0xa8: /* push gs */
4128 rc = emulate_push_sreg(ctxt, VCPU_SREG_GS);
4130 case 0xa9: /* pop gs */
4131 rc = emulate_pop_sreg(ctxt, VCPU_SREG_GS);
4135 emulate_2op_SrcV_nobyte(ctxt, "bts");
4137 case 0xac: /* shrd imm8, r, r/m */
4138 case 0xad: /* shrd cl, r, r/m */
4139 emulate_2op_cl(ctxt, "shrd");
4141 case 0xae: /* clflush */
4143 case 0xb0 ... 0xb1: /* cmpxchg */
4145 * Save real source value, then compare EAX against
4148 ctxt->src.orig_val = ctxt->src.val;
4149 ctxt->src.val = ctxt->regs[VCPU_REGS_RAX];
4150 emulate_2op_SrcV(ctxt, "cmp");
4151 if (ctxt->eflags & EFLG_ZF) {
4152 /* Success: write back to memory. */
4153 ctxt->dst.val = ctxt->src.orig_val;
4155 /* Failure: write the value we saw to EAX. */
4156 ctxt->dst.type = OP_REG;
4157 ctxt->dst.addr.reg = (unsigned long *)&ctxt->regs[VCPU_REGS_RAX];
4160 case 0xb2: /* lss */
4161 rc = emulate_load_segment(ctxt, VCPU_SREG_SS);
4165 emulate_2op_SrcV_nobyte(ctxt, "btr");
4167 case 0xb4: /* lfs */
4168 rc = emulate_load_segment(ctxt, VCPU_SREG_FS);
4170 case 0xb5: /* lgs */
4171 rc = emulate_load_segment(ctxt, VCPU_SREG_GS);
4173 case 0xb6 ... 0xb7: /* movzx */
4174 ctxt->dst.bytes = ctxt->op_bytes;
4175 ctxt->dst.val = (ctxt->d & ByteOp) ? (u8) ctxt->src.val
4176 : (u16) ctxt->src.val;
4178 case 0xba: /* Grp8 */
4179 switch (ctxt->modrm_reg & 3) {
4192 emulate_2op_SrcV_nobyte(ctxt, "btc");
4194 case 0xbc: { /* bsf */
4196 __asm__ ("bsf %2, %0; setz %1"
4197 : "=r"(ctxt->dst.val), "=q"(zf)
4198 : "r"(ctxt->src.val));
4199 ctxt->eflags &= ~X86_EFLAGS_ZF;
4201 ctxt->eflags |= X86_EFLAGS_ZF;
4202 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4206 case 0xbd: { /* bsr */
4208 __asm__ ("bsr %2, %0; setz %1"
4209 : "=r"(ctxt->dst.val), "=q"(zf)
4210 : "r"(ctxt->src.val));
4211 ctxt->eflags &= ~X86_EFLAGS_ZF;
4213 ctxt->eflags |= X86_EFLAGS_ZF;
4214 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4218 case 0xbe ... 0xbf: /* movsx */
4219 ctxt->dst.bytes = ctxt->op_bytes;
4220 ctxt->dst.val = (ctxt->d & ByteOp) ? (s8) ctxt->src.val :
4221 (s16) ctxt->src.val;
4223 case 0xc0 ... 0xc1: /* xadd */
4224 emulate_2op_SrcV(ctxt, "add");
4225 /* Write back the register source. */
4226 ctxt->src.val = ctxt->dst.orig_val;
4227 write_register_operand(&ctxt->src);
4229 case 0xc3: /* movnti */
4230 ctxt->dst.bytes = ctxt->op_bytes;
4231 ctxt->dst.val = (ctxt->op_bytes == 4) ? (u32) ctxt->src.val :
4232 (u64) ctxt->src.val;
4234 case 0xc7: /* Grp9 (cmpxchg8b) */
4238 goto cannot_emulate;
4241 if (rc != X86EMUL_CONTINUE)
4247 return EMULATION_FAILED;