1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
32 * Opcode effective-address decode tables.
33 * Note that we only emulate instructions that have at least one memory
34 * operand (excluding implicit stack references). We assume that stack
35 * references and instruction fetches will never occur in special memory
36 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
40 /* Operand sizes: 8-bit operands or specified/overridden size. */
41 #define ByteOp (1<<0) /* 8-bit operands. */
42 /* Destination operand type. */
43 #define ImplicitOps (1<<1) /* Implicit in opcode. No generic decode. */
44 #define DstReg (2<<1) /* Register operand. */
45 #define DstMem (3<<1) /* Memory operand. */
46 #define DstAcc (4<<1) /* Destination Accumulator */
47 #define DstDI (5<<1) /* Destination is in ES:(E)DI */
48 #define DstMem64 (6<<1) /* 64bit memory operand */
49 #define DstImmUByte (7<<1) /* 8-bit unsigned immediate operand */
50 #define DstDX (8<<1) /* Destination is in DX register */
51 #define DstMask (0xf<<1)
52 /* Source operand type. */
53 #define SrcNone (0<<5) /* No source operand. */
54 #define SrcReg (1<<5) /* Register operand. */
55 #define SrcMem (2<<5) /* Memory operand. */
56 #define SrcMem16 (3<<5) /* Memory operand (16-bit). */
57 #define SrcMem32 (4<<5) /* Memory operand (32-bit). */
58 #define SrcImm (5<<5) /* Immediate operand. */
59 #define SrcImmByte (6<<5) /* 8-bit sign-extended immediate operand. */
60 #define SrcOne (7<<5) /* Implied '1' */
61 #define SrcImmUByte (8<<5) /* 8-bit unsigned immediate operand. */
62 #define SrcImmU (9<<5) /* Immediate operand, unsigned */
63 #define SrcSI (0xa<<5) /* Source is in the DS:RSI */
64 #define SrcImmFAddr (0xb<<5) /* Source is immediate far address */
65 #define SrcMemFAddr (0xc<<5) /* Source is far address in memory */
66 #define SrcAcc (0xd<<5) /* Source Accumulator */
67 #define SrcImmU16 (0xe<<5) /* Immediate operand, unsigned, 16 bits */
68 #define SrcDX (0xf<<5) /* Source is in DX register */
69 #define SrcMask (0xf<<5)
70 /* Generic ModRM decode. */
72 /* Destination is only written; never read. */
75 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
76 #define String (1<<13) /* String instruction (rep capable) */
77 #define Stack (1<<14) /* Stack instruction (push/pop) */
78 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
79 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
80 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
81 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
82 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
83 #define Sse (1<<18) /* SSE Vector instruction */
85 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
86 #define VendorSpecific (1<<22) /* Vendor specific instruction */
87 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
88 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
89 #define Undefined (1<<25) /* No Such Instruction */
90 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
91 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
93 /* Source 2 operand type */
94 #define Src2None (0<<29)
95 #define Src2CL (1<<29)
96 #define Src2ImmByte (2<<29)
97 #define Src2One (3<<29)
98 #define Src2Imm (4<<29)
99 #define Src2Mask (7<<29)
101 #define X2(x...) x, x
102 #define X3(x...) X2(x), x
103 #define X4(x...) X2(x), X2(x)
104 #define X5(x...) X4(x), x
105 #define X6(x...) X4(x), X2(x)
106 #define X7(x...) X4(x), X3(x)
107 #define X8(x...) X4(x), X4(x)
108 #define X16(x...) X8(x), X8(x)
114 int (*execute)(struct x86_emulate_ctxt *ctxt);
115 struct opcode *group;
116 struct group_dual *gdual;
117 struct gprefix *gprefix;
119 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
123 struct opcode mod012[8];
124 struct opcode mod3[8];
128 struct opcode pfx_no;
129 struct opcode pfx_66;
130 struct opcode pfx_f2;
131 struct opcode pfx_f3;
134 /* EFLAGS bit definitions. */
135 #define EFLG_ID (1<<21)
136 #define EFLG_VIP (1<<20)
137 #define EFLG_VIF (1<<19)
138 #define EFLG_AC (1<<18)
139 #define EFLG_VM (1<<17)
140 #define EFLG_RF (1<<16)
141 #define EFLG_IOPL (3<<12)
142 #define EFLG_NT (1<<14)
143 #define EFLG_OF (1<<11)
144 #define EFLG_DF (1<<10)
145 #define EFLG_IF (1<<9)
146 #define EFLG_TF (1<<8)
147 #define EFLG_SF (1<<7)
148 #define EFLG_ZF (1<<6)
149 #define EFLG_AF (1<<4)
150 #define EFLG_PF (1<<2)
151 #define EFLG_CF (1<<0)
153 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
154 #define EFLG_RESERVED_ONE_MASK 2
157 * Instruction emulation:
158 * Most instructions are emulated directly via a fragment of inline assembly
159 * code. This allows us to save/restore EFLAGS and thus very easily pick up
160 * any modified flags.
163 #if defined(CONFIG_X86_64)
164 #define _LO32 "k" /* force 32-bit operand */
165 #define _STK "%%rsp" /* stack pointer */
166 #elif defined(__i386__)
167 #define _LO32 "" /* force 32-bit operand */
168 #define _STK "%%esp" /* stack pointer */
172 * These EFLAGS bits are restored from saved value during emulation, and
173 * any changes are written back to the saved value after emulation.
175 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
177 /* Before executing instruction: restore necessary bits in EFLAGS. */
178 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
179 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
180 "movl %"_sav",%"_LO32 _tmp"; " \
183 "movl %"_msk",%"_LO32 _tmp"; " \
184 "andl %"_LO32 _tmp",("_STK"); " \
186 "notl %"_LO32 _tmp"; " \
187 "andl %"_LO32 _tmp",("_STK"); " \
188 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
190 "orl %"_LO32 _tmp",("_STK"); " \
194 /* After executing instruction: write-back necessary bits in EFLAGS. */
195 #define _POST_EFLAGS(_sav, _msk, _tmp) \
196 /* _sav |= EFLAGS & _msk; */ \
199 "andl %"_msk",%"_LO32 _tmp"; " \
200 "orl %"_LO32 _tmp",%"_sav"; "
208 #define ____emulate_2op(_op, _src, _dst, _eflags, _x, _y, _suffix, _dsttype) \
210 __asm__ __volatile__ ( \
211 _PRE_EFLAGS("0", "4", "2") \
212 _op _suffix " %"_x"3,%1; " \
213 _POST_EFLAGS("0", "4", "2") \
214 : "=m" (_eflags), "+q" (*(_dsttype*)&(_dst).val),\
216 : _y ((_src).val), "i" (EFLAGS_MASK)); \
220 /* Raw emulation: instruction has two explicit operands. */
221 #define __emulate_2op_nobyte(_op,_src,_dst,_eflags,_wx,_wy,_lx,_ly,_qx,_qy) \
223 unsigned long _tmp; \
225 switch ((_dst).bytes) { \
227 ____emulate_2op(_op,_src,_dst,_eflags,_wx,_wy,"w",u16);\
230 ____emulate_2op(_op,_src,_dst,_eflags,_lx,_ly,"l",u32);\
233 ON64(____emulate_2op(_op,_src,_dst,_eflags,_qx,_qy,"q",u64)); \
238 #define __emulate_2op(_op,_src,_dst,_eflags,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
240 unsigned long _tmp; \
241 switch ((_dst).bytes) { \
243 ____emulate_2op(_op,_src,_dst,_eflags,_bx,_by,"b",u8); \
246 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
247 _wx, _wy, _lx, _ly, _qx, _qy); \
252 /* Source operand is byte-sized and may be restricted to just %cl. */
253 #define emulate_2op_SrcB(_op, _src, _dst, _eflags) \
254 __emulate_2op(_op, _src, _dst, _eflags, \
255 "b", "c", "b", "c", "b", "c", "b", "c")
257 /* Source operand is byte, word, long or quad sized. */
258 #define emulate_2op_SrcV(_op, _src, _dst, _eflags) \
259 __emulate_2op(_op, _src, _dst, _eflags, \
260 "b", "q", "w", "r", _LO32, "r", "", "r")
262 /* Source operand is word, long or quad sized. */
263 #define emulate_2op_SrcV_nobyte(_op, _src, _dst, _eflags) \
264 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
265 "w", "r", _LO32, "r", "", "r")
267 /* Instruction has three operands and one operand is stored in ECX register */
268 #define __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, _suffix, _type) \
270 unsigned long _tmp; \
271 _type _clv = (_cl).val; \
272 _type _srcv = (_src).val; \
273 _type _dstv = (_dst).val; \
275 __asm__ __volatile__ ( \
276 _PRE_EFLAGS("0", "5", "2") \
277 _op _suffix " %4,%1 \n" \
278 _POST_EFLAGS("0", "5", "2") \
279 : "=m" (_eflags), "+r" (_dstv), "=&r" (_tmp) \
280 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
283 (_cl).val = (unsigned long) _clv; \
284 (_src).val = (unsigned long) _srcv; \
285 (_dst).val = (unsigned long) _dstv; \
288 #define emulate_2op_cl(_op, _cl, _src, _dst, _eflags) \
290 switch ((_dst).bytes) { \
292 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
293 "w", unsigned short); \
296 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
297 "l", unsigned int); \
300 ON64(__emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
301 "q", unsigned long)); \
306 #define __emulate_1op(_op, _dst, _eflags, _suffix) \
308 unsigned long _tmp; \
310 __asm__ __volatile__ ( \
311 _PRE_EFLAGS("0", "3", "2") \
312 _op _suffix " %1; " \
313 _POST_EFLAGS("0", "3", "2") \
314 : "=m" (_eflags), "+m" ((_dst).val), \
316 : "i" (EFLAGS_MASK)); \
319 /* Instruction has only one explicit operand (no source operand). */
320 #define emulate_1op(_op, _dst, _eflags) \
322 switch ((_dst).bytes) { \
323 case 1: __emulate_1op(_op, _dst, _eflags, "b"); break; \
324 case 2: __emulate_1op(_op, _dst, _eflags, "w"); break; \
325 case 4: __emulate_1op(_op, _dst, _eflags, "l"); break; \
326 case 8: ON64(__emulate_1op(_op, _dst, _eflags, "q")); break; \
330 #define __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags, _suffix) \
332 unsigned long _tmp; \
334 __asm__ __volatile__ ( \
335 _PRE_EFLAGS("0", "4", "1") \
336 _op _suffix " %5; " \
337 _POST_EFLAGS("0", "4", "1") \
338 : "=m" (_eflags), "=&r" (_tmp), \
339 "+a" (_rax), "+d" (_rdx) \
340 : "i" (EFLAGS_MASK), "m" ((_src).val), \
341 "a" (_rax), "d" (_rdx)); \
344 #define __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, _eflags, _suffix, _ex) \
346 unsigned long _tmp; \
348 __asm__ __volatile__ ( \
349 _PRE_EFLAGS("0", "5", "1") \
351 _op _suffix " %6; " \
353 _POST_EFLAGS("0", "5", "1") \
354 ".pushsection .fixup,\"ax\" \n\t" \
355 "3: movb $1, %4 \n\t" \
358 _ASM_EXTABLE(1b, 3b) \
359 : "=m" (_eflags), "=&r" (_tmp), \
360 "+a" (_rax), "+d" (_rdx), "+qm"(_ex) \
361 : "i" (EFLAGS_MASK), "m" ((_src).val), \
362 "a" (_rax), "d" (_rdx)); \
365 /* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
366 #define emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags) \
368 switch((_src).bytes) { \
370 __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, \
374 __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, \
378 __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, \
382 ON64(__emulate_1op_rax_rdx(_op, _src, _rax, _rdx, \
388 #define emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, _eflags, _ex) \
390 switch((_src).bytes) { \
392 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
393 _eflags, "b", _ex); \
396 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
397 _eflags, "w", _ex); \
400 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
401 _eflags, "l", _ex); \
404 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
405 _eflags, "q", _ex)); \
410 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
411 enum x86_intercept intercept,
412 enum x86_intercept_stage stage)
414 struct x86_instruction_info info = {
415 .intercept = intercept,
416 .rep_prefix = ctxt->rep_prefix,
417 .modrm_mod = ctxt->modrm_mod,
418 .modrm_reg = ctxt->modrm_reg,
419 .modrm_rm = ctxt->modrm_rm,
420 .src_val = ctxt->src.val64,
421 .src_bytes = ctxt->src.bytes,
422 .dst_bytes = ctxt->dst.bytes,
423 .ad_bytes = ctxt->ad_bytes,
424 .next_rip = ctxt->eip,
427 return ctxt->ops->intercept(ctxt, &info, stage);
430 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
432 return (1UL << (ctxt->ad_bytes << 3)) - 1;
435 /* Access/update address held in a register, based on addressing mode. */
436 static inline unsigned long
437 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
439 if (ctxt->ad_bytes == sizeof(unsigned long))
442 return reg & ad_mask(ctxt);
445 static inline unsigned long
446 register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
448 return address_mask(ctxt, reg);
452 register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
454 if (ctxt->ad_bytes == sizeof(unsigned long))
457 *reg = (*reg & ~ad_mask(ctxt)) | ((*reg + inc) & ad_mask(ctxt));
460 static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
462 register_address_increment(ctxt, &ctxt->_eip, rel);
465 static u32 desc_limit_scaled(struct desc_struct *desc)
467 u32 limit = get_desc_limit(desc);
469 return desc->g ? (limit << 12) | 0xfff : limit;
472 static void set_seg_override(struct x86_emulate_ctxt *ctxt, int seg)
474 ctxt->has_seg_override = true;
475 ctxt->seg_override = seg;
478 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
480 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
483 return ctxt->ops->get_cached_segment_base(ctxt, seg);
486 static unsigned seg_override(struct x86_emulate_ctxt *ctxt)
488 if (!ctxt->has_seg_override)
491 return ctxt->seg_override;
494 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
495 u32 error, bool valid)
497 ctxt->exception.vector = vec;
498 ctxt->exception.error_code = error;
499 ctxt->exception.error_code_valid = valid;
500 return X86EMUL_PROPAGATE_FAULT;
503 static int emulate_db(struct x86_emulate_ctxt *ctxt)
505 return emulate_exception(ctxt, DB_VECTOR, 0, false);
508 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
510 return emulate_exception(ctxt, GP_VECTOR, err, true);
513 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
515 return emulate_exception(ctxt, SS_VECTOR, err, true);
518 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
520 return emulate_exception(ctxt, UD_VECTOR, 0, false);
523 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
525 return emulate_exception(ctxt, TS_VECTOR, err, true);
528 static int emulate_de(struct x86_emulate_ctxt *ctxt)
530 return emulate_exception(ctxt, DE_VECTOR, 0, false);
533 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
535 return emulate_exception(ctxt, NM_VECTOR, 0, false);
538 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
541 struct desc_struct desc;
543 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
547 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
552 struct desc_struct desc;
554 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
555 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
558 static int __linearize(struct x86_emulate_ctxt *ctxt,
559 struct segmented_address addr,
560 unsigned size, bool write, bool fetch,
563 struct desc_struct desc;
570 la = seg_base(ctxt, addr.seg) + addr.ea;
571 switch (ctxt->mode) {
572 case X86EMUL_MODE_REAL:
574 case X86EMUL_MODE_PROT64:
575 if (((signed long)la << 16) >> 16 != la)
576 return emulate_gp(ctxt, 0);
579 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
583 /* code segment or read-only data segment */
584 if (((desc.type & 8) || !(desc.type & 2)) && write)
586 /* unreadable code segment */
587 if (!fetch && (desc.type & 8) && !(desc.type & 2))
589 lim = desc_limit_scaled(&desc);
590 if ((desc.type & 8) || !(desc.type & 4)) {
591 /* expand-up segment */
592 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
595 /* exapand-down segment */
596 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
598 lim = desc.d ? 0xffffffff : 0xffff;
599 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
602 cpl = ctxt->ops->cpl(ctxt);
605 if (!(desc.type & 8)) {
609 } else if ((desc.type & 8) && !(desc.type & 4)) {
610 /* nonconforming code segment */
613 } else if ((desc.type & 8) && (desc.type & 4)) {
614 /* conforming code segment */
620 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
623 return X86EMUL_CONTINUE;
625 if (addr.seg == VCPU_SREG_SS)
626 return emulate_ss(ctxt, addr.seg);
628 return emulate_gp(ctxt, addr.seg);
631 static int linearize(struct x86_emulate_ctxt *ctxt,
632 struct segmented_address addr,
633 unsigned size, bool write,
636 return __linearize(ctxt, addr, size, write, false, linear);
640 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
641 struct segmented_address addr,
648 rc = linearize(ctxt, addr, size, false, &linear);
649 if (rc != X86EMUL_CONTINUE)
651 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
655 * Fetch the next byte of the instruction being emulated which is pointed to
656 * by ctxt->_eip, then increment ctxt->_eip.
658 * Also prefetch the remaining bytes of the instruction without crossing page
659 * boundary if they are not in fetch_cache yet.
661 static int do_insn_fetch_byte(struct x86_emulate_ctxt *ctxt, u8 *dest)
663 struct fetch_cache *fc = &ctxt->fetch;
667 if (ctxt->_eip == fc->end) {
668 unsigned long linear;
669 struct segmented_address addr = { .seg = VCPU_SREG_CS,
671 cur_size = fc->end - fc->start;
672 size = min(15UL - cur_size,
673 PAGE_SIZE - offset_in_page(ctxt->_eip));
674 rc = __linearize(ctxt, addr, size, false, true, &linear);
675 if (rc != X86EMUL_CONTINUE)
677 rc = ctxt->ops->fetch(ctxt, linear, fc->data + cur_size,
678 size, &ctxt->exception);
679 if (rc != X86EMUL_CONTINUE)
683 *dest = fc->data[ctxt->_eip - fc->start];
685 return X86EMUL_CONTINUE;
688 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
689 void *dest, unsigned size)
693 /* x86 instructions are limited to 15 bytes. */
694 if (ctxt->_eip + size - ctxt->eip > 15)
695 return X86EMUL_UNHANDLEABLE;
697 rc = do_insn_fetch_byte(ctxt, dest++);
698 if (rc != X86EMUL_CONTINUE)
701 return X86EMUL_CONTINUE;
704 /* Fetch next part of the instruction being emulated. */
705 #define insn_fetch(_type, _ctxt) \
706 ({ unsigned long _x; \
707 rc = do_insn_fetch(_ctxt, &_x, sizeof(_type)); \
708 if (rc != X86EMUL_CONTINUE) \
713 #define insn_fetch_arr(_arr, _size, _ctxt) \
714 ({ rc = do_insn_fetch(_ctxt, _arr, (_size)); \
715 if (rc != X86EMUL_CONTINUE) \
720 * Given the 'reg' portion of a ModRM byte, and a register block, return a
721 * pointer into the block that addresses the relevant register.
722 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
724 static void *decode_register(u8 modrm_reg, unsigned long *regs,
729 p = ®s[modrm_reg];
730 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
731 p = (unsigned char *)®s[modrm_reg & 3] + 1;
735 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
736 struct segmented_address addr,
737 u16 *size, unsigned long *address, int op_bytes)
744 rc = segmented_read_std(ctxt, addr, size, 2);
745 if (rc != X86EMUL_CONTINUE)
748 rc = segmented_read_std(ctxt, addr, address, op_bytes);
752 static int test_cc(unsigned int condition, unsigned int flags)
756 switch ((condition & 15) >> 1) {
758 rc |= (flags & EFLG_OF);
760 case 1: /* b/c/nae */
761 rc |= (flags & EFLG_CF);
764 rc |= (flags & EFLG_ZF);
767 rc |= (flags & (EFLG_CF|EFLG_ZF));
770 rc |= (flags & EFLG_SF);
773 rc |= (flags & EFLG_PF);
776 rc |= (flags & EFLG_ZF);
779 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
783 /* Odd condition identifiers (lsb == 1) have inverted sense. */
784 return (!!rc ^ (condition & 1));
787 static void fetch_register_operand(struct operand *op)
791 op->val = *(u8 *)op->addr.reg;
794 op->val = *(u16 *)op->addr.reg;
797 op->val = *(u32 *)op->addr.reg;
800 op->val = *(u64 *)op->addr.reg;
805 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
807 ctxt->ops->get_fpu(ctxt);
809 case 0: asm("movdqu %%xmm0, %0" : "=m"(*data)); break;
810 case 1: asm("movdqu %%xmm1, %0" : "=m"(*data)); break;
811 case 2: asm("movdqu %%xmm2, %0" : "=m"(*data)); break;
812 case 3: asm("movdqu %%xmm3, %0" : "=m"(*data)); break;
813 case 4: asm("movdqu %%xmm4, %0" : "=m"(*data)); break;
814 case 5: asm("movdqu %%xmm5, %0" : "=m"(*data)); break;
815 case 6: asm("movdqu %%xmm6, %0" : "=m"(*data)); break;
816 case 7: asm("movdqu %%xmm7, %0" : "=m"(*data)); break;
818 case 8: asm("movdqu %%xmm8, %0" : "=m"(*data)); break;
819 case 9: asm("movdqu %%xmm9, %0" : "=m"(*data)); break;
820 case 10: asm("movdqu %%xmm10, %0" : "=m"(*data)); break;
821 case 11: asm("movdqu %%xmm11, %0" : "=m"(*data)); break;
822 case 12: asm("movdqu %%xmm12, %0" : "=m"(*data)); break;
823 case 13: asm("movdqu %%xmm13, %0" : "=m"(*data)); break;
824 case 14: asm("movdqu %%xmm14, %0" : "=m"(*data)); break;
825 case 15: asm("movdqu %%xmm15, %0" : "=m"(*data)); break;
829 ctxt->ops->put_fpu(ctxt);
832 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
835 ctxt->ops->get_fpu(ctxt);
837 case 0: asm("movdqu %0, %%xmm0" : : "m"(*data)); break;
838 case 1: asm("movdqu %0, %%xmm1" : : "m"(*data)); break;
839 case 2: asm("movdqu %0, %%xmm2" : : "m"(*data)); break;
840 case 3: asm("movdqu %0, %%xmm3" : : "m"(*data)); break;
841 case 4: asm("movdqu %0, %%xmm4" : : "m"(*data)); break;
842 case 5: asm("movdqu %0, %%xmm5" : : "m"(*data)); break;
843 case 6: asm("movdqu %0, %%xmm6" : : "m"(*data)); break;
844 case 7: asm("movdqu %0, %%xmm7" : : "m"(*data)); break;
846 case 8: asm("movdqu %0, %%xmm8" : : "m"(*data)); break;
847 case 9: asm("movdqu %0, %%xmm9" : : "m"(*data)); break;
848 case 10: asm("movdqu %0, %%xmm10" : : "m"(*data)); break;
849 case 11: asm("movdqu %0, %%xmm11" : : "m"(*data)); break;
850 case 12: asm("movdqu %0, %%xmm12" : : "m"(*data)); break;
851 case 13: asm("movdqu %0, %%xmm13" : : "m"(*data)); break;
852 case 14: asm("movdqu %0, %%xmm14" : : "m"(*data)); break;
853 case 15: asm("movdqu %0, %%xmm15" : : "m"(*data)); break;
857 ctxt->ops->put_fpu(ctxt);
860 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
864 unsigned reg = ctxt->modrm_reg;
865 int highbyte_regs = ctxt->rex_prefix == 0;
867 if (!(ctxt->d & ModRM))
868 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
874 read_sse_reg(ctxt, &op->vec_val, reg);
879 if ((ctxt->d & ByteOp) && !inhibit_bytereg) {
880 op->addr.reg = decode_register(reg, ctxt->regs, highbyte_regs);
883 op->addr.reg = decode_register(reg, ctxt->regs, 0);
884 op->bytes = ctxt->op_bytes;
886 fetch_register_operand(op);
887 op->orig_val = op->val;
890 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
894 int index_reg = 0, base_reg = 0, scale;
895 int rc = X86EMUL_CONTINUE;
898 if (ctxt->rex_prefix) {
899 ctxt->modrm_reg = (ctxt->rex_prefix & 4) << 1; /* REX.R */
900 index_reg = (ctxt->rex_prefix & 2) << 2; /* REX.X */
901 ctxt->modrm_rm = base_reg = (ctxt->rex_prefix & 1) << 3; /* REG.B */
904 ctxt->modrm = insn_fetch(u8, ctxt);
905 ctxt->modrm_mod |= (ctxt->modrm & 0xc0) >> 6;
906 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
907 ctxt->modrm_rm |= (ctxt->modrm & 0x07);
908 ctxt->modrm_seg = VCPU_SREG_DS;
910 if (ctxt->modrm_mod == 3) {
912 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
913 op->addr.reg = decode_register(ctxt->modrm_rm,
914 ctxt->regs, ctxt->d & ByteOp);
918 op->addr.xmm = ctxt->modrm_rm;
919 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
922 fetch_register_operand(op);
928 if (ctxt->ad_bytes == 2) {
929 unsigned bx = ctxt->regs[VCPU_REGS_RBX];
930 unsigned bp = ctxt->regs[VCPU_REGS_RBP];
931 unsigned si = ctxt->regs[VCPU_REGS_RSI];
932 unsigned di = ctxt->regs[VCPU_REGS_RDI];
934 /* 16-bit ModR/M decode. */
935 switch (ctxt->modrm_mod) {
937 if (ctxt->modrm_rm == 6)
938 modrm_ea += insn_fetch(u16, ctxt);
941 modrm_ea += insn_fetch(s8, ctxt);
944 modrm_ea += insn_fetch(u16, ctxt);
947 switch (ctxt->modrm_rm) {
967 if (ctxt->modrm_mod != 0)
974 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
975 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
976 ctxt->modrm_seg = VCPU_SREG_SS;
977 modrm_ea = (u16)modrm_ea;
979 /* 32/64-bit ModR/M decode. */
980 if ((ctxt->modrm_rm & 7) == 4) {
981 sib = insn_fetch(u8, ctxt);
982 index_reg |= (sib >> 3) & 7;
986 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
987 modrm_ea += insn_fetch(s32, ctxt);
989 modrm_ea += ctxt->regs[base_reg];
991 modrm_ea += ctxt->regs[index_reg] << scale;
992 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
993 if (ctxt->mode == X86EMUL_MODE_PROT64)
994 ctxt->rip_relative = 1;
996 modrm_ea += ctxt->regs[ctxt->modrm_rm];
997 switch (ctxt->modrm_mod) {
999 if (ctxt->modrm_rm == 5)
1000 modrm_ea += insn_fetch(s32, ctxt);
1003 modrm_ea += insn_fetch(s8, ctxt);
1006 modrm_ea += insn_fetch(s32, ctxt);
1010 op->addr.mem.ea = modrm_ea;
1015 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1018 int rc = X86EMUL_CONTINUE;
1021 switch (ctxt->ad_bytes) {
1023 op->addr.mem.ea = insn_fetch(u16, ctxt);
1026 op->addr.mem.ea = insn_fetch(u32, ctxt);
1029 op->addr.mem.ea = insn_fetch(u64, ctxt);
1036 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1040 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1041 mask = ~(ctxt->dst.bytes * 8 - 1);
1043 if (ctxt->src.bytes == 2)
1044 sv = (s16)ctxt->src.val & (s16)mask;
1045 else if (ctxt->src.bytes == 4)
1046 sv = (s32)ctxt->src.val & (s32)mask;
1048 ctxt->dst.addr.mem.ea += (sv >> 3);
1051 /* only subword offset */
1052 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1055 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1056 unsigned long addr, void *dest, unsigned size)
1059 struct read_cache *mc = &ctxt->mem_read;
1062 int n = min(size, 8u);
1064 if (mc->pos < mc->end)
1067 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, n,
1069 if (rc != X86EMUL_CONTINUE)
1074 memcpy(dest, mc->data + mc->pos, n);
1079 return X86EMUL_CONTINUE;
1082 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1083 struct segmented_address addr,
1090 rc = linearize(ctxt, addr, size, false, &linear);
1091 if (rc != X86EMUL_CONTINUE)
1093 return read_emulated(ctxt, linear, data, size);
1096 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1097 struct segmented_address addr,
1104 rc = linearize(ctxt, addr, size, true, &linear);
1105 if (rc != X86EMUL_CONTINUE)
1107 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1111 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1112 struct segmented_address addr,
1113 const void *orig_data, const void *data,
1119 rc = linearize(ctxt, addr, size, true, &linear);
1120 if (rc != X86EMUL_CONTINUE)
1122 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1123 size, &ctxt->exception);
1126 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1127 unsigned int size, unsigned short port,
1130 struct read_cache *rc = &ctxt->io_read;
1132 if (rc->pos == rc->end) { /* refill pio read ahead */
1133 unsigned int in_page, n;
1134 unsigned int count = ctxt->rep_prefix ?
1135 address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) : 1;
1136 in_page = (ctxt->eflags & EFLG_DF) ?
1137 offset_in_page(ctxt->regs[VCPU_REGS_RDI]) :
1138 PAGE_SIZE - offset_in_page(ctxt->regs[VCPU_REGS_RDI]);
1139 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1143 rc->pos = rc->end = 0;
1144 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1149 memcpy(dest, rc->data + rc->pos, size);
1154 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1155 u16 selector, struct desc_ptr *dt)
1157 struct x86_emulate_ops *ops = ctxt->ops;
1159 if (selector & 1 << 2) {
1160 struct desc_struct desc;
1163 memset (dt, 0, sizeof *dt);
1164 if (!ops->get_segment(ctxt, &sel, &desc, NULL, VCPU_SREG_LDTR))
1167 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1168 dt->address = get_desc_base(&desc);
1170 ops->get_gdt(ctxt, dt);
1173 /* allowed just for 8 bytes segments */
1174 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1175 u16 selector, struct desc_struct *desc)
1178 u16 index = selector >> 3;
1181 get_descriptor_table_ptr(ctxt, selector, &dt);
1183 if (dt.size < index * 8 + 7)
1184 return emulate_gp(ctxt, selector & 0xfffc);
1186 addr = dt.address + index * 8;
1187 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1191 /* allowed just for 8 bytes segments */
1192 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1193 u16 selector, struct desc_struct *desc)
1196 u16 index = selector >> 3;
1199 get_descriptor_table_ptr(ctxt, selector, &dt);
1201 if (dt.size < index * 8 + 7)
1202 return emulate_gp(ctxt, selector & 0xfffc);
1204 addr = dt.address + index * 8;
1205 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1209 /* Does not support long mode */
1210 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1211 u16 selector, int seg)
1213 struct desc_struct seg_desc;
1215 unsigned err_vec = GP_VECTOR;
1217 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1220 memset(&seg_desc, 0, sizeof seg_desc);
1222 if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
1223 || ctxt->mode == X86EMUL_MODE_REAL) {
1224 /* set real mode segment descriptor */
1225 set_desc_base(&seg_desc, selector << 4);
1226 set_desc_limit(&seg_desc, 0xffff);
1233 /* NULL selector is not valid for TR, CS and SS */
1234 if ((seg == VCPU_SREG_CS || seg == VCPU_SREG_SS || seg == VCPU_SREG_TR)
1238 /* TR should be in GDT only */
1239 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1242 if (null_selector) /* for NULL selector skip all following checks */
1245 ret = read_segment_descriptor(ctxt, selector, &seg_desc);
1246 if (ret != X86EMUL_CONTINUE)
1249 err_code = selector & 0xfffc;
1250 err_vec = GP_VECTOR;
1252 /* can't load system descriptor into segment selecor */
1253 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1257 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1263 cpl = ctxt->ops->cpl(ctxt);
1268 * segment is not a writable data segment or segment
1269 * selector's RPL != CPL or segment selector's RPL != CPL
1271 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1275 if (!(seg_desc.type & 8))
1278 if (seg_desc.type & 4) {
1284 if (rpl > cpl || dpl != cpl)
1287 /* CS(RPL) <- CPL */
1288 selector = (selector & 0xfffc) | cpl;
1291 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1294 case VCPU_SREG_LDTR:
1295 if (seg_desc.s || seg_desc.type != 2)
1298 default: /* DS, ES, FS, or GS */
1300 * segment is not a data or readable code segment or
1301 * ((segment is a data or nonconforming code segment)
1302 * and (both RPL and CPL > DPL))
1304 if ((seg_desc.type & 0xa) == 0x8 ||
1305 (((seg_desc.type & 0xc) != 0xc) &&
1306 (rpl > dpl && cpl > dpl)))
1312 /* mark segment as accessed */
1314 ret = write_segment_descriptor(ctxt, selector, &seg_desc);
1315 if (ret != X86EMUL_CONTINUE)
1319 ctxt->ops->set_segment(ctxt, selector, &seg_desc, 0, seg);
1320 return X86EMUL_CONTINUE;
1322 emulate_exception(ctxt, err_vec, err_code, true);
1323 return X86EMUL_PROPAGATE_FAULT;
1326 static void write_register_operand(struct operand *op)
1328 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1329 switch (op->bytes) {
1331 *(u8 *)op->addr.reg = (u8)op->val;
1334 *(u16 *)op->addr.reg = (u16)op->val;
1337 *op->addr.reg = (u32)op->val;
1338 break; /* 64b: zero-extend */
1340 *op->addr.reg = op->val;
1345 static int writeback(struct x86_emulate_ctxt *ctxt)
1349 switch (ctxt->dst.type) {
1351 write_register_operand(&ctxt->dst);
1354 if (ctxt->lock_prefix)
1355 rc = segmented_cmpxchg(ctxt,
1357 &ctxt->dst.orig_val,
1361 rc = segmented_write(ctxt,
1365 if (rc != X86EMUL_CONTINUE)
1369 write_sse_reg(ctxt, &ctxt->dst.vec_val, ctxt->dst.addr.xmm);
1377 return X86EMUL_CONTINUE;
1380 static int em_push(struct x86_emulate_ctxt *ctxt)
1382 struct segmented_address addr;
1384 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], -ctxt->op_bytes);
1385 addr.ea = register_address(ctxt, ctxt->regs[VCPU_REGS_RSP]);
1386 addr.seg = VCPU_SREG_SS;
1388 /* Disable writeback. */
1389 ctxt->dst.type = OP_NONE;
1390 return segmented_write(ctxt, addr, &ctxt->src.val, ctxt->op_bytes);
1393 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1394 void *dest, int len)
1397 struct segmented_address addr;
1399 addr.ea = register_address(ctxt, ctxt->regs[VCPU_REGS_RSP]);
1400 addr.seg = VCPU_SREG_SS;
1401 rc = segmented_read(ctxt, addr, dest, len);
1402 if (rc != X86EMUL_CONTINUE)
1405 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], len);
1409 static int em_pop(struct x86_emulate_ctxt *ctxt)
1411 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1414 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1415 void *dest, int len)
1418 unsigned long val, change_mask;
1419 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1420 int cpl = ctxt->ops->cpl(ctxt);
1422 rc = emulate_pop(ctxt, &val, len);
1423 if (rc != X86EMUL_CONTINUE)
1426 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1427 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1429 switch(ctxt->mode) {
1430 case X86EMUL_MODE_PROT64:
1431 case X86EMUL_MODE_PROT32:
1432 case X86EMUL_MODE_PROT16:
1434 change_mask |= EFLG_IOPL;
1436 change_mask |= EFLG_IF;
1438 case X86EMUL_MODE_VM86:
1440 return emulate_gp(ctxt, 0);
1441 change_mask |= EFLG_IF;
1443 default: /* real mode */
1444 change_mask |= (EFLG_IOPL | EFLG_IF);
1448 *(unsigned long *)dest =
1449 (ctxt->eflags & ~change_mask) | (val & change_mask);
1454 static int em_popf(struct x86_emulate_ctxt *ctxt)
1456 ctxt->dst.type = OP_REG;
1457 ctxt->dst.addr.reg = &ctxt->eflags;
1458 ctxt->dst.bytes = ctxt->op_bytes;
1459 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1462 static int emulate_push_sreg(struct x86_emulate_ctxt *ctxt, int seg)
1464 ctxt->src.val = get_segment_selector(ctxt, seg);
1466 return em_push(ctxt);
1469 static int emulate_pop_sreg(struct x86_emulate_ctxt *ctxt, int seg)
1471 unsigned long selector;
1474 rc = emulate_pop(ctxt, &selector, ctxt->op_bytes);
1475 if (rc != X86EMUL_CONTINUE)
1478 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1482 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1484 unsigned long old_esp = ctxt->regs[VCPU_REGS_RSP];
1485 int rc = X86EMUL_CONTINUE;
1486 int reg = VCPU_REGS_RAX;
1488 while (reg <= VCPU_REGS_RDI) {
1489 (reg == VCPU_REGS_RSP) ?
1490 (ctxt->src.val = old_esp) : (ctxt->src.val = ctxt->regs[reg]);
1493 if (rc != X86EMUL_CONTINUE)
1502 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1504 ctxt->src.val = (unsigned long)ctxt->eflags;
1505 return em_push(ctxt);
1508 static int em_popa(struct x86_emulate_ctxt *ctxt)
1510 int rc = X86EMUL_CONTINUE;
1511 int reg = VCPU_REGS_RDI;
1513 while (reg >= VCPU_REGS_RAX) {
1514 if (reg == VCPU_REGS_RSP) {
1515 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP],
1520 rc = emulate_pop(ctxt, &ctxt->regs[reg], ctxt->op_bytes);
1521 if (rc != X86EMUL_CONTINUE)
1528 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1530 struct x86_emulate_ops *ops = ctxt->ops;
1537 /* TODO: Add limit checks */
1538 ctxt->src.val = ctxt->eflags;
1540 if (rc != X86EMUL_CONTINUE)
1543 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1545 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1547 if (rc != X86EMUL_CONTINUE)
1550 ctxt->src.val = ctxt->_eip;
1552 if (rc != X86EMUL_CONTINUE)
1555 ops->get_idt(ctxt, &dt);
1557 eip_addr = dt.address + (irq << 2);
1558 cs_addr = dt.address + (irq << 2) + 2;
1560 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1561 if (rc != X86EMUL_CONTINUE)
1564 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1565 if (rc != X86EMUL_CONTINUE)
1568 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1569 if (rc != X86EMUL_CONTINUE)
1577 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
1579 switch(ctxt->mode) {
1580 case X86EMUL_MODE_REAL:
1581 return emulate_int_real(ctxt, irq);
1582 case X86EMUL_MODE_VM86:
1583 case X86EMUL_MODE_PROT16:
1584 case X86EMUL_MODE_PROT32:
1585 case X86EMUL_MODE_PROT64:
1587 /* Protected mode interrupts unimplemented yet */
1588 return X86EMUL_UNHANDLEABLE;
1592 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
1594 int rc = X86EMUL_CONTINUE;
1595 unsigned long temp_eip = 0;
1596 unsigned long temp_eflags = 0;
1597 unsigned long cs = 0;
1598 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1599 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1600 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1601 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1603 /* TODO: Add stack limit check */
1605 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
1607 if (rc != X86EMUL_CONTINUE)
1610 if (temp_eip & ~0xffff)
1611 return emulate_gp(ctxt, 0);
1613 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1615 if (rc != X86EMUL_CONTINUE)
1618 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
1620 if (rc != X86EMUL_CONTINUE)
1623 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1625 if (rc != X86EMUL_CONTINUE)
1628 ctxt->_eip = temp_eip;
1631 if (ctxt->op_bytes == 4)
1632 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1633 else if (ctxt->op_bytes == 2) {
1634 ctxt->eflags &= ~0xffff;
1635 ctxt->eflags |= temp_eflags;
1638 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1639 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1644 static int em_iret(struct x86_emulate_ctxt *ctxt)
1646 switch(ctxt->mode) {
1647 case X86EMUL_MODE_REAL:
1648 return emulate_iret_real(ctxt);
1649 case X86EMUL_MODE_VM86:
1650 case X86EMUL_MODE_PROT16:
1651 case X86EMUL_MODE_PROT32:
1652 case X86EMUL_MODE_PROT64:
1654 /* iret from protected mode unimplemented yet */
1655 return X86EMUL_UNHANDLEABLE;
1659 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
1664 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1666 rc = load_segment_descriptor(ctxt, sel, VCPU_SREG_CS);
1667 if (rc != X86EMUL_CONTINUE)
1671 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
1672 return X86EMUL_CONTINUE;
1675 static int em_grp1a(struct x86_emulate_ctxt *ctxt)
1677 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->dst.bytes);
1680 static int em_grp2(struct x86_emulate_ctxt *ctxt)
1682 switch (ctxt->modrm_reg) {
1684 emulate_2op_SrcB("rol", ctxt->src, ctxt->dst, ctxt->eflags);
1687 emulate_2op_SrcB("ror", ctxt->src, ctxt->dst, ctxt->eflags);
1690 emulate_2op_SrcB("rcl", ctxt->src, ctxt->dst, ctxt->eflags);
1693 emulate_2op_SrcB("rcr", ctxt->src, ctxt->dst, ctxt->eflags);
1695 case 4: /* sal/shl */
1696 case 6: /* sal/shl */
1697 emulate_2op_SrcB("sal", ctxt->src, ctxt->dst, ctxt->eflags);
1700 emulate_2op_SrcB("shr", ctxt->src, ctxt->dst, ctxt->eflags);
1703 emulate_2op_SrcB("sar", ctxt->src, ctxt->dst, ctxt->eflags);
1706 return X86EMUL_CONTINUE;
1709 static int em_grp3(struct x86_emulate_ctxt *ctxt)
1711 unsigned long *rax = &ctxt->regs[VCPU_REGS_RAX];
1712 unsigned long *rdx = &ctxt->regs[VCPU_REGS_RDX];
1715 switch (ctxt->modrm_reg) {
1716 case 0 ... 1: /* test */
1717 emulate_2op_SrcV("test", ctxt->src, ctxt->dst, ctxt->eflags);
1720 ctxt->dst.val = ~ctxt->dst.val;
1723 emulate_1op("neg", ctxt->dst, ctxt->eflags);
1726 emulate_1op_rax_rdx("mul", ctxt->src, *rax, *rdx, ctxt->eflags);
1729 emulate_1op_rax_rdx("imul", ctxt->src, *rax, *rdx, ctxt->eflags);
1732 emulate_1op_rax_rdx_ex("div", ctxt->src, *rax, *rdx,
1736 emulate_1op_rax_rdx_ex("idiv", ctxt->src, *rax, *rdx,
1740 return X86EMUL_UNHANDLEABLE;
1743 return emulate_de(ctxt);
1744 return X86EMUL_CONTINUE;
1747 static int em_grp45(struct x86_emulate_ctxt *ctxt)
1749 int rc = X86EMUL_CONTINUE;
1751 switch (ctxt->modrm_reg) {
1753 emulate_1op("inc", ctxt->dst, ctxt->eflags);
1756 emulate_1op("dec", ctxt->dst, ctxt->eflags);
1758 case 2: /* call near abs */ {
1760 old_eip = ctxt->_eip;
1761 ctxt->_eip = ctxt->src.val;
1762 ctxt->src.val = old_eip;
1766 case 4: /* jmp abs */
1767 ctxt->_eip = ctxt->src.val;
1769 case 5: /* jmp far */
1770 rc = em_jmp_far(ctxt);
1779 static int em_grp9(struct x86_emulate_ctxt *ctxt)
1781 u64 old = ctxt->dst.orig_val64;
1783 if (((u32) (old >> 0) != (u32) ctxt->regs[VCPU_REGS_RAX]) ||
1784 ((u32) (old >> 32) != (u32) ctxt->regs[VCPU_REGS_RDX])) {
1785 ctxt->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
1786 ctxt->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1787 ctxt->eflags &= ~EFLG_ZF;
1789 ctxt->dst.val64 = ((u64)ctxt->regs[VCPU_REGS_RCX] << 32) |
1790 (u32) ctxt->regs[VCPU_REGS_RBX];
1792 ctxt->eflags |= EFLG_ZF;
1794 return X86EMUL_CONTINUE;
1797 static int em_ret(struct x86_emulate_ctxt *ctxt)
1799 ctxt->dst.type = OP_REG;
1800 ctxt->dst.addr.reg = &ctxt->_eip;
1801 ctxt->dst.bytes = ctxt->op_bytes;
1802 return em_pop(ctxt);
1805 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
1810 rc = emulate_pop(ctxt, &ctxt->_eip, ctxt->op_bytes);
1811 if (rc != X86EMUL_CONTINUE)
1813 if (ctxt->op_bytes == 4)
1814 ctxt->_eip = (u32)ctxt->_eip;
1815 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1816 if (rc != X86EMUL_CONTINUE)
1818 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1822 static int emulate_load_segment(struct x86_emulate_ctxt *ctxt, int seg)
1827 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1829 rc = load_segment_descriptor(ctxt, sel, seg);
1830 if (rc != X86EMUL_CONTINUE)
1833 ctxt->dst.val = ctxt->src.val;
1838 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1839 struct desc_struct *cs, struct desc_struct *ss)
1843 memset(cs, 0, sizeof(struct desc_struct));
1844 ctxt->ops->get_segment(ctxt, &selector, cs, NULL, VCPU_SREG_CS);
1845 memset(ss, 0, sizeof(struct desc_struct));
1847 cs->l = 0; /* will be adjusted later */
1848 set_desc_base(cs, 0); /* flat segment */
1849 cs->g = 1; /* 4kb granularity */
1850 set_desc_limit(cs, 0xfffff); /* 4GB limit */
1851 cs->type = 0x0b; /* Read, Execute, Accessed */
1853 cs->dpl = 0; /* will be adjusted later */
1857 set_desc_base(ss, 0); /* flat segment */
1858 set_desc_limit(ss, 0xfffff); /* 4GB limit */
1859 ss->g = 1; /* 4kb granularity */
1861 ss->type = 0x03; /* Read/Write, Accessed */
1862 ss->d = 1; /* 32bit stack segment */
1867 static int em_syscall(struct x86_emulate_ctxt *ctxt)
1869 struct x86_emulate_ops *ops = ctxt->ops;
1870 struct desc_struct cs, ss;
1875 /* syscall is not available in real mode */
1876 if (ctxt->mode == X86EMUL_MODE_REAL ||
1877 ctxt->mode == X86EMUL_MODE_VM86)
1878 return emulate_ud(ctxt);
1880 ops->get_msr(ctxt, MSR_EFER, &efer);
1881 setup_syscalls_segments(ctxt, &cs, &ss);
1883 ops->get_msr(ctxt, MSR_STAR, &msr_data);
1885 cs_sel = (u16)(msr_data & 0xfffc);
1886 ss_sel = (u16)(msr_data + 8);
1888 if (efer & EFER_LMA) {
1892 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
1893 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
1895 ctxt->regs[VCPU_REGS_RCX] = ctxt->_eip;
1896 if (efer & EFER_LMA) {
1897 #ifdef CONFIG_X86_64
1898 ctxt->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
1901 ctxt->mode == X86EMUL_MODE_PROT64 ?
1902 MSR_LSTAR : MSR_CSTAR, &msr_data);
1903 ctxt->_eip = msr_data;
1905 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
1906 ctxt->eflags &= ~(msr_data | EFLG_RF);
1910 ops->get_msr(ctxt, MSR_STAR, &msr_data);
1911 ctxt->_eip = (u32)msr_data;
1913 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1916 return X86EMUL_CONTINUE;
1919 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
1921 struct x86_emulate_ops *ops = ctxt->ops;
1922 struct desc_struct cs, ss;
1927 ops->get_msr(ctxt, MSR_EFER, &efer);
1928 /* inject #GP if in real mode */
1929 if (ctxt->mode == X86EMUL_MODE_REAL)
1930 return emulate_gp(ctxt, 0);
1932 /* XXX sysenter/sysexit have not been tested in 64bit mode.
1933 * Therefore, we inject an #UD.
1935 if (ctxt->mode == X86EMUL_MODE_PROT64)
1936 return emulate_ud(ctxt);
1938 setup_syscalls_segments(ctxt, &cs, &ss);
1940 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
1941 switch (ctxt->mode) {
1942 case X86EMUL_MODE_PROT32:
1943 if ((msr_data & 0xfffc) == 0x0)
1944 return emulate_gp(ctxt, 0);
1946 case X86EMUL_MODE_PROT64:
1947 if (msr_data == 0x0)
1948 return emulate_gp(ctxt, 0);
1952 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1953 cs_sel = (u16)msr_data;
1954 cs_sel &= ~SELECTOR_RPL_MASK;
1955 ss_sel = cs_sel + 8;
1956 ss_sel &= ~SELECTOR_RPL_MASK;
1957 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
1962 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
1963 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
1965 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
1966 ctxt->_eip = msr_data;
1968 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
1969 ctxt->regs[VCPU_REGS_RSP] = msr_data;
1971 return X86EMUL_CONTINUE;
1974 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
1976 struct x86_emulate_ops *ops = ctxt->ops;
1977 struct desc_struct cs, ss;
1980 u16 cs_sel = 0, ss_sel = 0;
1982 /* inject #GP if in real mode or Virtual 8086 mode */
1983 if (ctxt->mode == X86EMUL_MODE_REAL ||
1984 ctxt->mode == X86EMUL_MODE_VM86)
1985 return emulate_gp(ctxt, 0);
1987 setup_syscalls_segments(ctxt, &cs, &ss);
1989 if ((ctxt->rex_prefix & 0x8) != 0x0)
1990 usermode = X86EMUL_MODE_PROT64;
1992 usermode = X86EMUL_MODE_PROT32;
1996 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
1998 case X86EMUL_MODE_PROT32:
1999 cs_sel = (u16)(msr_data + 16);
2000 if ((msr_data & 0xfffc) == 0x0)
2001 return emulate_gp(ctxt, 0);
2002 ss_sel = (u16)(msr_data + 24);
2004 case X86EMUL_MODE_PROT64:
2005 cs_sel = (u16)(msr_data + 32);
2006 if (msr_data == 0x0)
2007 return emulate_gp(ctxt, 0);
2008 ss_sel = cs_sel + 8;
2013 cs_sel |= SELECTOR_RPL_MASK;
2014 ss_sel |= SELECTOR_RPL_MASK;
2016 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2017 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2019 ctxt->_eip = ctxt->regs[VCPU_REGS_RDX];
2020 ctxt->regs[VCPU_REGS_RSP] = ctxt->regs[VCPU_REGS_RCX];
2022 return X86EMUL_CONTINUE;
2025 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2028 if (ctxt->mode == X86EMUL_MODE_REAL)
2030 if (ctxt->mode == X86EMUL_MODE_VM86)
2032 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2033 return ctxt->ops->cpl(ctxt) > iopl;
2036 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2039 struct x86_emulate_ops *ops = ctxt->ops;
2040 struct desc_struct tr_seg;
2043 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2044 unsigned mask = (1 << len) - 1;
2047 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2050 if (desc_limit_scaled(&tr_seg) < 103)
2052 base = get_desc_base(&tr_seg);
2053 #ifdef CONFIG_X86_64
2054 base |= ((u64)base3) << 32;
2056 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2057 if (r != X86EMUL_CONTINUE)
2059 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2061 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2062 if (r != X86EMUL_CONTINUE)
2064 if ((perm >> bit_idx) & mask)
2069 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2075 if (emulator_bad_iopl(ctxt))
2076 if (!emulator_io_port_access_allowed(ctxt, port, len))
2079 ctxt->perm_ok = true;
2084 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2085 struct tss_segment_16 *tss)
2087 tss->ip = ctxt->_eip;
2088 tss->flag = ctxt->eflags;
2089 tss->ax = ctxt->regs[VCPU_REGS_RAX];
2090 tss->cx = ctxt->regs[VCPU_REGS_RCX];
2091 tss->dx = ctxt->regs[VCPU_REGS_RDX];
2092 tss->bx = ctxt->regs[VCPU_REGS_RBX];
2093 tss->sp = ctxt->regs[VCPU_REGS_RSP];
2094 tss->bp = ctxt->regs[VCPU_REGS_RBP];
2095 tss->si = ctxt->regs[VCPU_REGS_RSI];
2096 tss->di = ctxt->regs[VCPU_REGS_RDI];
2098 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2099 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2100 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2101 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2102 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2105 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2106 struct tss_segment_16 *tss)
2110 ctxt->_eip = tss->ip;
2111 ctxt->eflags = tss->flag | 2;
2112 ctxt->regs[VCPU_REGS_RAX] = tss->ax;
2113 ctxt->regs[VCPU_REGS_RCX] = tss->cx;
2114 ctxt->regs[VCPU_REGS_RDX] = tss->dx;
2115 ctxt->regs[VCPU_REGS_RBX] = tss->bx;
2116 ctxt->regs[VCPU_REGS_RSP] = tss->sp;
2117 ctxt->regs[VCPU_REGS_RBP] = tss->bp;
2118 ctxt->regs[VCPU_REGS_RSI] = tss->si;
2119 ctxt->regs[VCPU_REGS_RDI] = tss->di;
2122 * SDM says that segment selectors are loaded before segment
2125 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2126 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2127 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2128 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2129 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2132 * Now load segment descriptors. If fault happenes at this stage
2133 * it is handled in a context of new task
2135 ret = load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR);
2136 if (ret != X86EMUL_CONTINUE)
2138 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2139 if (ret != X86EMUL_CONTINUE)
2141 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2142 if (ret != X86EMUL_CONTINUE)
2144 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2145 if (ret != X86EMUL_CONTINUE)
2147 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2148 if (ret != X86EMUL_CONTINUE)
2151 return X86EMUL_CONTINUE;
2154 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2155 u16 tss_selector, u16 old_tss_sel,
2156 ulong old_tss_base, struct desc_struct *new_desc)
2158 struct x86_emulate_ops *ops = ctxt->ops;
2159 struct tss_segment_16 tss_seg;
2161 u32 new_tss_base = get_desc_base(new_desc);
2163 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2165 if (ret != X86EMUL_CONTINUE)
2166 /* FIXME: need to provide precise fault address */
2169 save_state_to_tss16(ctxt, &tss_seg);
2171 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2173 if (ret != X86EMUL_CONTINUE)
2174 /* FIXME: need to provide precise fault address */
2177 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2179 if (ret != X86EMUL_CONTINUE)
2180 /* FIXME: need to provide precise fault address */
2183 if (old_tss_sel != 0xffff) {
2184 tss_seg.prev_task_link = old_tss_sel;
2186 ret = ops->write_std(ctxt, new_tss_base,
2187 &tss_seg.prev_task_link,
2188 sizeof tss_seg.prev_task_link,
2190 if (ret != X86EMUL_CONTINUE)
2191 /* FIXME: need to provide precise fault address */
2195 return load_state_from_tss16(ctxt, &tss_seg);
2198 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2199 struct tss_segment_32 *tss)
2201 tss->cr3 = ctxt->ops->get_cr(ctxt, 3);
2202 tss->eip = ctxt->_eip;
2203 tss->eflags = ctxt->eflags;
2204 tss->eax = ctxt->regs[VCPU_REGS_RAX];
2205 tss->ecx = ctxt->regs[VCPU_REGS_RCX];
2206 tss->edx = ctxt->regs[VCPU_REGS_RDX];
2207 tss->ebx = ctxt->regs[VCPU_REGS_RBX];
2208 tss->esp = ctxt->regs[VCPU_REGS_RSP];
2209 tss->ebp = ctxt->regs[VCPU_REGS_RBP];
2210 tss->esi = ctxt->regs[VCPU_REGS_RSI];
2211 tss->edi = ctxt->regs[VCPU_REGS_RDI];
2213 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2214 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2215 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2216 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2217 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2218 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2219 tss->ldt_selector = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2222 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2223 struct tss_segment_32 *tss)
2227 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2228 return emulate_gp(ctxt, 0);
2229 ctxt->_eip = tss->eip;
2230 ctxt->eflags = tss->eflags | 2;
2231 ctxt->regs[VCPU_REGS_RAX] = tss->eax;
2232 ctxt->regs[VCPU_REGS_RCX] = tss->ecx;
2233 ctxt->regs[VCPU_REGS_RDX] = tss->edx;
2234 ctxt->regs[VCPU_REGS_RBX] = tss->ebx;
2235 ctxt->regs[VCPU_REGS_RSP] = tss->esp;
2236 ctxt->regs[VCPU_REGS_RBP] = tss->ebp;
2237 ctxt->regs[VCPU_REGS_RSI] = tss->esi;
2238 ctxt->regs[VCPU_REGS_RDI] = tss->edi;
2241 * SDM says that segment selectors are loaded before segment
2244 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2245 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2246 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2247 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2248 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2249 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2250 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2253 * Now load segment descriptors. If fault happenes at this stage
2254 * it is handled in a context of new task
2256 ret = load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2257 if (ret != X86EMUL_CONTINUE)
2259 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2260 if (ret != X86EMUL_CONTINUE)
2262 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2263 if (ret != X86EMUL_CONTINUE)
2265 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2266 if (ret != X86EMUL_CONTINUE)
2268 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2269 if (ret != X86EMUL_CONTINUE)
2271 ret = load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS);
2272 if (ret != X86EMUL_CONTINUE)
2274 ret = load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS);
2275 if (ret != X86EMUL_CONTINUE)
2278 return X86EMUL_CONTINUE;
2281 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2282 u16 tss_selector, u16 old_tss_sel,
2283 ulong old_tss_base, struct desc_struct *new_desc)
2285 struct x86_emulate_ops *ops = ctxt->ops;
2286 struct tss_segment_32 tss_seg;
2288 u32 new_tss_base = get_desc_base(new_desc);
2290 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2292 if (ret != X86EMUL_CONTINUE)
2293 /* FIXME: need to provide precise fault address */
2296 save_state_to_tss32(ctxt, &tss_seg);
2298 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2300 if (ret != X86EMUL_CONTINUE)
2301 /* FIXME: need to provide precise fault address */
2304 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2306 if (ret != X86EMUL_CONTINUE)
2307 /* FIXME: need to provide precise fault address */
2310 if (old_tss_sel != 0xffff) {
2311 tss_seg.prev_task_link = old_tss_sel;
2313 ret = ops->write_std(ctxt, new_tss_base,
2314 &tss_seg.prev_task_link,
2315 sizeof tss_seg.prev_task_link,
2317 if (ret != X86EMUL_CONTINUE)
2318 /* FIXME: need to provide precise fault address */
2322 return load_state_from_tss32(ctxt, &tss_seg);
2325 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2326 u16 tss_selector, int reason,
2327 bool has_error_code, u32 error_code)
2329 struct x86_emulate_ops *ops = ctxt->ops;
2330 struct desc_struct curr_tss_desc, next_tss_desc;
2332 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2333 ulong old_tss_base =
2334 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2337 /* FIXME: old_tss_base == ~0 ? */
2339 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2340 if (ret != X86EMUL_CONTINUE)
2342 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2343 if (ret != X86EMUL_CONTINUE)
2346 /* FIXME: check that next_tss_desc is tss */
2348 if (reason != TASK_SWITCH_IRET) {
2349 if ((tss_selector & 3) > next_tss_desc.dpl ||
2350 ops->cpl(ctxt) > next_tss_desc.dpl)
2351 return emulate_gp(ctxt, 0);
2354 desc_limit = desc_limit_scaled(&next_tss_desc);
2355 if (!next_tss_desc.p ||
2356 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2357 desc_limit < 0x2b)) {
2358 emulate_ts(ctxt, tss_selector & 0xfffc);
2359 return X86EMUL_PROPAGATE_FAULT;
2362 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2363 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2364 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2367 if (reason == TASK_SWITCH_IRET)
2368 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2370 /* set back link to prev task only if NT bit is set in eflags
2371 note that old_tss_sel is not used afetr this point */
2372 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2373 old_tss_sel = 0xffff;
2375 if (next_tss_desc.type & 8)
2376 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2377 old_tss_base, &next_tss_desc);
2379 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2380 old_tss_base, &next_tss_desc);
2381 if (ret != X86EMUL_CONTINUE)
2384 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2385 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2387 if (reason != TASK_SWITCH_IRET) {
2388 next_tss_desc.type |= (1 << 1); /* set busy flag */
2389 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2392 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2393 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2395 if (has_error_code) {
2396 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2397 ctxt->lock_prefix = 0;
2398 ctxt->src.val = (unsigned long) error_code;
2399 ret = em_push(ctxt);
2405 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2406 u16 tss_selector, int reason,
2407 bool has_error_code, u32 error_code)
2411 ctxt->_eip = ctxt->eip;
2412 ctxt->dst.type = OP_NONE;
2414 rc = emulator_do_task_switch(ctxt, tss_selector, reason,
2415 has_error_code, error_code);
2417 if (rc == X86EMUL_CONTINUE)
2418 ctxt->eip = ctxt->_eip;
2420 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2423 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned seg,
2424 int reg, struct operand *op)
2426 int df = (ctxt->eflags & EFLG_DF) ? -1 : 1;
2428 register_address_increment(ctxt, &ctxt->regs[reg], df * op->bytes);
2429 op->addr.mem.ea = register_address(ctxt, ctxt->regs[reg]);
2430 op->addr.mem.seg = seg;
2433 static int em_das(struct x86_emulate_ctxt *ctxt)
2436 bool af, cf, old_cf;
2438 cf = ctxt->eflags & X86_EFLAGS_CF;
2444 af = ctxt->eflags & X86_EFLAGS_AF;
2445 if ((al & 0x0f) > 9 || af) {
2447 cf = old_cf | (al >= 250);
2452 if (old_al > 0x99 || old_cf) {
2458 /* Set PF, ZF, SF */
2459 ctxt->src.type = OP_IMM;
2461 ctxt->src.bytes = 1;
2462 emulate_2op_SrcV("or", ctxt->src, ctxt->dst, ctxt->eflags);
2463 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2465 ctxt->eflags |= X86_EFLAGS_CF;
2467 ctxt->eflags |= X86_EFLAGS_AF;
2468 return X86EMUL_CONTINUE;
2471 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2477 old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2478 old_eip = ctxt->_eip;
2480 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2481 if (load_segment_descriptor(ctxt, sel, VCPU_SREG_CS))
2482 return X86EMUL_CONTINUE;
2485 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
2487 ctxt->src.val = old_cs;
2489 if (rc != X86EMUL_CONTINUE)
2492 ctxt->src.val = old_eip;
2493 return em_push(ctxt);
2496 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
2500 ctxt->dst.type = OP_REG;
2501 ctxt->dst.addr.reg = &ctxt->_eip;
2502 ctxt->dst.bytes = ctxt->op_bytes;
2503 rc = emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
2504 if (rc != X86EMUL_CONTINUE)
2506 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], ctxt->src.val);
2507 return X86EMUL_CONTINUE;
2510 static int em_add(struct x86_emulate_ctxt *ctxt)
2512 emulate_2op_SrcV("add", ctxt->src, ctxt->dst, ctxt->eflags);
2513 return X86EMUL_CONTINUE;
2516 static int em_or(struct x86_emulate_ctxt *ctxt)
2518 emulate_2op_SrcV("or", ctxt->src, ctxt->dst, ctxt->eflags);
2519 return X86EMUL_CONTINUE;
2522 static int em_adc(struct x86_emulate_ctxt *ctxt)
2524 emulate_2op_SrcV("adc", ctxt->src, ctxt->dst, ctxt->eflags);
2525 return X86EMUL_CONTINUE;
2528 static int em_sbb(struct x86_emulate_ctxt *ctxt)
2530 emulate_2op_SrcV("sbb", ctxt->src, ctxt->dst, ctxt->eflags);
2531 return X86EMUL_CONTINUE;
2534 static int em_and(struct x86_emulate_ctxt *ctxt)
2536 emulate_2op_SrcV("and", ctxt->src, ctxt->dst, ctxt->eflags);
2537 return X86EMUL_CONTINUE;
2540 static int em_sub(struct x86_emulate_ctxt *ctxt)
2542 emulate_2op_SrcV("sub", ctxt->src, ctxt->dst, ctxt->eflags);
2543 return X86EMUL_CONTINUE;
2546 static int em_xor(struct x86_emulate_ctxt *ctxt)
2548 emulate_2op_SrcV("xor", ctxt->src, ctxt->dst, ctxt->eflags);
2549 return X86EMUL_CONTINUE;
2552 static int em_cmp(struct x86_emulate_ctxt *ctxt)
2554 emulate_2op_SrcV("cmp", ctxt->src, ctxt->dst, ctxt->eflags);
2555 /* Disable writeback. */
2556 ctxt->dst.type = OP_NONE;
2557 return X86EMUL_CONTINUE;
2560 static int em_test(struct x86_emulate_ctxt *ctxt)
2562 emulate_2op_SrcV("test", ctxt->src, ctxt->dst, ctxt->eflags);
2563 return X86EMUL_CONTINUE;
2566 static int em_xchg(struct x86_emulate_ctxt *ctxt)
2568 /* Write back the register source. */
2569 ctxt->src.val = ctxt->dst.val;
2570 write_register_operand(&ctxt->src);
2572 /* Write back the memory destination with implicit LOCK prefix. */
2573 ctxt->dst.val = ctxt->src.orig_val;
2574 ctxt->lock_prefix = 1;
2575 return X86EMUL_CONTINUE;
2578 static int em_imul(struct x86_emulate_ctxt *ctxt)
2580 emulate_2op_SrcV_nobyte("imul", ctxt->src, ctxt->dst, ctxt->eflags);
2581 return X86EMUL_CONTINUE;
2584 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
2586 ctxt->dst.val = ctxt->src2.val;
2587 return em_imul(ctxt);
2590 static int em_cwd(struct x86_emulate_ctxt *ctxt)
2592 ctxt->dst.type = OP_REG;
2593 ctxt->dst.bytes = ctxt->src.bytes;
2594 ctxt->dst.addr.reg = &ctxt->regs[VCPU_REGS_RDX];
2595 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
2597 return X86EMUL_CONTINUE;
2600 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
2604 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
2605 ctxt->regs[VCPU_REGS_RAX] = (u32)tsc;
2606 ctxt->regs[VCPU_REGS_RDX] = tsc >> 32;
2607 return X86EMUL_CONTINUE;
2610 static int em_mov(struct x86_emulate_ctxt *ctxt)
2612 ctxt->dst.val = ctxt->src.val;
2613 return X86EMUL_CONTINUE;
2616 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
2618 if (ctxt->modrm_reg > VCPU_SREG_GS)
2619 return emulate_ud(ctxt);
2621 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
2622 return X86EMUL_CONTINUE;
2625 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
2627 u16 sel = ctxt->src.val;
2629 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
2630 return emulate_ud(ctxt);
2632 if (ctxt->modrm_reg == VCPU_SREG_SS)
2633 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
2635 /* Disable writeback. */
2636 ctxt->dst.type = OP_NONE;
2637 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
2640 static int em_movdqu(struct x86_emulate_ctxt *ctxt)
2642 memcpy(&ctxt->dst.vec_val, &ctxt->src.vec_val, ctxt->op_bytes);
2643 return X86EMUL_CONTINUE;
2646 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
2651 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
2652 if (rc == X86EMUL_CONTINUE)
2653 ctxt->ops->invlpg(ctxt, linear);
2654 /* Disable writeback. */
2655 ctxt->dst.type = OP_NONE;
2656 return X86EMUL_CONTINUE;
2659 static int em_clts(struct x86_emulate_ctxt *ctxt)
2663 cr0 = ctxt->ops->get_cr(ctxt, 0);
2665 ctxt->ops->set_cr(ctxt, 0, cr0);
2666 return X86EMUL_CONTINUE;
2669 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
2673 if (ctxt->modrm_mod != 3 || ctxt->modrm_rm != 1)
2674 return X86EMUL_UNHANDLEABLE;
2676 rc = ctxt->ops->fix_hypercall(ctxt);
2677 if (rc != X86EMUL_CONTINUE)
2680 /* Let the processor re-execute the fixed hypercall */
2681 ctxt->_eip = ctxt->eip;
2682 /* Disable writeback. */
2683 ctxt->dst.type = OP_NONE;
2684 return X86EMUL_CONTINUE;
2687 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
2689 struct desc_ptr desc_ptr;
2692 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
2693 &desc_ptr.size, &desc_ptr.address,
2695 if (rc != X86EMUL_CONTINUE)
2697 ctxt->ops->set_gdt(ctxt, &desc_ptr);
2698 /* Disable writeback. */
2699 ctxt->dst.type = OP_NONE;
2700 return X86EMUL_CONTINUE;
2703 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
2707 rc = ctxt->ops->fix_hypercall(ctxt);
2709 /* Disable writeback. */
2710 ctxt->dst.type = OP_NONE;
2714 static int em_lidt(struct x86_emulate_ctxt *ctxt)
2716 struct desc_ptr desc_ptr;
2719 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
2720 &desc_ptr.size, &desc_ptr.address,
2722 if (rc != X86EMUL_CONTINUE)
2724 ctxt->ops->set_idt(ctxt, &desc_ptr);
2725 /* Disable writeback. */
2726 ctxt->dst.type = OP_NONE;
2727 return X86EMUL_CONTINUE;
2730 static int em_smsw(struct x86_emulate_ctxt *ctxt)
2732 ctxt->dst.bytes = 2;
2733 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
2734 return X86EMUL_CONTINUE;
2737 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
2739 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
2740 | (ctxt->src.val & 0x0f));
2741 ctxt->dst.type = OP_NONE;
2742 return X86EMUL_CONTINUE;
2745 static int em_loop(struct x86_emulate_ctxt *ctxt)
2747 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RCX], -1);
2748 if ((address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) != 0) &&
2749 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
2750 jmp_rel(ctxt, ctxt->src.val);
2752 return X86EMUL_CONTINUE;
2755 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
2757 if (address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) == 0)
2758 jmp_rel(ctxt, ctxt->src.val);
2760 return X86EMUL_CONTINUE;
2763 static int em_cli(struct x86_emulate_ctxt *ctxt)
2765 if (emulator_bad_iopl(ctxt))
2766 return emulate_gp(ctxt, 0);
2768 ctxt->eflags &= ~X86_EFLAGS_IF;
2769 return X86EMUL_CONTINUE;
2772 static int em_sti(struct x86_emulate_ctxt *ctxt)
2774 if (emulator_bad_iopl(ctxt))
2775 return emulate_gp(ctxt, 0);
2777 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
2778 ctxt->eflags |= X86_EFLAGS_IF;
2779 return X86EMUL_CONTINUE;
2782 static bool valid_cr(int nr)
2794 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
2796 if (!valid_cr(ctxt->modrm_reg))
2797 return emulate_ud(ctxt);
2799 return X86EMUL_CONTINUE;
2802 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
2804 u64 new_val = ctxt->src.val64;
2805 int cr = ctxt->modrm_reg;
2808 static u64 cr_reserved_bits[] = {
2809 0xffffffff00000000ULL,
2810 0, 0, 0, /* CR3 checked later */
2817 return emulate_ud(ctxt);
2819 if (new_val & cr_reserved_bits[cr])
2820 return emulate_gp(ctxt, 0);
2825 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
2826 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
2827 return emulate_gp(ctxt, 0);
2829 cr4 = ctxt->ops->get_cr(ctxt, 4);
2830 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2832 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
2833 !(cr4 & X86_CR4_PAE))
2834 return emulate_gp(ctxt, 0);
2841 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2842 if (efer & EFER_LMA)
2843 rsvd = CR3_L_MODE_RESERVED_BITS;
2844 else if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PAE)
2845 rsvd = CR3_PAE_RESERVED_BITS;
2846 else if (ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PG)
2847 rsvd = CR3_NONPAE_RESERVED_BITS;
2850 return emulate_gp(ctxt, 0);
2857 cr4 = ctxt->ops->get_cr(ctxt, 4);
2858 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2860 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
2861 return emulate_gp(ctxt, 0);
2867 return X86EMUL_CONTINUE;
2870 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
2874 ctxt->ops->get_dr(ctxt, 7, &dr7);
2876 /* Check if DR7.Global_Enable is set */
2877 return dr7 & (1 << 13);
2880 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
2882 int dr = ctxt->modrm_reg;
2886 return emulate_ud(ctxt);
2888 cr4 = ctxt->ops->get_cr(ctxt, 4);
2889 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
2890 return emulate_ud(ctxt);
2892 if (check_dr7_gd(ctxt))
2893 return emulate_db(ctxt);
2895 return X86EMUL_CONTINUE;
2898 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
2900 u64 new_val = ctxt->src.val64;
2901 int dr = ctxt->modrm_reg;
2903 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
2904 return emulate_gp(ctxt, 0);
2906 return check_dr_read(ctxt);
2909 static int check_svme(struct x86_emulate_ctxt *ctxt)
2913 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2915 if (!(efer & EFER_SVME))
2916 return emulate_ud(ctxt);
2918 return X86EMUL_CONTINUE;
2921 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
2923 u64 rax = ctxt->regs[VCPU_REGS_RAX];
2925 /* Valid physical address? */
2926 if (rax & 0xffff000000000000ULL)
2927 return emulate_gp(ctxt, 0);
2929 return check_svme(ctxt);
2932 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
2934 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
2936 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
2937 return emulate_ud(ctxt);
2939 return X86EMUL_CONTINUE;
2942 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
2944 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
2945 u64 rcx = ctxt->regs[VCPU_REGS_RCX];
2947 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
2949 return emulate_gp(ctxt, 0);
2951 return X86EMUL_CONTINUE;
2954 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
2956 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
2957 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
2958 return emulate_gp(ctxt, 0);
2960 return X86EMUL_CONTINUE;
2963 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
2965 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
2966 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
2967 return emulate_gp(ctxt, 0);
2969 return X86EMUL_CONTINUE;
2972 #define D(_y) { .flags = (_y) }
2973 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
2974 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
2975 .check_perm = (_p) }
2977 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
2978 #define G(_f, _g) { .flags = ((_f) | Group), .u.group = (_g) }
2979 #define GD(_f, _g) { .flags = ((_f) | GroupDual), .u.gdual = (_g) }
2980 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
2981 #define II(_f, _e, _i) \
2982 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
2983 #define IIP(_f, _e, _i, _p) \
2984 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
2985 .check_perm = (_p) }
2986 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
2988 #define D2bv(_f) D((_f) | ByteOp), D(_f)
2989 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
2990 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
2992 #define I6ALU(_f, _e) I2bv((_f) | DstMem | SrcReg | ModRM, _e), \
2993 I2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
2994 I2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
2996 static struct opcode group7_rm1[] = {
2997 DI(SrcNone | ModRM | Priv, monitor),
2998 DI(SrcNone | ModRM | Priv, mwait),
3002 static struct opcode group7_rm3[] = {
3003 DIP(SrcNone | ModRM | Prot | Priv, vmrun, check_svme_pa),
3004 II(SrcNone | ModRM | Prot | VendorSpecific, em_vmmcall, vmmcall),
3005 DIP(SrcNone | ModRM | Prot | Priv, vmload, check_svme_pa),
3006 DIP(SrcNone | ModRM | Prot | Priv, vmsave, check_svme_pa),
3007 DIP(SrcNone | ModRM | Prot | Priv, stgi, check_svme),
3008 DIP(SrcNone | ModRM | Prot | Priv, clgi, check_svme),
3009 DIP(SrcNone | ModRM | Prot | Priv, skinit, check_svme),
3010 DIP(SrcNone | ModRM | Prot | Priv, invlpga, check_svme),
3013 static struct opcode group7_rm7[] = {
3015 DIP(SrcNone | ModRM, rdtscp, check_rdtsc),
3019 static struct opcode group1[] = {
3030 static struct opcode group1A[] = {
3031 D(DstMem | SrcNone | ModRM | Mov | Stack), N, N, N, N, N, N, N,
3034 static struct opcode group3[] = {
3035 D(DstMem | SrcImm | ModRM), D(DstMem | SrcImm | ModRM),
3036 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
3037 X4(D(SrcMem | ModRM)),
3040 static struct opcode group4[] = {
3041 D(ByteOp | DstMem | SrcNone | ModRM | Lock), D(ByteOp | DstMem | SrcNone | ModRM | Lock),
3045 static struct opcode group5[] = {
3046 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
3047 D(SrcMem | ModRM | Stack),
3048 I(SrcMemFAddr | ModRM | ImplicitOps | Stack, em_call_far),
3049 D(SrcMem | ModRM | Stack), D(SrcMemFAddr | ModRM | ImplicitOps),
3050 D(SrcMem | ModRM | Stack), N,
3053 static struct opcode group6[] = {
3054 DI(ModRM | Prot, sldt),
3055 DI(ModRM | Prot, str),
3056 DI(ModRM | Prot | Priv, lldt),
3057 DI(ModRM | Prot | Priv, ltr),
3061 static struct group_dual group7 = { {
3062 DI(ModRM | Mov | DstMem | Priv, sgdt),
3063 DI(ModRM | Mov | DstMem | Priv, sidt),
3064 II(ModRM | SrcMem | Priv, em_lgdt, lgdt),
3065 II(ModRM | SrcMem | Priv, em_lidt, lidt),
3066 II(SrcNone | ModRM | DstMem | Mov, em_smsw, smsw), N,
3067 II(SrcMem16 | ModRM | Mov | Priv, em_lmsw, lmsw),
3068 II(SrcMem | ModRM | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3070 I(SrcNone | ModRM | Priv | VendorSpecific, em_vmcall),
3072 N, EXT(0, group7_rm3),
3073 II(SrcNone | ModRM | DstMem | Mov, em_smsw, smsw), N,
3074 II(SrcMem16 | ModRM | Mov | Priv, em_lmsw, lmsw), EXT(0, group7_rm7),
3077 static struct opcode group8[] = {
3079 D(DstMem | SrcImmByte | ModRM), D(DstMem | SrcImmByte | ModRM | Lock),
3080 D(DstMem | SrcImmByte | ModRM | Lock), D(DstMem | SrcImmByte | ModRM | Lock),
3083 static struct group_dual group9 = { {
3084 N, D(DstMem64 | ModRM | Lock), N, N, N, N, N, N,
3086 N, N, N, N, N, N, N, N,
3089 static struct opcode group11[] = {
3090 I(DstMem | SrcImm | ModRM | Mov, em_mov), X7(D(Undefined)),
3093 static struct gprefix pfx_0f_6f_0f_7f = {
3094 N, N, N, I(Sse, em_movdqu),
3097 static struct opcode opcode_table[256] = {
3099 I6ALU(Lock, em_add),
3100 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
3103 D(ImplicitOps | Stack | No64), N,
3105 I6ALU(Lock, em_adc),
3106 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
3108 I6ALU(Lock, em_sbb),
3109 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
3111 I6ALU(Lock, em_and), N, N,
3113 I6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3115 I6ALU(Lock, em_xor), N, N,
3117 I6ALU(0, em_cmp), N, N,
3121 X8(I(SrcReg | Stack, em_push)),
3123 X8(I(DstReg | Stack, em_pop)),
3125 I(ImplicitOps | Stack | No64, em_pusha),
3126 I(ImplicitOps | Stack | No64, em_popa),
3127 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
3130 I(SrcImm | Mov | Stack, em_push),
3131 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3132 I(SrcImmByte | Mov | Stack, em_push),
3133 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3134 D2bvIP(DstDI | SrcDX | Mov | String, ins, check_perm_in), /* insb, insw/insd */
3135 D2bvIP(SrcSI | DstDX | String, outs, check_perm_out), /* outsb, outsw/outsd */
3139 G(ByteOp | DstMem | SrcImm | ModRM | Group, group1),
3140 G(DstMem | SrcImm | ModRM | Group, group1),
3141 G(ByteOp | DstMem | SrcImm | ModRM | No64 | Group, group1),
3142 G(DstMem | SrcImmByte | ModRM | Group, group1),
3143 I2bv(DstMem | SrcReg | ModRM, em_test),
3144 I2bv(DstMem | SrcReg | ModRM | Lock, em_xchg),
3146 I2bv(DstMem | SrcReg | ModRM | Mov, em_mov),
3147 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3148 I(DstMem | SrcNone | ModRM | Mov, em_mov_rm_sreg),
3149 D(ModRM | SrcMem | NoAccess | DstReg),
3150 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
3153 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3155 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3156 I(SrcImmFAddr | No64, em_call_far), N,
3157 II(ImplicitOps | Stack, em_pushf, pushf),
3158 II(ImplicitOps | Stack, em_popf, popf), N, N,
3160 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3161 I2bv(DstMem | SrcAcc | Mov | MemAbs, em_mov),
3162 I2bv(SrcSI | DstDI | Mov | String, em_mov),
3163 I2bv(SrcSI | DstDI | String, em_cmp),
3165 I2bv(DstAcc | SrcImm, em_test),
3166 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
3167 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3168 I2bv(SrcAcc | DstDI | String, em_cmp),
3170 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3172 X8(I(DstReg | SrcImm | Mov, em_mov)),
3174 D2bv(DstMem | SrcImmByte | ModRM),
3175 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3176 I(ImplicitOps | Stack, em_ret),
3177 D(DstReg | SrcMemFAddr | ModRM | No64), D(DstReg | SrcMemFAddr | ModRM | No64),
3178 G(ByteOp, group11), G(0, group11),
3180 N, N, N, I(ImplicitOps | Stack, em_ret_far),
3181 D(ImplicitOps), DI(SrcImmByte, intn),
3182 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
3184 D2bv(DstMem | SrcOne | ModRM), D2bv(DstMem | ModRM),
3187 N, N, N, N, N, N, N, N,
3189 X3(I(SrcImmByte, em_loop)),
3190 I(SrcImmByte, em_jcxz),
3191 D2bvIP(SrcImmUByte | DstAcc, in, check_perm_in),
3192 D2bvIP(SrcAcc | DstImmUByte, out, check_perm_out),
3194 D(SrcImm | Stack), D(SrcImm | ImplicitOps),
3195 I(SrcImmFAddr | No64, em_jmp_far), D(SrcImmByte | ImplicitOps),
3196 D2bvIP(SrcDX | DstAcc, in, check_perm_in),
3197 D2bvIP(SrcAcc | DstDX, out, check_perm_out),
3199 N, DI(ImplicitOps, icebp), N, N,
3200 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
3201 G(ByteOp, group3), G(0, group3),
3203 D(ImplicitOps), D(ImplicitOps),
3204 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
3205 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
3208 static struct opcode twobyte_table[256] = {
3210 G(0, group6), GD(0, &group7), N, N,
3211 N, I(ImplicitOps | VendorSpecific, em_syscall),
3212 II(ImplicitOps | Priv, em_clts, clts), N,
3213 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3214 N, D(ImplicitOps | ModRM), N, N,
3216 N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
3218 DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
3219 DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
3220 DIP(ModRM | SrcMem | Priv | Op3264, cr_write, check_cr_write),
3221 DIP(ModRM | SrcMem | Priv | Op3264, dr_write, check_dr_write),
3223 N, N, N, N, N, N, N, N,
3225 DI(ImplicitOps | Priv, wrmsr),
3226 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
3227 DI(ImplicitOps | Priv, rdmsr),
3228 DIP(ImplicitOps | Priv, rdpmc, check_rdpmc),
3229 I(ImplicitOps | VendorSpecific, em_sysenter),
3230 I(ImplicitOps | Priv | VendorSpecific, em_sysexit),
3232 N, N, N, N, N, N, N, N,
3234 X16(D(DstReg | SrcMem | ModRM | Mov)),
3236 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3241 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3246 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3250 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3252 D(ImplicitOps | Stack), D(ImplicitOps | Stack),
3253 DI(ImplicitOps, cpuid), D(DstMem | SrcReg | ModRM | BitOp),
3254 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3255 D(DstMem | SrcReg | Src2CL | ModRM), N, N,
3257 D(ImplicitOps | Stack), D(ImplicitOps | Stack),
3258 DI(ImplicitOps, rsm), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3259 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3260 D(DstMem | SrcReg | Src2CL | ModRM),
3261 D(ModRM), I(DstReg | SrcMem | ModRM, em_imul),
3263 D2bv(DstMem | SrcReg | ModRM | Lock),
3264 D(DstReg | SrcMemFAddr | ModRM), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3265 D(DstReg | SrcMemFAddr | ModRM), D(DstReg | SrcMemFAddr | ModRM),
3266 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3269 G(BitOp, group8), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3270 D(DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
3271 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3273 D2bv(DstMem | SrcReg | ModRM | Lock),
3274 N, D(DstMem | SrcReg | ModRM | Mov),
3275 N, N, N, GD(0, &group9),
3276 N, N, N, N, N, N, N, N,
3278 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3280 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3282 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
3298 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
3302 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3308 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
3309 unsigned size, bool sign_extension)
3311 int rc = X86EMUL_CONTINUE;
3315 op->addr.mem.ea = ctxt->_eip;
3316 /* NB. Immediates are sign-extended as necessary. */
3317 switch (op->bytes) {
3319 op->val = insn_fetch(s8, ctxt);
3322 op->val = insn_fetch(s16, ctxt);
3325 op->val = insn_fetch(s32, ctxt);
3328 if (!sign_extension) {
3329 switch (op->bytes) {
3337 op->val &= 0xffffffff;
3345 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
3347 int rc = X86EMUL_CONTINUE;
3348 int mode = ctxt->mode;
3349 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
3350 bool op_prefix = false;
3351 struct opcode opcode;
3352 struct operand memop = { .type = OP_NONE }, *memopp = NULL;
3354 ctxt->_eip = ctxt->eip;
3355 ctxt->fetch.start = ctxt->_eip;
3356 ctxt->fetch.end = ctxt->fetch.start + insn_len;
3358 memcpy(ctxt->fetch.data, insn, insn_len);
3361 case X86EMUL_MODE_REAL:
3362 case X86EMUL_MODE_VM86:
3363 case X86EMUL_MODE_PROT16:
3364 def_op_bytes = def_ad_bytes = 2;
3366 case X86EMUL_MODE_PROT32:
3367 def_op_bytes = def_ad_bytes = 4;
3369 #ifdef CONFIG_X86_64
3370 case X86EMUL_MODE_PROT64:
3379 ctxt->op_bytes = def_op_bytes;
3380 ctxt->ad_bytes = def_ad_bytes;
3382 /* Legacy prefixes. */
3384 switch (ctxt->b = insn_fetch(u8, ctxt)) {
3385 case 0x66: /* operand-size override */
3387 /* switch between 2/4 bytes */
3388 ctxt->op_bytes = def_op_bytes ^ 6;
3390 case 0x67: /* address-size override */
3391 if (mode == X86EMUL_MODE_PROT64)
3392 /* switch between 4/8 bytes */
3393 ctxt->ad_bytes = def_ad_bytes ^ 12;
3395 /* switch between 2/4 bytes */
3396 ctxt->ad_bytes = def_ad_bytes ^ 6;
3398 case 0x26: /* ES override */
3399 case 0x2e: /* CS override */
3400 case 0x36: /* SS override */
3401 case 0x3e: /* DS override */
3402 set_seg_override(ctxt, (ctxt->b >> 3) & 3);
3404 case 0x64: /* FS override */
3405 case 0x65: /* GS override */
3406 set_seg_override(ctxt, ctxt->b & 7);
3408 case 0x40 ... 0x4f: /* REX */
3409 if (mode != X86EMUL_MODE_PROT64)
3411 ctxt->rex_prefix = ctxt->b;
3413 case 0xf0: /* LOCK */
3414 ctxt->lock_prefix = 1;
3416 case 0xf2: /* REPNE/REPNZ */
3417 case 0xf3: /* REP/REPE/REPZ */
3418 ctxt->rep_prefix = ctxt->b;
3424 /* Any legacy prefix after a REX prefix nullifies its effect. */
3426 ctxt->rex_prefix = 0;
3432 if (ctxt->rex_prefix & 8)
3433 ctxt->op_bytes = 8; /* REX.W */
3435 /* Opcode byte(s). */
3436 opcode = opcode_table[ctxt->b];
3437 /* Two-byte opcode? */
3438 if (ctxt->b == 0x0f) {
3440 ctxt->b = insn_fetch(u8, ctxt);
3441 opcode = twobyte_table[ctxt->b];
3443 ctxt->d = opcode.flags;
3445 while (ctxt->d & GroupMask) {
3446 switch (ctxt->d & GroupMask) {
3448 ctxt->modrm = insn_fetch(u8, ctxt);
3450 goffset = (ctxt->modrm >> 3) & 7;
3451 opcode = opcode.u.group[goffset];
3454 ctxt->modrm = insn_fetch(u8, ctxt);
3456 goffset = (ctxt->modrm >> 3) & 7;
3457 if ((ctxt->modrm >> 6) == 3)
3458 opcode = opcode.u.gdual->mod3[goffset];
3460 opcode = opcode.u.gdual->mod012[goffset];
3463 goffset = ctxt->modrm & 7;
3464 opcode = opcode.u.group[goffset];
3467 if (ctxt->rep_prefix && op_prefix)
3468 return X86EMUL_UNHANDLEABLE;
3469 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
3470 switch (simd_prefix) {
3471 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
3472 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
3473 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
3474 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
3478 return X86EMUL_UNHANDLEABLE;
3481 ctxt->d &= ~GroupMask;
3482 ctxt->d |= opcode.flags;
3485 ctxt->execute = opcode.u.execute;
3486 ctxt->check_perm = opcode.check_perm;
3487 ctxt->intercept = opcode.intercept;
3490 if (ctxt->d == 0 || (ctxt->d & Undefined))
3493 if (!(ctxt->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
3496 if (mode == X86EMUL_MODE_PROT64 && (ctxt->d & Stack))
3499 if (ctxt->d & Op3264) {
3500 if (mode == X86EMUL_MODE_PROT64)
3507 ctxt->op_bytes = 16;
3509 /* ModRM and SIB bytes. */
3510 if (ctxt->d & ModRM) {
3511 rc = decode_modrm(ctxt, &memop);
3512 if (!ctxt->has_seg_override)
3513 set_seg_override(ctxt, ctxt->modrm_seg);
3514 } else if (ctxt->d & MemAbs)
3515 rc = decode_abs(ctxt, &memop);
3516 if (rc != X86EMUL_CONTINUE)
3519 if (!ctxt->has_seg_override)
3520 set_seg_override(ctxt, VCPU_SREG_DS);
3522 memop.addr.mem.seg = seg_override(ctxt);
3524 if (memop.type == OP_MEM && ctxt->ad_bytes != 8)
3525 memop.addr.mem.ea = (u32)memop.addr.mem.ea;
3528 * Decode and fetch the source operand: register, memory
3531 switch (ctxt->d & SrcMask) {
3535 decode_register_operand(ctxt, &ctxt->src, 0);
3544 memop.bytes = (ctxt->d & ByteOp) ? 1 :
3548 memopp = &ctxt->src;
3551 rc = decode_imm(ctxt, &ctxt->src, 2, false);
3554 rc = decode_imm(ctxt, &ctxt->src, imm_size(ctxt), true);
3557 rc = decode_imm(ctxt, &ctxt->src, imm_size(ctxt), false);
3560 rc = decode_imm(ctxt, &ctxt->src, 1, true);
3563 rc = decode_imm(ctxt, &ctxt->src, 1, false);
3566 ctxt->src.type = OP_REG;
3567 ctxt->src.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3568 ctxt->src.addr.reg = &ctxt->regs[VCPU_REGS_RAX];
3569 fetch_register_operand(&ctxt->src);
3572 ctxt->src.bytes = 1;
3576 ctxt->src.type = OP_MEM;
3577 ctxt->src.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3578 ctxt->src.addr.mem.ea =
3579 register_address(ctxt, ctxt->regs[VCPU_REGS_RSI]);
3580 ctxt->src.addr.mem.seg = seg_override(ctxt);
3584 ctxt->src.type = OP_IMM;
3585 ctxt->src.addr.mem.ea = ctxt->_eip;
3586 ctxt->src.bytes = ctxt->op_bytes + 2;
3587 insn_fetch_arr(ctxt->src.valptr, ctxt->src.bytes, ctxt);
3590 memop.bytes = ctxt->op_bytes + 2;
3594 ctxt->src.type = OP_REG;
3595 ctxt->src.bytes = 2;
3596 ctxt->src.addr.reg = &ctxt->regs[VCPU_REGS_RDX];
3597 fetch_register_operand(&ctxt->src);
3601 if (rc != X86EMUL_CONTINUE)
3605 * Decode and fetch the second source operand: register, memory
3608 switch (ctxt->d & Src2Mask) {
3612 ctxt->src2.bytes = 1;
3613 ctxt->src2.val = ctxt->regs[VCPU_REGS_RCX] & 0xff;
3616 rc = decode_imm(ctxt, &ctxt->src2, 1, true);
3619 ctxt->src2.bytes = 1;
3623 rc = decode_imm(ctxt, &ctxt->src2, imm_size(ctxt), true);
3627 if (rc != X86EMUL_CONTINUE)
3630 /* Decode and fetch the destination operand: register or memory. */
3631 switch (ctxt->d & DstMask) {
3633 decode_register_operand(ctxt, &ctxt->dst,
3634 ctxt->twobyte && (ctxt->b == 0xb6 || ctxt->b == 0xb7));
3637 ctxt->dst.type = OP_IMM;
3638 ctxt->dst.addr.mem.ea = ctxt->_eip;
3639 ctxt->dst.bytes = 1;
3640 ctxt->dst.val = insn_fetch(u8, ctxt);
3645 memopp = &ctxt->dst;
3646 if ((ctxt->d & DstMask) == DstMem64)
3647 ctxt->dst.bytes = 8;
3649 ctxt->dst.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3650 if (ctxt->d & BitOp)
3651 fetch_bit_operand(ctxt);
3652 ctxt->dst.orig_val = ctxt->dst.val;
3655 ctxt->dst.type = OP_REG;
3656 ctxt->dst.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3657 ctxt->dst.addr.reg = &ctxt->regs[VCPU_REGS_RAX];
3658 fetch_register_operand(&ctxt->dst);
3659 ctxt->dst.orig_val = ctxt->dst.val;
3662 ctxt->dst.type = OP_MEM;
3663 ctxt->dst.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3664 ctxt->dst.addr.mem.ea =
3665 register_address(ctxt, ctxt->regs[VCPU_REGS_RDI]);
3666 ctxt->dst.addr.mem.seg = VCPU_SREG_ES;
3670 ctxt->dst.type = OP_REG;
3671 ctxt->dst.bytes = 2;
3672 ctxt->dst.addr.reg = &ctxt->regs[VCPU_REGS_RDX];
3673 fetch_register_operand(&ctxt->dst);
3676 /* Special instructions do their own operand decoding. */
3678 ctxt->dst.type = OP_NONE; /* Disable writeback. */
3683 if (memopp && memopp->type == OP_MEM && ctxt->rip_relative)
3684 memopp->addr.mem.ea += ctxt->_eip;
3686 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
3689 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
3691 /* The second termination condition only applies for REPE
3692 * and REPNE. Test if the repeat string operation prefix is
3693 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
3694 * corresponding termination condition according to:
3695 * - if REPE/REPZ and ZF = 0 then done
3696 * - if REPNE/REPNZ and ZF = 1 then done
3698 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
3699 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
3700 && (((ctxt->rep_prefix == REPE_PREFIX) &&
3701 ((ctxt->eflags & EFLG_ZF) == 0))
3702 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
3703 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
3709 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
3711 struct x86_emulate_ops *ops = ctxt->ops;
3713 int rc = X86EMUL_CONTINUE;
3714 int saved_dst_type = ctxt->dst.type;
3716 ctxt->mem_read.pos = 0;
3718 if (ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) {
3719 rc = emulate_ud(ctxt);
3723 /* LOCK prefix is allowed only with some instructions */
3724 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
3725 rc = emulate_ud(ctxt);
3729 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
3730 rc = emulate_ud(ctxt);
3735 && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)
3736 || !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
3737 rc = emulate_ud(ctxt);
3741 if ((ctxt->d & Sse) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
3742 rc = emulate_nm(ctxt);
3746 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
3747 rc = emulator_check_intercept(ctxt, ctxt->intercept,
3748 X86_ICPT_PRE_EXCEPT);
3749 if (rc != X86EMUL_CONTINUE)
3753 /* Privileged instruction can be executed only in CPL=0 */
3754 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
3755 rc = emulate_gp(ctxt, 0);
3759 /* Instruction can only be executed in protected mode */
3760 if ((ctxt->d & Prot) && !(ctxt->mode & X86EMUL_MODE_PROT)) {
3761 rc = emulate_ud(ctxt);
3765 /* Do instruction specific permission checks */
3766 if (ctxt->check_perm) {
3767 rc = ctxt->check_perm(ctxt);
3768 if (rc != X86EMUL_CONTINUE)
3772 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
3773 rc = emulator_check_intercept(ctxt, ctxt->intercept,
3774 X86_ICPT_POST_EXCEPT);
3775 if (rc != X86EMUL_CONTINUE)
3779 if (ctxt->rep_prefix && (ctxt->d & String)) {
3780 /* All REP prefixes have the same first termination condition */
3781 if (address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) == 0) {
3782 ctxt->eip = ctxt->_eip;
3787 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
3788 rc = segmented_read(ctxt, ctxt->src.addr.mem,
3789 ctxt->src.valptr, ctxt->src.bytes);
3790 if (rc != X86EMUL_CONTINUE)
3792 ctxt->src.orig_val64 = ctxt->src.val64;
3795 if (ctxt->src2.type == OP_MEM) {
3796 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
3797 &ctxt->src2.val, ctxt->src2.bytes);
3798 if (rc != X86EMUL_CONTINUE)
3802 if ((ctxt->d & DstMask) == ImplicitOps)
3806 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
3807 /* optimisation - avoid slow emulated read if Mov */
3808 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
3809 &ctxt->dst.val, ctxt->dst.bytes);
3810 if (rc != X86EMUL_CONTINUE)
3813 ctxt->dst.orig_val = ctxt->dst.val;
3817 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
3818 rc = emulator_check_intercept(ctxt, ctxt->intercept,
3819 X86_ICPT_POST_MEMACCESS);
3820 if (rc != X86EMUL_CONTINUE)
3824 if (ctxt->execute) {
3825 rc = ctxt->execute(ctxt);
3826 if (rc != X86EMUL_CONTINUE)
3835 case 0x06: /* push es */
3836 rc = emulate_push_sreg(ctxt, VCPU_SREG_ES);
3838 case 0x07: /* pop es */
3839 rc = emulate_pop_sreg(ctxt, VCPU_SREG_ES);
3841 case 0x0e: /* push cs */
3842 rc = emulate_push_sreg(ctxt, VCPU_SREG_CS);
3844 case 0x16: /* push ss */
3845 rc = emulate_push_sreg(ctxt, VCPU_SREG_SS);
3847 case 0x17: /* pop ss */
3848 rc = emulate_pop_sreg(ctxt, VCPU_SREG_SS);
3850 case 0x1e: /* push ds */
3851 rc = emulate_push_sreg(ctxt, VCPU_SREG_DS);
3853 case 0x1f: /* pop ds */
3854 rc = emulate_pop_sreg(ctxt, VCPU_SREG_DS);
3856 case 0x40 ... 0x47: /* inc r16/r32 */
3857 emulate_1op("inc", ctxt->dst, ctxt->eflags);
3859 case 0x48 ... 0x4f: /* dec r16/r32 */
3860 emulate_1op("dec", ctxt->dst, ctxt->eflags);
3862 case 0x63: /* movsxd */
3863 if (ctxt->mode != X86EMUL_MODE_PROT64)
3864 goto cannot_emulate;
3865 ctxt->dst.val = (s32) ctxt->src.val;
3867 case 0x6c: /* insb */
3868 case 0x6d: /* insw/insd */
3869 ctxt->src.val = ctxt->regs[VCPU_REGS_RDX];
3871 case 0x6e: /* outsb */
3872 case 0x6f: /* outsw/outsd */
3873 ctxt->dst.val = ctxt->regs[VCPU_REGS_RDX];
3876 case 0x70 ... 0x7f: /* jcc (short) */
3877 if (test_cc(ctxt->b, ctxt->eflags))
3878 jmp_rel(ctxt, ctxt->src.val);
3880 case 0x8d: /* lea r16/r32, m */
3881 ctxt->dst.val = ctxt->src.addr.mem.ea;
3883 case 0x8f: /* pop (sole member of Grp1a) */
3884 rc = em_grp1a(ctxt);
3886 case 0x90 ... 0x97: /* nop / xchg reg, rax */
3887 if (ctxt->dst.addr.reg == &ctxt->regs[VCPU_REGS_RAX])
3891 case 0x98: /* cbw/cwde/cdqe */
3892 switch (ctxt->op_bytes) {
3893 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
3894 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
3895 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
3901 case 0xc4: /* les */
3902 rc = emulate_load_segment(ctxt, VCPU_SREG_ES);
3904 case 0xc5: /* lds */
3905 rc = emulate_load_segment(ctxt, VCPU_SREG_DS);
3907 case 0xcc: /* int3 */
3908 rc = emulate_int(ctxt, 3);
3910 case 0xcd: /* int n */
3911 rc = emulate_int(ctxt, ctxt->src.val);
3913 case 0xce: /* into */
3914 if (ctxt->eflags & EFLG_OF)
3915 rc = emulate_int(ctxt, 4);
3917 case 0xd0 ... 0xd1: /* Grp2 */
3920 case 0xd2 ... 0xd3: /* Grp2 */
3921 ctxt->src.val = ctxt->regs[VCPU_REGS_RCX];
3924 case 0xe4: /* inb */
3927 case 0xe6: /* outb */
3928 case 0xe7: /* out */
3930 case 0xe8: /* call (near) */ {
3931 long int rel = ctxt->src.val;
3932 ctxt->src.val = (unsigned long) ctxt->_eip;
3937 case 0xe9: /* jmp rel */
3938 case 0xeb: /* jmp rel short */
3939 jmp_rel(ctxt, ctxt->src.val);
3940 ctxt->dst.type = OP_NONE; /* Disable writeback. */
3942 case 0xec: /* in al,dx */
3943 case 0xed: /* in (e/r)ax,dx */
3945 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3947 goto done; /* IO is needed */
3949 case 0xee: /* out dx,al */
3950 case 0xef: /* out dx,(e/r)ax */
3952 ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3954 ctxt->dst.type = OP_NONE; /* Disable writeback. */
3956 case 0xf4: /* hlt */
3957 ctxt->ops->halt(ctxt);
3959 case 0xf5: /* cmc */
3960 /* complement carry flag from eflags reg */
3961 ctxt->eflags ^= EFLG_CF;
3963 case 0xf6 ... 0xf7: /* Grp3 */
3966 case 0xf8: /* clc */
3967 ctxt->eflags &= ~EFLG_CF;
3969 case 0xf9: /* stc */
3970 ctxt->eflags |= EFLG_CF;
3972 case 0xfc: /* cld */
3973 ctxt->eflags &= ~EFLG_DF;
3975 case 0xfd: /* std */
3976 ctxt->eflags |= EFLG_DF;
3978 case 0xfe: /* Grp4 */
3979 rc = em_grp45(ctxt);
3981 case 0xff: /* Grp5 */
3982 rc = em_grp45(ctxt);
3985 goto cannot_emulate;
3988 if (rc != X86EMUL_CONTINUE)
3992 rc = writeback(ctxt);
3993 if (rc != X86EMUL_CONTINUE)
3997 * restore dst type in case the decoding will be reused
3998 * (happens for string instruction )
4000 ctxt->dst.type = saved_dst_type;
4002 if ((ctxt->d & SrcMask) == SrcSI)
4003 string_addr_inc(ctxt, seg_override(ctxt),
4004 VCPU_REGS_RSI, &ctxt->src);
4006 if ((ctxt->d & DstMask) == DstDI)
4007 string_addr_inc(ctxt, VCPU_SREG_ES, VCPU_REGS_RDI,
4010 if (ctxt->rep_prefix && (ctxt->d & String)) {
4011 struct read_cache *r = &ctxt->io_read;
4012 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RCX], -1);
4014 if (!string_insn_completed(ctxt)) {
4016 * Re-enter guest when pio read ahead buffer is empty
4017 * or, if it is not used, after each 1024 iteration.
4019 if ((r->end != 0 || ctxt->regs[VCPU_REGS_RCX] & 0x3ff) &&
4020 (r->end == 0 || r->end != r->pos)) {
4022 * Reset read cache. Usually happens before
4023 * decode, but since instruction is restarted
4024 * we have to do it here.
4026 ctxt->mem_read.end = 0;
4027 return EMULATION_RESTART;
4029 goto done; /* skip rip writeback */
4033 ctxt->eip = ctxt->_eip;
4036 if (rc == X86EMUL_PROPAGATE_FAULT)
4037 ctxt->have_exception = true;
4038 if (rc == X86EMUL_INTERCEPTED)
4039 return EMULATION_INTERCEPTED;
4041 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4045 case 0x09: /* wbinvd */
4046 (ctxt->ops->wbinvd)(ctxt);
4048 case 0x08: /* invd */
4049 case 0x0d: /* GrpP (prefetch) */
4050 case 0x18: /* Grp16 (prefetch/nop) */
4052 case 0x20: /* mov cr, reg */
4053 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
4055 case 0x21: /* mov from dr to reg */
4056 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
4058 case 0x22: /* mov reg, cr */
4059 if (ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val)) {
4060 emulate_gp(ctxt, 0);
4061 rc = X86EMUL_PROPAGATE_FAULT;
4064 ctxt->dst.type = OP_NONE;
4066 case 0x23: /* mov from reg to dr */
4067 if (ops->set_dr(ctxt, ctxt->modrm_reg, ctxt->src.val &
4068 ((ctxt->mode == X86EMUL_MODE_PROT64) ?
4069 ~0ULL : ~0U)) < 0) {
4070 /* #UD condition is already handled by the code above */
4071 emulate_gp(ctxt, 0);
4072 rc = X86EMUL_PROPAGATE_FAULT;
4076 ctxt->dst.type = OP_NONE; /* no writeback */
4080 msr_data = (u32)ctxt->regs[VCPU_REGS_RAX]
4081 | ((u64)ctxt->regs[VCPU_REGS_RDX] << 32);
4082 if (ops->set_msr(ctxt, ctxt->regs[VCPU_REGS_RCX], msr_data)) {
4083 emulate_gp(ctxt, 0);
4084 rc = X86EMUL_PROPAGATE_FAULT;
4087 rc = X86EMUL_CONTINUE;
4091 if (ops->get_msr(ctxt, ctxt->regs[VCPU_REGS_RCX], &msr_data)) {
4092 emulate_gp(ctxt, 0);
4093 rc = X86EMUL_PROPAGATE_FAULT;
4096 ctxt->regs[VCPU_REGS_RAX] = (u32)msr_data;
4097 ctxt->regs[VCPU_REGS_RDX] = msr_data >> 32;
4099 rc = X86EMUL_CONTINUE;
4101 case 0x40 ... 0x4f: /* cmov */
4102 ctxt->dst.val = ctxt->dst.orig_val = ctxt->src.val;
4103 if (!test_cc(ctxt->b, ctxt->eflags))
4104 ctxt->dst.type = OP_NONE; /* no writeback */
4106 case 0x80 ... 0x8f: /* jnz rel, etc*/
4107 if (test_cc(ctxt->b, ctxt->eflags))
4108 jmp_rel(ctxt, ctxt->src.val);
4110 case 0x90 ... 0x9f: /* setcc r/m8 */
4111 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
4113 case 0xa0: /* push fs */
4114 rc = emulate_push_sreg(ctxt, VCPU_SREG_FS);
4116 case 0xa1: /* pop fs */
4117 rc = emulate_pop_sreg(ctxt, VCPU_SREG_FS);
4121 ctxt->dst.type = OP_NONE;
4122 /* only subword offset */
4123 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
4124 emulate_2op_SrcV_nobyte("bt", ctxt->src, ctxt->dst, ctxt->eflags);
4126 case 0xa4: /* shld imm8, r, r/m */
4127 case 0xa5: /* shld cl, r, r/m */
4128 emulate_2op_cl("shld", ctxt->src2, ctxt->src, ctxt->dst, ctxt->eflags);
4130 case 0xa8: /* push gs */
4131 rc = emulate_push_sreg(ctxt, VCPU_SREG_GS);
4133 case 0xa9: /* pop gs */
4134 rc = emulate_pop_sreg(ctxt, VCPU_SREG_GS);
4138 emulate_2op_SrcV_nobyte("bts", ctxt->src, ctxt->dst, ctxt->eflags);
4140 case 0xac: /* shrd imm8, r, r/m */
4141 case 0xad: /* shrd cl, r, r/m */
4142 emulate_2op_cl("shrd", ctxt->src2, ctxt->src, ctxt->dst, ctxt->eflags);
4144 case 0xae: /* clflush */
4146 case 0xb0 ... 0xb1: /* cmpxchg */
4148 * Save real source value, then compare EAX against
4151 ctxt->src.orig_val = ctxt->src.val;
4152 ctxt->src.val = ctxt->regs[VCPU_REGS_RAX];
4153 emulate_2op_SrcV("cmp", ctxt->src, ctxt->dst, ctxt->eflags);
4154 if (ctxt->eflags & EFLG_ZF) {
4155 /* Success: write back to memory. */
4156 ctxt->dst.val = ctxt->src.orig_val;
4158 /* Failure: write the value we saw to EAX. */
4159 ctxt->dst.type = OP_REG;
4160 ctxt->dst.addr.reg = (unsigned long *)&ctxt->regs[VCPU_REGS_RAX];
4163 case 0xb2: /* lss */
4164 rc = emulate_load_segment(ctxt, VCPU_SREG_SS);
4168 emulate_2op_SrcV_nobyte("btr", ctxt->src, ctxt->dst, ctxt->eflags);
4170 case 0xb4: /* lfs */
4171 rc = emulate_load_segment(ctxt, VCPU_SREG_FS);
4173 case 0xb5: /* lgs */
4174 rc = emulate_load_segment(ctxt, VCPU_SREG_GS);
4176 case 0xb6 ... 0xb7: /* movzx */
4177 ctxt->dst.bytes = ctxt->op_bytes;
4178 ctxt->dst.val = (ctxt->d & ByteOp) ? (u8) ctxt->src.val
4179 : (u16) ctxt->src.val;
4181 case 0xba: /* Grp8 */
4182 switch (ctxt->modrm_reg & 3) {
4195 emulate_2op_SrcV_nobyte("btc", ctxt->src, ctxt->dst, ctxt->eflags);
4197 case 0xbc: { /* bsf */
4199 __asm__ ("bsf %2, %0; setz %1"
4200 : "=r"(ctxt->dst.val), "=q"(zf)
4201 : "r"(ctxt->src.val));
4202 ctxt->eflags &= ~X86_EFLAGS_ZF;
4204 ctxt->eflags |= X86_EFLAGS_ZF;
4205 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4209 case 0xbd: { /* bsr */
4211 __asm__ ("bsr %2, %0; setz %1"
4212 : "=r"(ctxt->dst.val), "=q"(zf)
4213 : "r"(ctxt->src.val));
4214 ctxt->eflags &= ~X86_EFLAGS_ZF;
4216 ctxt->eflags |= X86_EFLAGS_ZF;
4217 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4221 case 0xbe ... 0xbf: /* movsx */
4222 ctxt->dst.bytes = ctxt->op_bytes;
4223 ctxt->dst.val = (ctxt->d & ByteOp) ? (s8) ctxt->src.val :
4224 (s16) ctxt->src.val;
4226 case 0xc0 ... 0xc1: /* xadd */
4227 emulate_2op_SrcV("add", ctxt->src, ctxt->dst, ctxt->eflags);
4228 /* Write back the register source. */
4229 ctxt->src.val = ctxt->dst.orig_val;
4230 write_register_operand(&ctxt->src);
4232 case 0xc3: /* movnti */
4233 ctxt->dst.bytes = ctxt->op_bytes;
4234 ctxt->dst.val = (ctxt->op_bytes == 4) ? (u32) ctxt->src.val :
4235 (u64) ctxt->src.val;
4237 case 0xc7: /* Grp9 (cmpxchg8b) */
4241 goto cannot_emulate;
4244 if (rc != X86EMUL_CONTINUE)
4250 return EMULATION_FAILED;