1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affilates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
26 #include <public/xen.h>
27 #define DPRINTF(_f, _a ...) printf(_f , ## _a)
29 #include <linux/kvm_host.h>
30 #include "kvm_cache_regs.h"
31 #define DPRINTF(x...) do {} while (0)
33 #include <linux/module.h>
34 #include <asm/kvm_emulate.h>
40 * Opcode effective-address decode tables.
41 * Note that we only emulate instructions that have at least one memory
42 * operand (excluding implicit stack references). We assume that stack
43 * references and instruction fetches will never occur in special memory
44 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
48 /* Operand sizes: 8-bit operands or specified/overridden size. */
49 #define ByteOp (1<<16) /* 8-bit operands. */
50 /* Destination operand type. */
51 #define ImplicitOps (1<<17) /* Implicit in opcode. No generic decode. */
52 #define DstReg (2<<17) /* Register operand. */
53 #define DstMem (3<<17) /* Memory operand. */
54 #define DstAcc (4<<17) /* Destination Accumulator */
55 #define DstDI (5<<17) /* Destination is in ES:(E)DI */
56 #define DstMem64 (6<<17) /* 64bit memory operand */
57 #define DstMask (7<<17)
58 /* Source operand type. */
59 #define SrcNone (0<<4) /* No source operand. */
60 #define SrcImplicit (0<<4) /* Source operand is implicit in the opcode. */
61 #define SrcReg (1<<4) /* Register operand. */
62 #define SrcMem (2<<4) /* Memory operand. */
63 #define SrcMem16 (3<<4) /* Memory operand (16-bit). */
64 #define SrcMem32 (4<<4) /* Memory operand (32-bit). */
65 #define SrcImm (5<<4) /* Immediate operand. */
66 #define SrcImmByte (6<<4) /* 8-bit sign-extended immediate operand. */
67 #define SrcOne (7<<4) /* Implied '1' */
68 #define SrcImmUByte (8<<4) /* 8-bit unsigned immediate operand. */
69 #define SrcImmU (9<<4) /* Immediate operand, unsigned */
70 #define SrcSI (0xa<<4) /* Source is in the DS:RSI */
71 #define SrcImmFAddr (0xb<<4) /* Source is immediate far address */
72 #define SrcMemFAddr (0xc<<4) /* Source is far address in memory */
73 #define SrcAcc (0xd<<4) /* Source Accumulator */
74 #define SrcMask (0xf<<4)
75 /* Generic ModRM decode. */
77 /* Destination is only written; never read. */
80 #define MemAbs (1<<11) /* Memory operand is absolute displacement */
81 #define String (1<<12) /* String instruction (rep capable) */
82 #define Stack (1<<13) /* Stack instruction (push/pop) */
83 #define Group (1<<14) /* Bits 3:5 of modrm byte extend opcode */
84 #define GroupDual (1<<15) /* Alternate decoding of mod == 3 */
85 #define GroupMask 0x0f /* Group number stored in bits 0:3 */
87 #define Undefined (1<<25) /* No Such Instruction */
88 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
89 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
91 /* Source 2 operand type */
92 #define Src2None (0<<29)
93 #define Src2CL (1<<29)
94 #define Src2ImmByte (2<<29)
95 #define Src2One (3<<29)
96 #define Src2Mask (7<<29)
99 #define X3(x) X2(x), x
100 #define X4(x) X2(x), X2(x)
101 #define X5(x) X4(x), x
102 #define X6(x) X4(x), X2(x)
103 #define X7(x) X4(x), X3(x)
104 #define X8(x) X4(x), X4(x)
105 #define X16(x) X8(x), X8(x)
108 NoGrp, Group3, Group4, Group5, Group7, Group8, Group9,
114 struct opcode *group;
115 struct group_dual *gdual;
120 struct opcode mod012[8];
121 struct opcode mod3[8];
124 #define D(_y) { .flags = (_y) }
126 #define G(_f, _g) { .flags = ((_f) | Group), .u.group = (_g) }
127 #define GD(_f, _g) { .flags = ((_f) | Group | GroupDual), .u.gdual = (_g) }
129 static struct opcode group1[] = {
133 static struct opcode group1A[] = {
134 D(DstMem | SrcNone | ModRM | Mov | Stack), N, N, N, N, N, N, N,
137 static struct opcode group_table[] = {
139 D(DstMem | SrcImm | ModRM), D(DstMem | SrcImm | ModRM),
140 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
143 D(ByteOp | DstMem | SrcNone | ModRM | Lock), D(ByteOp | DstMem | SrcNone | ModRM | Lock),
146 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
147 D(SrcMem | ModRM | Stack), N,
148 D(SrcMem | ModRM | Stack), D(SrcMemFAddr | ModRM | ImplicitOps),
149 D(SrcMem | ModRM | Stack), N,
151 N, N, D(ModRM | SrcMem | Priv), D(ModRM | SrcMem | Priv),
152 D(SrcNone | ModRM | DstMem | Mov), N,
153 D(SrcMem16 | ModRM | Mov | Priv), D(SrcMem | ModRM | ByteOp | Priv),
156 D(DstMem | SrcImmByte | ModRM), D(DstMem | SrcImmByte | ModRM | Lock),
157 D(DstMem | SrcImmByte | ModRM | Lock), D(DstMem | SrcImmByte | ModRM | Lock),
159 N, D(DstMem64 | ModRM | Lock), N, N, N, N, N, N,
162 static struct opcode group2_table[] = {
164 D(SrcNone | ModRM | Priv), N, N, D(SrcNone | ModRM | Priv),
165 D(SrcNone | ModRM | DstMem | Mov), N,
166 D(SrcMem16 | ModRM | Mov | Priv), N,
168 N, N, N, N, N, N, N, N,
171 static struct opcode opcode_table[256] = {
173 D(ByteOp | DstMem | SrcReg | ModRM | Lock), D(DstMem | SrcReg | ModRM | Lock),
174 D(ByteOp | DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
175 D(ByteOp | DstAcc | SrcImm), D(DstAcc | SrcImm),
176 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
178 D(ByteOp | DstMem | SrcReg | ModRM | Lock), D(DstMem | SrcReg | ModRM | Lock),
179 D(ByteOp | DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
180 D(ByteOp | DstAcc | SrcImm), D(DstAcc | SrcImm),
181 D(ImplicitOps | Stack | No64), N,
183 D(ByteOp | DstMem | SrcReg | ModRM | Lock), D(DstMem | SrcReg | ModRM | Lock),
184 D(ByteOp | DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
185 D(ByteOp | DstAcc | SrcImm), D(DstAcc | SrcImm),
186 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
188 D(ByteOp | DstMem | SrcReg | ModRM | Lock), D(DstMem | SrcReg | ModRM | Lock),
189 D(ByteOp | DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
190 D(ByteOp | DstAcc | SrcImm), D(DstAcc | SrcImm),
191 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
193 D(ByteOp | DstMem | SrcReg | ModRM | Lock), D(DstMem | SrcReg | ModRM | Lock),
194 D(ByteOp | DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
195 D(ByteOp | DstAcc | SrcImmByte), D(DstAcc | SrcImm), N, N,
197 D(ByteOp | DstMem | SrcReg | ModRM | Lock), D(DstMem | SrcReg | ModRM | Lock),
198 D(ByteOp | DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
199 D(ByteOp | DstAcc | SrcImmByte), D(DstAcc | SrcImm), N, N,
201 D(ByteOp | DstMem | SrcReg | ModRM | Lock), D(DstMem | SrcReg | ModRM | Lock),
202 D(ByteOp | DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
203 D(ByteOp | DstAcc | SrcImmByte), D(DstAcc | SrcImm), N, N,
205 D(ByteOp | DstMem | SrcReg | ModRM), D(DstMem | SrcReg | ModRM),
206 D(ByteOp | DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
207 D(ByteOp | DstAcc | SrcImm), D(DstAcc | SrcImm),
212 X8(D(SrcReg | Stack)),
214 X8(D(DstReg | Stack)),
216 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
217 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
220 D(SrcImm | Mov | Stack), N, D(SrcImmByte | Mov | Stack), N,
221 D(DstDI | ByteOp | Mov | String), D(DstDI | Mov | String), /* insb, insw/insd */
222 D(SrcSI | ByteOp | ImplicitOps | String), D(SrcSI | ImplicitOps | String), /* outsb, outsw/outsd */
226 G(ByteOp | DstMem | SrcImm | ModRM | Group, group1),
227 G(DstMem | SrcImm | ModRM | Group, group1),
228 G(ByteOp | DstMem | SrcImm | ModRM | No64 | Group, group1),
229 G(DstMem | SrcImmByte | ModRM | Group, group1),
230 D(ByteOp | DstMem | SrcReg | ModRM), D(DstMem | SrcReg | ModRM),
231 D(ByteOp | DstMem | SrcReg | ModRM | Lock), D(DstMem | SrcReg | ModRM | Lock),
233 D(ByteOp | DstMem | SrcReg | ModRM | Mov), D(DstMem | SrcReg | ModRM | Mov),
234 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem | ModRM | Mov),
235 D(DstMem | SrcNone | ModRM | Mov), D(ModRM | DstReg),
236 D(ImplicitOps | SrcMem16 | ModRM), G(0, group1A),
238 D(DstReg), D(DstReg), D(DstReg), D(DstReg), D(DstReg), D(DstReg), D(DstReg), D(DstReg),
240 N, N, D(SrcImmFAddr | No64), N,
241 D(ImplicitOps | Stack), D(ImplicitOps | Stack), N, N,
243 D(ByteOp | DstAcc | SrcMem | Mov | MemAbs), D(DstAcc | SrcMem | Mov | MemAbs),
244 D(ByteOp | DstMem | SrcAcc | Mov | MemAbs), D(DstMem | SrcAcc | Mov | MemAbs),
245 D(ByteOp | SrcSI | DstDI | Mov | String), D(SrcSI | DstDI | Mov | String),
246 D(ByteOp | SrcSI | DstDI | String), D(SrcSI | DstDI | String),
248 D(DstAcc | SrcImmByte | ByteOp), D(DstAcc | SrcImm), D(ByteOp | DstDI | Mov | String), D(DstDI | Mov | String),
249 D(ByteOp | SrcSI | DstAcc | Mov | String), D(SrcSI | DstAcc | Mov | String),
250 D(ByteOp | DstDI | String), D(DstDI | String),
252 X8(D(ByteOp | DstReg | SrcImm | Mov)),
254 X8(D(DstReg | SrcImm | Mov)),
256 D(ByteOp | DstMem | SrcImm | ModRM), D(DstMem | SrcImmByte | ModRM),
257 N, D(ImplicitOps | Stack), N, N,
258 D(ByteOp | DstMem | SrcImm | ModRM | Mov), D(DstMem | SrcImm | ModRM | Mov),
260 N, N, N, D(ImplicitOps | Stack),
261 D(ImplicitOps), D(SrcImmByte), D(ImplicitOps | No64), D(ImplicitOps),
263 D(ByteOp | DstMem | SrcImplicit | ModRM), D(DstMem | SrcImplicit | ModRM),
264 D(ByteOp | DstMem | SrcImplicit | ModRM), D(DstMem | SrcImplicit | ModRM),
267 N, N, N, N, N, N, N, N,
270 D(ByteOp | SrcImmUByte | DstAcc), D(SrcImmUByte | DstAcc),
271 D(ByteOp | SrcImmUByte | DstAcc), D(SrcImmUByte | DstAcc),
273 D(SrcImm | Stack), D(SrcImm | ImplicitOps),
274 D(SrcImmFAddr | No64), D(SrcImmByte | ImplicitOps),
275 D(SrcNone | ByteOp | DstAcc), D(SrcNone | DstAcc),
276 D(SrcNone | ByteOp | DstAcc), D(SrcNone | DstAcc),
279 D(ImplicitOps | Priv), D(ImplicitOps), D(ByteOp | Group | Group3), D(Group | Group3),
281 D(ImplicitOps), N, D(ImplicitOps), D(ImplicitOps),
282 D(ImplicitOps), D(ImplicitOps), D(Group | Group4), D(Group | Group5),
285 static struct opcode twobyte_table[256] = {
287 N, D(Group | GroupDual | Group7), N, N,
288 N, D(ImplicitOps), D(ImplicitOps | Priv), N,
289 D(ImplicitOps | Priv), D(ImplicitOps | Priv), N, N,
290 N, D(ImplicitOps | ModRM), N, N,
292 N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
294 D(ModRM | ImplicitOps | Priv), D(ModRM | Priv),
295 D(ModRM | ImplicitOps | Priv), D(ModRM | Priv),
297 N, N, N, N, N, N, N, N,
299 D(ImplicitOps | Priv), N, D(ImplicitOps | Priv), N,
300 D(ImplicitOps), D(ImplicitOps | Priv), N, N,
301 N, N, N, N, N, N, N, N,
303 X16(D(DstReg | SrcMem | ModRM | Mov)),
305 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
307 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
309 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
313 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
315 D(ImplicitOps | Stack), D(ImplicitOps | Stack),
316 N, D(DstMem | SrcReg | ModRM | BitOp),
317 D(DstMem | SrcReg | Src2ImmByte | ModRM),
318 D(DstMem | SrcReg | Src2CL | ModRM), N, N,
320 D(ImplicitOps | Stack), D(ImplicitOps | Stack),
321 N, D(DstMem | SrcReg | ModRM | BitOp | Lock),
322 D(DstMem | SrcReg | Src2ImmByte | ModRM),
323 D(DstMem | SrcReg | Src2CL | ModRM),
326 D(ByteOp | DstMem | SrcReg | ModRM | Lock), D(DstMem | SrcReg | ModRM | Lock),
327 N, D(DstMem | SrcReg | ModRM | BitOp | Lock),
328 N, N, D(ByteOp | DstReg | SrcMem | ModRM | Mov),
329 D(DstReg | SrcMem16 | ModRM | Mov),
332 D(Group | Group8), D(DstMem | SrcReg | ModRM | BitOp | Lock),
333 N, N, D(ByteOp | DstReg | SrcMem | ModRM | Mov),
334 D(DstReg | SrcMem16 | ModRM | Mov),
336 N, N, N, D(DstMem | SrcReg | ModRM | Mov),
337 N, N, N, D(Group | GroupDual | Group9),
338 N, N, N, N, N, N, N, N,
340 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
342 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
344 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
352 /* EFLAGS bit definitions. */
353 #define EFLG_ID (1<<21)
354 #define EFLG_VIP (1<<20)
355 #define EFLG_VIF (1<<19)
356 #define EFLG_AC (1<<18)
357 #define EFLG_VM (1<<17)
358 #define EFLG_RF (1<<16)
359 #define EFLG_IOPL (3<<12)
360 #define EFLG_NT (1<<14)
361 #define EFLG_OF (1<<11)
362 #define EFLG_DF (1<<10)
363 #define EFLG_IF (1<<9)
364 #define EFLG_TF (1<<8)
365 #define EFLG_SF (1<<7)
366 #define EFLG_ZF (1<<6)
367 #define EFLG_AF (1<<4)
368 #define EFLG_PF (1<<2)
369 #define EFLG_CF (1<<0)
371 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
372 #define EFLG_RESERVED_ONE_MASK 2
375 * Instruction emulation:
376 * Most instructions are emulated directly via a fragment of inline assembly
377 * code. This allows us to save/restore EFLAGS and thus very easily pick up
378 * any modified flags.
381 #if defined(CONFIG_X86_64)
382 #define _LO32 "k" /* force 32-bit operand */
383 #define _STK "%%rsp" /* stack pointer */
384 #elif defined(__i386__)
385 #define _LO32 "" /* force 32-bit operand */
386 #define _STK "%%esp" /* stack pointer */
390 * These EFLAGS bits are restored from saved value during emulation, and
391 * any changes are written back to the saved value after emulation.
393 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
395 /* Before executing instruction: restore necessary bits in EFLAGS. */
396 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
397 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
398 "movl %"_sav",%"_LO32 _tmp"; " \
401 "movl %"_msk",%"_LO32 _tmp"; " \
402 "andl %"_LO32 _tmp",("_STK"); " \
404 "notl %"_LO32 _tmp"; " \
405 "andl %"_LO32 _tmp",("_STK"); " \
406 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
408 "orl %"_LO32 _tmp",("_STK"); " \
412 /* After executing instruction: write-back necessary bits in EFLAGS. */
413 #define _POST_EFLAGS(_sav, _msk, _tmp) \
414 /* _sav |= EFLAGS & _msk; */ \
417 "andl %"_msk",%"_LO32 _tmp"; " \
418 "orl %"_LO32 _tmp",%"_sav"; "
426 #define ____emulate_2op(_op, _src, _dst, _eflags, _x, _y, _suffix) \
428 __asm__ __volatile__ ( \
429 _PRE_EFLAGS("0", "4", "2") \
430 _op _suffix " %"_x"3,%1; " \
431 _POST_EFLAGS("0", "4", "2") \
432 : "=m" (_eflags), "=m" ((_dst).val), \
434 : _y ((_src).val), "i" (EFLAGS_MASK)); \
438 /* Raw emulation: instruction has two explicit operands. */
439 #define __emulate_2op_nobyte(_op,_src,_dst,_eflags,_wx,_wy,_lx,_ly,_qx,_qy) \
441 unsigned long _tmp; \
443 switch ((_dst).bytes) { \
445 ____emulate_2op(_op,_src,_dst,_eflags,_wx,_wy,"w"); \
448 ____emulate_2op(_op,_src,_dst,_eflags,_lx,_ly,"l"); \
451 ON64(____emulate_2op(_op,_src,_dst,_eflags,_qx,_qy,"q")); \
456 #define __emulate_2op(_op,_src,_dst,_eflags,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
458 unsigned long _tmp; \
459 switch ((_dst).bytes) { \
461 ____emulate_2op(_op,_src,_dst,_eflags,_bx,_by,"b"); \
464 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
465 _wx, _wy, _lx, _ly, _qx, _qy); \
470 /* Source operand is byte-sized and may be restricted to just %cl. */
471 #define emulate_2op_SrcB(_op, _src, _dst, _eflags) \
472 __emulate_2op(_op, _src, _dst, _eflags, \
473 "b", "c", "b", "c", "b", "c", "b", "c")
475 /* Source operand is byte, word, long or quad sized. */
476 #define emulate_2op_SrcV(_op, _src, _dst, _eflags) \
477 __emulate_2op(_op, _src, _dst, _eflags, \
478 "b", "q", "w", "r", _LO32, "r", "", "r")
480 /* Source operand is word, long or quad sized. */
481 #define emulate_2op_SrcV_nobyte(_op, _src, _dst, _eflags) \
482 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
483 "w", "r", _LO32, "r", "", "r")
485 /* Instruction has three operands and one operand is stored in ECX register */
486 #define __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, _suffix, _type) \
488 unsigned long _tmp; \
489 _type _clv = (_cl).val; \
490 _type _srcv = (_src).val; \
491 _type _dstv = (_dst).val; \
493 __asm__ __volatile__ ( \
494 _PRE_EFLAGS("0", "5", "2") \
495 _op _suffix " %4,%1 \n" \
496 _POST_EFLAGS("0", "5", "2") \
497 : "=m" (_eflags), "+r" (_dstv), "=&r" (_tmp) \
498 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
501 (_cl).val = (unsigned long) _clv; \
502 (_src).val = (unsigned long) _srcv; \
503 (_dst).val = (unsigned long) _dstv; \
506 #define emulate_2op_cl(_op, _cl, _src, _dst, _eflags) \
508 switch ((_dst).bytes) { \
510 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
511 "w", unsigned short); \
514 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
515 "l", unsigned int); \
518 ON64(__emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
519 "q", unsigned long)); \
524 #define __emulate_1op(_op, _dst, _eflags, _suffix) \
526 unsigned long _tmp; \
528 __asm__ __volatile__ ( \
529 _PRE_EFLAGS("0", "3", "2") \
530 _op _suffix " %1; " \
531 _POST_EFLAGS("0", "3", "2") \
532 : "=m" (_eflags), "+m" ((_dst).val), \
534 : "i" (EFLAGS_MASK)); \
537 /* Instruction has only one explicit operand (no source operand). */
538 #define emulate_1op(_op, _dst, _eflags) \
540 switch ((_dst).bytes) { \
541 case 1: __emulate_1op(_op, _dst, _eflags, "b"); break; \
542 case 2: __emulate_1op(_op, _dst, _eflags, "w"); break; \
543 case 4: __emulate_1op(_op, _dst, _eflags, "l"); break; \
544 case 8: ON64(__emulate_1op(_op, _dst, _eflags, "q")); break; \
548 /* Fetch next part of the instruction being emulated. */
549 #define insn_fetch(_type, _size, _eip) \
550 ({ unsigned long _x; \
551 rc = do_insn_fetch(ctxt, ops, (_eip), &_x, (_size)); \
552 if (rc != X86EMUL_CONTINUE) \
558 #define insn_fetch_arr(_arr, _size, _eip) \
559 ({ rc = do_insn_fetch(ctxt, ops, (_eip), _arr, (_size)); \
560 if (rc != X86EMUL_CONTINUE) \
565 static inline unsigned long ad_mask(struct decode_cache *c)
567 return (1UL << (c->ad_bytes << 3)) - 1;
570 /* Access/update address held in a register, based on addressing mode. */
571 static inline unsigned long
572 address_mask(struct decode_cache *c, unsigned long reg)
574 if (c->ad_bytes == sizeof(unsigned long))
577 return reg & ad_mask(c);
580 static inline unsigned long
581 register_address(struct decode_cache *c, unsigned long base, unsigned long reg)
583 return base + address_mask(c, reg);
587 register_address_increment(struct decode_cache *c, unsigned long *reg, int inc)
589 if (c->ad_bytes == sizeof(unsigned long))
592 *reg = (*reg & ~ad_mask(c)) | ((*reg + inc) & ad_mask(c));
595 static inline void jmp_rel(struct decode_cache *c, int rel)
597 register_address_increment(c, &c->eip, rel);
600 static void set_seg_override(struct decode_cache *c, int seg)
602 c->has_seg_override = true;
603 c->seg_override = seg;
606 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt,
607 struct x86_emulate_ops *ops, int seg)
609 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
612 return ops->get_cached_segment_base(seg, ctxt->vcpu);
615 static unsigned long seg_override_base(struct x86_emulate_ctxt *ctxt,
616 struct x86_emulate_ops *ops,
617 struct decode_cache *c)
619 if (!c->has_seg_override)
622 return seg_base(ctxt, ops, c->seg_override);
625 static unsigned long es_base(struct x86_emulate_ctxt *ctxt,
626 struct x86_emulate_ops *ops)
628 return seg_base(ctxt, ops, VCPU_SREG_ES);
631 static unsigned long ss_base(struct x86_emulate_ctxt *ctxt,
632 struct x86_emulate_ops *ops)
634 return seg_base(ctxt, ops, VCPU_SREG_SS);
637 static void emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
638 u32 error, bool valid)
640 ctxt->exception = vec;
641 ctxt->error_code = error;
642 ctxt->error_code_valid = valid;
643 ctxt->restart = false;
646 static void emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
648 emulate_exception(ctxt, GP_VECTOR, err, true);
651 static void emulate_pf(struct x86_emulate_ctxt *ctxt, unsigned long addr,
655 emulate_exception(ctxt, PF_VECTOR, err, true);
658 static void emulate_ud(struct x86_emulate_ctxt *ctxt)
660 emulate_exception(ctxt, UD_VECTOR, 0, false);
663 static void emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
665 emulate_exception(ctxt, TS_VECTOR, err, true);
668 static int do_fetch_insn_byte(struct x86_emulate_ctxt *ctxt,
669 struct x86_emulate_ops *ops,
670 unsigned long eip, u8 *dest)
672 struct fetch_cache *fc = &ctxt->decode.fetch;
676 if (eip == fc->end) {
677 cur_size = fc->end - fc->start;
678 size = min(15UL - cur_size, PAGE_SIZE - offset_in_page(eip));
679 rc = ops->fetch(ctxt->cs_base + eip, fc->data + cur_size,
680 size, ctxt->vcpu, NULL);
681 if (rc != X86EMUL_CONTINUE)
685 *dest = fc->data[eip - fc->start];
686 return X86EMUL_CONTINUE;
689 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
690 struct x86_emulate_ops *ops,
691 unsigned long eip, void *dest, unsigned size)
695 /* x86 instructions are limited to 15 bytes. */
696 if (eip + size - ctxt->eip > 15)
697 return X86EMUL_UNHANDLEABLE;
699 rc = do_fetch_insn_byte(ctxt, ops, eip++, dest++);
700 if (rc != X86EMUL_CONTINUE)
703 return X86EMUL_CONTINUE;
707 * Given the 'reg' portion of a ModRM byte, and a register block, return a
708 * pointer into the block that addresses the relevant register.
709 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
711 static void *decode_register(u8 modrm_reg, unsigned long *regs,
716 p = ®s[modrm_reg];
717 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
718 p = (unsigned char *)®s[modrm_reg & 3] + 1;
722 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
723 struct x86_emulate_ops *ops,
725 u16 *size, unsigned long *address, int op_bytes)
732 rc = ops->read_std((unsigned long)ptr, (unsigned long *)size, 2,
734 if (rc != X86EMUL_CONTINUE)
736 rc = ops->read_std((unsigned long)ptr + 2, address, op_bytes,
741 static int test_cc(unsigned int condition, unsigned int flags)
745 switch ((condition & 15) >> 1) {
747 rc |= (flags & EFLG_OF);
749 case 1: /* b/c/nae */
750 rc |= (flags & EFLG_CF);
753 rc |= (flags & EFLG_ZF);
756 rc |= (flags & (EFLG_CF|EFLG_ZF));
759 rc |= (flags & EFLG_SF);
762 rc |= (flags & EFLG_PF);
765 rc |= (flags & EFLG_ZF);
768 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
772 /* Odd condition identifiers (lsb == 1) have inverted sense. */
773 return (!!rc ^ (condition & 1));
776 static void decode_register_operand(struct operand *op,
777 struct decode_cache *c,
780 unsigned reg = c->modrm_reg;
781 int highbyte_regs = c->rex_prefix == 0;
784 reg = (c->b & 7) | ((c->rex_prefix & 1) << 3);
786 if ((c->d & ByteOp) && !inhibit_bytereg) {
787 op->ptr = decode_register(reg, c->regs, highbyte_regs);
788 op->val = *(u8 *)op->ptr;
791 op->ptr = decode_register(reg, c->regs, 0);
792 op->bytes = c->op_bytes;
795 op->val = *(u16 *)op->ptr;
798 op->val = *(u32 *)op->ptr;
801 op->val = *(u64 *) op->ptr;
805 op->orig_val = op->val;
808 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
809 struct x86_emulate_ops *ops)
811 struct decode_cache *c = &ctxt->decode;
813 int index_reg = 0, base_reg = 0, scale;
814 int rc = X86EMUL_CONTINUE;
817 c->modrm_reg = (c->rex_prefix & 4) << 1; /* REX.R */
818 index_reg = (c->rex_prefix & 2) << 2; /* REX.X */
819 c->modrm_rm = base_reg = (c->rex_prefix & 1) << 3; /* REG.B */
822 c->modrm = insn_fetch(u8, 1, c->eip);
823 c->modrm_mod |= (c->modrm & 0xc0) >> 6;
824 c->modrm_reg |= (c->modrm & 0x38) >> 3;
825 c->modrm_rm |= (c->modrm & 0x07);
829 if (c->modrm_mod == 3) {
830 c->modrm_ptr = decode_register(c->modrm_rm,
831 c->regs, c->d & ByteOp);
832 c->modrm_val = *(unsigned long *)c->modrm_ptr;
836 if (c->ad_bytes == 2) {
837 unsigned bx = c->regs[VCPU_REGS_RBX];
838 unsigned bp = c->regs[VCPU_REGS_RBP];
839 unsigned si = c->regs[VCPU_REGS_RSI];
840 unsigned di = c->regs[VCPU_REGS_RDI];
842 /* 16-bit ModR/M decode. */
843 switch (c->modrm_mod) {
845 if (c->modrm_rm == 6)
846 c->modrm_ea += insn_fetch(u16, 2, c->eip);
849 c->modrm_ea += insn_fetch(s8, 1, c->eip);
852 c->modrm_ea += insn_fetch(u16, 2, c->eip);
855 switch (c->modrm_rm) {
857 c->modrm_ea += bx + si;
860 c->modrm_ea += bx + di;
863 c->modrm_ea += bp + si;
866 c->modrm_ea += bp + di;
875 if (c->modrm_mod != 0)
882 if (c->modrm_rm == 2 || c->modrm_rm == 3 ||
883 (c->modrm_rm == 6 && c->modrm_mod != 0))
884 if (!c->has_seg_override)
885 set_seg_override(c, VCPU_SREG_SS);
886 c->modrm_ea = (u16)c->modrm_ea;
888 /* 32/64-bit ModR/M decode. */
889 if ((c->modrm_rm & 7) == 4) {
890 sib = insn_fetch(u8, 1, c->eip);
891 index_reg |= (sib >> 3) & 7;
895 if ((base_reg & 7) == 5 && c->modrm_mod == 0)
896 c->modrm_ea += insn_fetch(s32, 4, c->eip);
898 c->modrm_ea += c->regs[base_reg];
900 c->modrm_ea += c->regs[index_reg] << scale;
901 } else if ((c->modrm_rm & 7) == 5 && c->modrm_mod == 0) {
902 if (ctxt->mode == X86EMUL_MODE_PROT64)
905 c->modrm_ea += c->regs[c->modrm_rm];
906 switch (c->modrm_mod) {
908 if (c->modrm_rm == 5)
909 c->modrm_ea += insn_fetch(s32, 4, c->eip);
912 c->modrm_ea += insn_fetch(s8, 1, c->eip);
915 c->modrm_ea += insn_fetch(s32, 4, c->eip);
923 static int decode_abs(struct x86_emulate_ctxt *ctxt,
924 struct x86_emulate_ops *ops)
926 struct decode_cache *c = &ctxt->decode;
927 int rc = X86EMUL_CONTINUE;
929 switch (c->ad_bytes) {
931 c->modrm_ea = insn_fetch(u16, 2, c->eip);
934 c->modrm_ea = insn_fetch(u32, 4, c->eip);
937 c->modrm_ea = insn_fetch(u64, 8, c->eip);
945 x86_decode_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
947 struct decode_cache *c = &ctxt->decode;
948 int rc = X86EMUL_CONTINUE;
949 int mode = ctxt->mode;
950 int def_op_bytes, def_ad_bytes, group, dual, goffset;
951 struct opcode opcode, *g_mod012, *g_mod3;
953 /* we cannot decode insn before we complete previous rep insn */
954 WARN_ON(ctxt->restart);
957 c->fetch.start = c->fetch.end = c->eip;
958 ctxt->cs_base = seg_base(ctxt, ops, VCPU_SREG_CS);
961 case X86EMUL_MODE_REAL:
962 case X86EMUL_MODE_VM86:
963 case X86EMUL_MODE_PROT16:
964 def_op_bytes = def_ad_bytes = 2;
966 case X86EMUL_MODE_PROT32:
967 def_op_bytes = def_ad_bytes = 4;
970 case X86EMUL_MODE_PROT64:
979 c->op_bytes = def_op_bytes;
980 c->ad_bytes = def_ad_bytes;
982 /* Legacy prefixes. */
984 switch (c->b = insn_fetch(u8, 1, c->eip)) {
985 case 0x66: /* operand-size override */
986 /* switch between 2/4 bytes */
987 c->op_bytes = def_op_bytes ^ 6;
989 case 0x67: /* address-size override */
990 if (mode == X86EMUL_MODE_PROT64)
991 /* switch between 4/8 bytes */
992 c->ad_bytes = def_ad_bytes ^ 12;
994 /* switch between 2/4 bytes */
995 c->ad_bytes = def_ad_bytes ^ 6;
997 case 0x26: /* ES override */
998 case 0x2e: /* CS override */
999 case 0x36: /* SS override */
1000 case 0x3e: /* DS override */
1001 set_seg_override(c, (c->b >> 3) & 3);
1003 case 0x64: /* FS override */
1004 case 0x65: /* GS override */
1005 set_seg_override(c, c->b & 7);
1007 case 0x40 ... 0x4f: /* REX */
1008 if (mode != X86EMUL_MODE_PROT64)
1010 c->rex_prefix = c->b;
1012 case 0xf0: /* LOCK */
1015 case 0xf2: /* REPNE/REPNZ */
1016 c->rep_prefix = REPNE_PREFIX;
1018 case 0xf3: /* REP/REPE/REPZ */
1019 c->rep_prefix = REPE_PREFIX;
1025 /* Any legacy prefix after a REX prefix nullifies its effect. */
1034 if (c->rex_prefix & 8)
1035 c->op_bytes = 8; /* REX.W */
1037 /* Opcode byte(s). */
1038 opcode = opcode_table[c->b];
1039 if (opcode.flags == 0) {
1040 /* Two-byte opcode? */
1043 c->b = insn_fetch(u8, 1, c->eip);
1044 opcode = twobyte_table[c->b];
1047 c->d = opcode.flags;
1050 group = c->d & GroupMask;
1051 dual = c->d & GroupDual;
1052 c->modrm = insn_fetch(u8, 1, c->eip);
1056 g_mod012 = g_mod3 = &group_table[group * 8];
1057 if (c->d & GroupDual)
1058 g_mod3 = &group2_table[group * 8];
1060 if (c->d & GroupDual) {
1061 g_mod012 = opcode.u.gdual->mod012;
1062 g_mod3 = opcode.u.gdual->mod3;
1064 g_mod012 = g_mod3 = opcode.u.group;
1067 c->d &= ~(Group | GroupDual | GroupMask);
1069 goffset = (c->modrm >> 3) & 7;
1071 if ((c->modrm >> 6) == 3)
1072 opcode = g_mod3[goffset];
1074 opcode = g_mod012[goffset];
1075 c->d |= opcode.flags;
1079 if (c->d == 0 || (c->d & Undefined)) {
1080 DPRINTF("Cannot emulate %02x\n", c->b);
1084 if (mode == X86EMUL_MODE_PROT64 && (c->d & Stack))
1087 /* ModRM and SIB bytes. */
1089 rc = decode_modrm(ctxt, ops);
1090 else if (c->d & MemAbs)
1091 rc = decode_abs(ctxt, ops);
1092 if (rc != X86EMUL_CONTINUE)
1095 if (!c->has_seg_override)
1096 set_seg_override(c, VCPU_SREG_DS);
1098 if (!(!c->twobyte && c->b == 0x8d))
1099 c->modrm_ea += seg_override_base(ctxt, ops, c);
1101 if (c->ad_bytes != 8)
1102 c->modrm_ea = (u32)c->modrm_ea;
1104 if (c->rip_relative)
1105 c->modrm_ea += c->eip;
1108 * Decode and fetch the source operand: register, memory
1111 switch (c->d & SrcMask) {
1115 decode_register_operand(&c->src, c, 0);
1124 c->src.bytes = (c->d & ByteOp) ? 1 :
1126 /* Don't fetch the address for invlpg: it could be unmapped. */
1127 if (c->twobyte && c->b == 0x01 && c->modrm_reg == 7)
1131 * For instructions with a ModR/M byte, switch to register
1132 * access if Mod = 3.
1134 if ((c->d & ModRM) && c->modrm_mod == 3) {
1135 c->src.type = OP_REG;
1136 c->src.val = c->modrm_val;
1137 c->src.ptr = c->modrm_ptr;
1140 c->src.type = OP_MEM;
1141 c->src.ptr = (unsigned long *)c->modrm_ea;
1146 c->src.type = OP_IMM;
1147 c->src.ptr = (unsigned long *)c->eip;
1148 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1149 if (c->src.bytes == 8)
1151 /* NB. Immediates are sign-extended as necessary. */
1152 switch (c->src.bytes) {
1154 c->src.val = insn_fetch(s8, 1, c->eip);
1157 c->src.val = insn_fetch(s16, 2, c->eip);
1160 c->src.val = insn_fetch(s32, 4, c->eip);
1163 if ((c->d & SrcMask) == SrcImmU) {
1164 switch (c->src.bytes) {
1169 c->src.val &= 0xffff;
1172 c->src.val &= 0xffffffff;
1179 c->src.type = OP_IMM;
1180 c->src.ptr = (unsigned long *)c->eip;
1182 if ((c->d & SrcMask) == SrcImmByte)
1183 c->src.val = insn_fetch(s8, 1, c->eip);
1185 c->src.val = insn_fetch(u8, 1, c->eip);
1188 c->src.type = OP_REG;
1189 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1190 c->src.ptr = &c->regs[VCPU_REGS_RAX];
1191 switch (c->src.bytes) {
1193 c->src.val = *(u8 *)c->src.ptr;
1196 c->src.val = *(u16 *)c->src.ptr;
1199 c->src.val = *(u32 *)c->src.ptr;
1202 c->src.val = *(u64 *)c->src.ptr;
1211 c->src.type = OP_MEM;
1212 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1213 c->src.ptr = (unsigned long *)
1214 register_address(c, seg_override_base(ctxt, ops, c),
1215 c->regs[VCPU_REGS_RSI]);
1219 c->src.type = OP_IMM;
1220 c->src.ptr = (unsigned long *)c->eip;
1221 c->src.bytes = c->op_bytes + 2;
1222 insn_fetch_arr(c->src.valptr, c->src.bytes, c->eip);
1225 c->src.type = OP_MEM;
1226 c->src.ptr = (unsigned long *)c->modrm_ea;
1227 c->src.bytes = c->op_bytes + 2;
1232 * Decode and fetch the second source operand: register, memory
1235 switch (c->d & Src2Mask) {
1240 c->src2.val = c->regs[VCPU_REGS_RCX] & 0x8;
1243 c->src2.type = OP_IMM;
1244 c->src2.ptr = (unsigned long *)c->eip;
1246 c->src2.val = insn_fetch(u8, 1, c->eip);
1254 /* Decode and fetch the destination operand: register or memory. */
1255 switch (c->d & DstMask) {
1257 /* Special instructions do their own operand decoding. */
1260 decode_register_operand(&c->dst, c,
1261 c->twobyte && (c->b == 0xb6 || c->b == 0xb7));
1265 if ((c->d & ModRM) && c->modrm_mod == 3) {
1266 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1267 c->dst.type = OP_REG;
1268 c->dst.val = c->dst.orig_val = c->modrm_val;
1269 c->dst.ptr = c->modrm_ptr;
1272 c->dst.type = OP_MEM;
1273 c->dst.ptr = (unsigned long *)c->modrm_ea;
1274 if ((c->d & DstMask) == DstMem64)
1277 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1280 unsigned long mask = ~(c->dst.bytes * 8 - 1);
1282 c->dst.ptr = (void *)c->dst.ptr +
1283 (c->src.val & mask) / 8;
1287 c->dst.type = OP_REG;
1288 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1289 c->dst.ptr = &c->regs[VCPU_REGS_RAX];
1290 switch (c->dst.bytes) {
1292 c->dst.val = *(u8 *)c->dst.ptr;
1295 c->dst.val = *(u16 *)c->dst.ptr;
1298 c->dst.val = *(u32 *)c->dst.ptr;
1301 c->dst.val = *(u64 *)c->dst.ptr;
1304 c->dst.orig_val = c->dst.val;
1307 c->dst.type = OP_MEM;
1308 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1309 c->dst.ptr = (unsigned long *)
1310 register_address(c, es_base(ctxt, ops),
1311 c->regs[VCPU_REGS_RDI]);
1317 return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0;
1320 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1321 struct x86_emulate_ops *ops,
1322 unsigned long addr, void *dest, unsigned size)
1325 struct read_cache *mc = &ctxt->decode.mem_read;
1329 int n = min(size, 8u);
1331 if (mc->pos < mc->end)
1334 rc = ops->read_emulated(addr, mc->data + mc->end, n, &err,
1336 if (rc == X86EMUL_PROPAGATE_FAULT)
1337 emulate_pf(ctxt, addr, err);
1338 if (rc != X86EMUL_CONTINUE)
1343 memcpy(dest, mc->data + mc->pos, n);
1348 return X86EMUL_CONTINUE;
1351 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1352 struct x86_emulate_ops *ops,
1353 unsigned int size, unsigned short port,
1356 struct read_cache *rc = &ctxt->decode.io_read;
1358 if (rc->pos == rc->end) { /* refill pio read ahead */
1359 struct decode_cache *c = &ctxt->decode;
1360 unsigned int in_page, n;
1361 unsigned int count = c->rep_prefix ?
1362 address_mask(c, c->regs[VCPU_REGS_RCX]) : 1;
1363 in_page = (ctxt->eflags & EFLG_DF) ?
1364 offset_in_page(c->regs[VCPU_REGS_RDI]) :
1365 PAGE_SIZE - offset_in_page(c->regs[VCPU_REGS_RDI]);
1366 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1370 rc->pos = rc->end = 0;
1371 if (!ops->pio_in_emulated(size, port, rc->data, n, ctxt->vcpu))
1376 memcpy(dest, rc->data + rc->pos, size);
1381 static u32 desc_limit_scaled(struct desc_struct *desc)
1383 u32 limit = get_desc_limit(desc);
1385 return desc->g ? (limit << 12) | 0xfff : limit;
1388 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1389 struct x86_emulate_ops *ops,
1390 u16 selector, struct desc_ptr *dt)
1392 if (selector & 1 << 2) {
1393 struct desc_struct desc;
1394 memset (dt, 0, sizeof *dt);
1395 if (!ops->get_cached_descriptor(&desc, VCPU_SREG_LDTR, ctxt->vcpu))
1398 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1399 dt->address = get_desc_base(&desc);
1401 ops->get_gdt(dt, ctxt->vcpu);
1404 /* allowed just for 8 bytes segments */
1405 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1406 struct x86_emulate_ops *ops,
1407 u16 selector, struct desc_struct *desc)
1410 u16 index = selector >> 3;
1415 get_descriptor_table_ptr(ctxt, ops, selector, &dt);
1417 if (dt.size < index * 8 + 7) {
1418 emulate_gp(ctxt, selector & 0xfffc);
1419 return X86EMUL_PROPAGATE_FAULT;
1421 addr = dt.address + index * 8;
1422 ret = ops->read_std(addr, desc, sizeof *desc, ctxt->vcpu, &err);
1423 if (ret == X86EMUL_PROPAGATE_FAULT)
1424 emulate_pf(ctxt, addr, err);
1429 /* allowed just for 8 bytes segments */
1430 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1431 struct x86_emulate_ops *ops,
1432 u16 selector, struct desc_struct *desc)
1435 u16 index = selector >> 3;
1440 get_descriptor_table_ptr(ctxt, ops, selector, &dt);
1442 if (dt.size < index * 8 + 7) {
1443 emulate_gp(ctxt, selector & 0xfffc);
1444 return X86EMUL_PROPAGATE_FAULT;
1447 addr = dt.address + index * 8;
1448 ret = ops->write_std(addr, desc, sizeof *desc, ctxt->vcpu, &err);
1449 if (ret == X86EMUL_PROPAGATE_FAULT)
1450 emulate_pf(ctxt, addr, err);
1455 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1456 struct x86_emulate_ops *ops,
1457 u16 selector, int seg)
1459 struct desc_struct seg_desc;
1461 unsigned err_vec = GP_VECTOR;
1463 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1466 memset(&seg_desc, 0, sizeof seg_desc);
1468 if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
1469 || ctxt->mode == X86EMUL_MODE_REAL) {
1470 /* set real mode segment descriptor */
1471 set_desc_base(&seg_desc, selector << 4);
1472 set_desc_limit(&seg_desc, 0xffff);
1479 /* NULL selector is not valid for TR, CS and SS */
1480 if ((seg == VCPU_SREG_CS || seg == VCPU_SREG_SS || seg == VCPU_SREG_TR)
1484 /* TR should be in GDT only */
1485 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1488 if (null_selector) /* for NULL selector skip all following checks */
1491 ret = read_segment_descriptor(ctxt, ops, selector, &seg_desc);
1492 if (ret != X86EMUL_CONTINUE)
1495 err_code = selector & 0xfffc;
1496 err_vec = GP_VECTOR;
1498 /* can't load system descriptor into segment selecor */
1499 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1503 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1509 cpl = ops->cpl(ctxt->vcpu);
1514 * segment is not a writable data segment or segment
1515 * selector's RPL != CPL or segment selector's RPL != CPL
1517 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1521 if (!(seg_desc.type & 8))
1524 if (seg_desc.type & 4) {
1530 if (rpl > cpl || dpl != cpl)
1533 /* CS(RPL) <- CPL */
1534 selector = (selector & 0xfffc) | cpl;
1537 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1540 case VCPU_SREG_LDTR:
1541 if (seg_desc.s || seg_desc.type != 2)
1544 default: /* DS, ES, FS, or GS */
1546 * segment is not a data or readable code segment or
1547 * ((segment is a data or nonconforming code segment)
1548 * and (both RPL and CPL > DPL))
1550 if ((seg_desc.type & 0xa) == 0x8 ||
1551 (((seg_desc.type & 0xc) != 0xc) &&
1552 (rpl > dpl && cpl > dpl)))
1558 /* mark segment as accessed */
1560 ret = write_segment_descriptor(ctxt, ops, selector, &seg_desc);
1561 if (ret != X86EMUL_CONTINUE)
1565 ops->set_segment_selector(selector, seg, ctxt->vcpu);
1566 ops->set_cached_descriptor(&seg_desc, seg, ctxt->vcpu);
1567 return X86EMUL_CONTINUE;
1569 emulate_exception(ctxt, err_vec, err_code, true);
1570 return X86EMUL_PROPAGATE_FAULT;
1573 static inline int writeback(struct x86_emulate_ctxt *ctxt,
1574 struct x86_emulate_ops *ops)
1577 struct decode_cache *c = &ctxt->decode;
1580 switch (c->dst.type) {
1582 /* The 4-byte case *is* correct:
1583 * in 64-bit mode we zero-extend.
1585 switch (c->dst.bytes) {
1587 *(u8 *)c->dst.ptr = (u8)c->dst.val;
1590 *(u16 *)c->dst.ptr = (u16)c->dst.val;
1593 *c->dst.ptr = (u32)c->dst.val;
1594 break; /* 64b: zero-ext */
1596 *c->dst.ptr = c->dst.val;
1602 rc = ops->cmpxchg_emulated(
1603 (unsigned long)c->dst.ptr,
1610 rc = ops->write_emulated(
1611 (unsigned long)c->dst.ptr,
1616 if (rc == X86EMUL_PROPAGATE_FAULT)
1618 (unsigned long)c->dst.ptr, err);
1619 if (rc != X86EMUL_CONTINUE)
1628 return X86EMUL_CONTINUE;
1631 static inline void emulate_push(struct x86_emulate_ctxt *ctxt,
1632 struct x86_emulate_ops *ops)
1634 struct decode_cache *c = &ctxt->decode;
1636 c->dst.type = OP_MEM;
1637 c->dst.bytes = c->op_bytes;
1638 c->dst.val = c->src.val;
1639 register_address_increment(c, &c->regs[VCPU_REGS_RSP], -c->op_bytes);
1640 c->dst.ptr = (void *) register_address(c, ss_base(ctxt, ops),
1641 c->regs[VCPU_REGS_RSP]);
1644 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1645 struct x86_emulate_ops *ops,
1646 void *dest, int len)
1648 struct decode_cache *c = &ctxt->decode;
1651 rc = read_emulated(ctxt, ops, register_address(c, ss_base(ctxt, ops),
1652 c->regs[VCPU_REGS_RSP]),
1654 if (rc != X86EMUL_CONTINUE)
1657 register_address_increment(c, &c->regs[VCPU_REGS_RSP], len);
1661 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1662 struct x86_emulate_ops *ops,
1663 void *dest, int len)
1666 unsigned long val, change_mask;
1667 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1668 int cpl = ops->cpl(ctxt->vcpu);
1670 rc = emulate_pop(ctxt, ops, &val, len);
1671 if (rc != X86EMUL_CONTINUE)
1674 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1675 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1677 switch(ctxt->mode) {
1678 case X86EMUL_MODE_PROT64:
1679 case X86EMUL_MODE_PROT32:
1680 case X86EMUL_MODE_PROT16:
1682 change_mask |= EFLG_IOPL;
1684 change_mask |= EFLG_IF;
1686 case X86EMUL_MODE_VM86:
1688 emulate_gp(ctxt, 0);
1689 return X86EMUL_PROPAGATE_FAULT;
1691 change_mask |= EFLG_IF;
1693 default: /* real mode */
1694 change_mask |= (EFLG_IOPL | EFLG_IF);
1698 *(unsigned long *)dest =
1699 (ctxt->eflags & ~change_mask) | (val & change_mask);
1704 static void emulate_push_sreg(struct x86_emulate_ctxt *ctxt,
1705 struct x86_emulate_ops *ops, int seg)
1707 struct decode_cache *c = &ctxt->decode;
1709 c->src.val = ops->get_segment_selector(seg, ctxt->vcpu);
1711 emulate_push(ctxt, ops);
1714 static int emulate_pop_sreg(struct x86_emulate_ctxt *ctxt,
1715 struct x86_emulate_ops *ops, int seg)
1717 struct decode_cache *c = &ctxt->decode;
1718 unsigned long selector;
1721 rc = emulate_pop(ctxt, ops, &selector, c->op_bytes);
1722 if (rc != X86EMUL_CONTINUE)
1725 rc = load_segment_descriptor(ctxt, ops, (u16)selector, seg);
1729 static int emulate_pusha(struct x86_emulate_ctxt *ctxt,
1730 struct x86_emulate_ops *ops)
1732 struct decode_cache *c = &ctxt->decode;
1733 unsigned long old_esp = c->regs[VCPU_REGS_RSP];
1734 int rc = X86EMUL_CONTINUE;
1735 int reg = VCPU_REGS_RAX;
1737 while (reg <= VCPU_REGS_RDI) {
1738 (reg == VCPU_REGS_RSP) ?
1739 (c->src.val = old_esp) : (c->src.val = c->regs[reg]);
1741 emulate_push(ctxt, ops);
1743 rc = writeback(ctxt, ops);
1744 if (rc != X86EMUL_CONTINUE)
1750 /* Disable writeback. */
1751 c->dst.type = OP_NONE;
1756 static int emulate_popa(struct x86_emulate_ctxt *ctxt,
1757 struct x86_emulate_ops *ops)
1759 struct decode_cache *c = &ctxt->decode;
1760 int rc = X86EMUL_CONTINUE;
1761 int reg = VCPU_REGS_RDI;
1763 while (reg >= VCPU_REGS_RAX) {
1764 if (reg == VCPU_REGS_RSP) {
1765 register_address_increment(c, &c->regs[VCPU_REGS_RSP],
1770 rc = emulate_pop(ctxt, ops, &c->regs[reg], c->op_bytes);
1771 if (rc != X86EMUL_CONTINUE)
1778 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt,
1779 struct x86_emulate_ops *ops)
1781 struct decode_cache *c = &ctxt->decode;
1782 int rc = X86EMUL_CONTINUE;
1783 unsigned long temp_eip = 0;
1784 unsigned long temp_eflags = 0;
1785 unsigned long cs = 0;
1786 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1787 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1788 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1789 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1791 /* TODO: Add stack limit check */
1793 rc = emulate_pop(ctxt, ops, &temp_eip, c->op_bytes);
1795 if (rc != X86EMUL_CONTINUE)
1798 if (temp_eip & ~0xffff) {
1799 emulate_gp(ctxt, 0);
1800 return X86EMUL_PROPAGATE_FAULT;
1803 rc = emulate_pop(ctxt, ops, &cs, c->op_bytes);
1805 if (rc != X86EMUL_CONTINUE)
1808 rc = emulate_pop(ctxt, ops, &temp_eflags, c->op_bytes);
1810 if (rc != X86EMUL_CONTINUE)
1813 rc = load_segment_descriptor(ctxt, ops, (u16)cs, VCPU_SREG_CS);
1815 if (rc != X86EMUL_CONTINUE)
1821 if (c->op_bytes == 4)
1822 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1823 else if (c->op_bytes == 2) {
1824 ctxt->eflags &= ~0xffff;
1825 ctxt->eflags |= temp_eflags;
1828 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1829 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1834 static inline int emulate_iret(struct x86_emulate_ctxt *ctxt,
1835 struct x86_emulate_ops* ops)
1837 switch(ctxt->mode) {
1838 case X86EMUL_MODE_REAL:
1839 return emulate_iret_real(ctxt, ops);
1840 case X86EMUL_MODE_VM86:
1841 case X86EMUL_MODE_PROT16:
1842 case X86EMUL_MODE_PROT32:
1843 case X86EMUL_MODE_PROT64:
1845 /* iret from protected mode unimplemented yet */
1846 return X86EMUL_UNHANDLEABLE;
1850 static inline int emulate_grp1a(struct x86_emulate_ctxt *ctxt,
1851 struct x86_emulate_ops *ops)
1853 struct decode_cache *c = &ctxt->decode;
1855 return emulate_pop(ctxt, ops, &c->dst.val, c->dst.bytes);
1858 static inline void emulate_grp2(struct x86_emulate_ctxt *ctxt)
1860 struct decode_cache *c = &ctxt->decode;
1861 switch (c->modrm_reg) {
1863 emulate_2op_SrcB("rol", c->src, c->dst, ctxt->eflags);
1866 emulate_2op_SrcB("ror", c->src, c->dst, ctxt->eflags);
1869 emulate_2op_SrcB("rcl", c->src, c->dst, ctxt->eflags);
1872 emulate_2op_SrcB("rcr", c->src, c->dst, ctxt->eflags);
1874 case 4: /* sal/shl */
1875 case 6: /* sal/shl */
1876 emulate_2op_SrcB("sal", c->src, c->dst, ctxt->eflags);
1879 emulate_2op_SrcB("shr", c->src, c->dst, ctxt->eflags);
1882 emulate_2op_SrcB("sar", c->src, c->dst, ctxt->eflags);
1887 static inline int emulate_grp3(struct x86_emulate_ctxt *ctxt,
1888 struct x86_emulate_ops *ops)
1890 struct decode_cache *c = &ctxt->decode;
1892 switch (c->modrm_reg) {
1893 case 0 ... 1: /* test */
1894 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
1897 c->dst.val = ~c->dst.val;
1900 emulate_1op("neg", c->dst, ctxt->eflags);
1908 static inline int emulate_grp45(struct x86_emulate_ctxt *ctxt,
1909 struct x86_emulate_ops *ops)
1911 struct decode_cache *c = &ctxt->decode;
1913 switch (c->modrm_reg) {
1915 emulate_1op("inc", c->dst, ctxt->eflags);
1918 emulate_1op("dec", c->dst, ctxt->eflags);
1920 case 2: /* call near abs */ {
1923 c->eip = c->src.val;
1924 c->src.val = old_eip;
1925 emulate_push(ctxt, ops);
1928 case 4: /* jmp abs */
1929 c->eip = c->src.val;
1932 emulate_push(ctxt, ops);
1935 return X86EMUL_CONTINUE;
1938 static inline int emulate_grp9(struct x86_emulate_ctxt *ctxt,
1939 struct x86_emulate_ops *ops)
1941 struct decode_cache *c = &ctxt->decode;
1942 u64 old = c->dst.orig_val64;
1944 if (((u32) (old >> 0) != (u32) c->regs[VCPU_REGS_RAX]) ||
1945 ((u32) (old >> 32) != (u32) c->regs[VCPU_REGS_RDX])) {
1946 c->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
1947 c->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1948 ctxt->eflags &= ~EFLG_ZF;
1950 c->dst.val64 = ((u64)c->regs[VCPU_REGS_RCX] << 32) |
1951 (u32) c->regs[VCPU_REGS_RBX];
1953 ctxt->eflags |= EFLG_ZF;
1955 return X86EMUL_CONTINUE;
1958 static int emulate_ret_far(struct x86_emulate_ctxt *ctxt,
1959 struct x86_emulate_ops *ops)
1961 struct decode_cache *c = &ctxt->decode;
1965 rc = emulate_pop(ctxt, ops, &c->eip, c->op_bytes);
1966 if (rc != X86EMUL_CONTINUE)
1968 if (c->op_bytes == 4)
1969 c->eip = (u32)c->eip;
1970 rc = emulate_pop(ctxt, ops, &cs, c->op_bytes);
1971 if (rc != X86EMUL_CONTINUE)
1973 rc = load_segment_descriptor(ctxt, ops, (u16)cs, VCPU_SREG_CS);
1978 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1979 struct x86_emulate_ops *ops, struct desc_struct *cs,
1980 struct desc_struct *ss)
1982 memset(cs, 0, sizeof(struct desc_struct));
1983 ops->get_cached_descriptor(cs, VCPU_SREG_CS, ctxt->vcpu);
1984 memset(ss, 0, sizeof(struct desc_struct));
1986 cs->l = 0; /* will be adjusted later */
1987 set_desc_base(cs, 0); /* flat segment */
1988 cs->g = 1; /* 4kb granularity */
1989 set_desc_limit(cs, 0xfffff); /* 4GB limit */
1990 cs->type = 0x0b; /* Read, Execute, Accessed */
1992 cs->dpl = 0; /* will be adjusted later */
1996 set_desc_base(ss, 0); /* flat segment */
1997 set_desc_limit(ss, 0xfffff); /* 4GB limit */
1998 ss->g = 1; /* 4kb granularity */
2000 ss->type = 0x03; /* Read/Write, Accessed */
2001 ss->d = 1; /* 32bit stack segment */
2007 emulate_syscall(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
2009 struct decode_cache *c = &ctxt->decode;
2010 struct desc_struct cs, ss;
2014 /* syscall is not available in real mode */
2015 if (ctxt->mode == X86EMUL_MODE_REAL ||
2016 ctxt->mode == X86EMUL_MODE_VM86) {
2018 return X86EMUL_PROPAGATE_FAULT;
2021 setup_syscalls_segments(ctxt, ops, &cs, &ss);
2023 ops->get_msr(ctxt->vcpu, MSR_STAR, &msr_data);
2025 cs_sel = (u16)(msr_data & 0xfffc);
2026 ss_sel = (u16)(msr_data + 8);
2028 if (is_long_mode(ctxt->vcpu)) {
2032 ops->set_cached_descriptor(&cs, VCPU_SREG_CS, ctxt->vcpu);
2033 ops->set_segment_selector(cs_sel, VCPU_SREG_CS, ctxt->vcpu);
2034 ops->set_cached_descriptor(&ss, VCPU_SREG_SS, ctxt->vcpu);
2035 ops->set_segment_selector(ss_sel, VCPU_SREG_SS, ctxt->vcpu);
2037 c->regs[VCPU_REGS_RCX] = c->eip;
2038 if (is_long_mode(ctxt->vcpu)) {
2039 #ifdef CONFIG_X86_64
2040 c->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
2042 ops->get_msr(ctxt->vcpu,
2043 ctxt->mode == X86EMUL_MODE_PROT64 ?
2044 MSR_LSTAR : MSR_CSTAR, &msr_data);
2047 ops->get_msr(ctxt->vcpu, MSR_SYSCALL_MASK, &msr_data);
2048 ctxt->eflags &= ~(msr_data | EFLG_RF);
2052 ops->get_msr(ctxt->vcpu, MSR_STAR, &msr_data);
2053 c->eip = (u32)msr_data;
2055 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2058 return X86EMUL_CONTINUE;
2062 emulate_sysenter(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
2064 struct decode_cache *c = &ctxt->decode;
2065 struct desc_struct cs, ss;
2069 /* inject #GP if in real mode */
2070 if (ctxt->mode == X86EMUL_MODE_REAL) {
2071 emulate_gp(ctxt, 0);
2072 return X86EMUL_PROPAGATE_FAULT;
2075 /* XXX sysenter/sysexit have not been tested in 64bit mode.
2076 * Therefore, we inject an #UD.
2078 if (ctxt->mode == X86EMUL_MODE_PROT64) {
2080 return X86EMUL_PROPAGATE_FAULT;
2083 setup_syscalls_segments(ctxt, ops, &cs, &ss);
2085 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
2086 switch (ctxt->mode) {
2087 case X86EMUL_MODE_PROT32:
2088 if ((msr_data & 0xfffc) == 0x0) {
2089 emulate_gp(ctxt, 0);
2090 return X86EMUL_PROPAGATE_FAULT;
2093 case X86EMUL_MODE_PROT64:
2094 if (msr_data == 0x0) {
2095 emulate_gp(ctxt, 0);
2096 return X86EMUL_PROPAGATE_FAULT;
2101 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2102 cs_sel = (u16)msr_data;
2103 cs_sel &= ~SELECTOR_RPL_MASK;
2104 ss_sel = cs_sel + 8;
2105 ss_sel &= ~SELECTOR_RPL_MASK;
2106 if (ctxt->mode == X86EMUL_MODE_PROT64
2107 || is_long_mode(ctxt->vcpu)) {
2112 ops->set_cached_descriptor(&cs, VCPU_SREG_CS, ctxt->vcpu);
2113 ops->set_segment_selector(cs_sel, VCPU_SREG_CS, ctxt->vcpu);
2114 ops->set_cached_descriptor(&ss, VCPU_SREG_SS, ctxt->vcpu);
2115 ops->set_segment_selector(ss_sel, VCPU_SREG_SS, ctxt->vcpu);
2117 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_EIP, &msr_data);
2120 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_ESP, &msr_data);
2121 c->regs[VCPU_REGS_RSP] = msr_data;
2123 return X86EMUL_CONTINUE;
2127 emulate_sysexit(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
2129 struct decode_cache *c = &ctxt->decode;
2130 struct desc_struct cs, ss;
2135 /* inject #GP if in real mode or Virtual 8086 mode */
2136 if (ctxt->mode == X86EMUL_MODE_REAL ||
2137 ctxt->mode == X86EMUL_MODE_VM86) {
2138 emulate_gp(ctxt, 0);
2139 return X86EMUL_PROPAGATE_FAULT;
2142 setup_syscalls_segments(ctxt, ops, &cs, &ss);
2144 if ((c->rex_prefix & 0x8) != 0x0)
2145 usermode = X86EMUL_MODE_PROT64;
2147 usermode = X86EMUL_MODE_PROT32;
2151 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
2153 case X86EMUL_MODE_PROT32:
2154 cs_sel = (u16)(msr_data + 16);
2155 if ((msr_data & 0xfffc) == 0x0) {
2156 emulate_gp(ctxt, 0);
2157 return X86EMUL_PROPAGATE_FAULT;
2159 ss_sel = (u16)(msr_data + 24);
2161 case X86EMUL_MODE_PROT64:
2162 cs_sel = (u16)(msr_data + 32);
2163 if (msr_data == 0x0) {
2164 emulate_gp(ctxt, 0);
2165 return X86EMUL_PROPAGATE_FAULT;
2167 ss_sel = cs_sel + 8;
2172 cs_sel |= SELECTOR_RPL_MASK;
2173 ss_sel |= SELECTOR_RPL_MASK;
2175 ops->set_cached_descriptor(&cs, VCPU_SREG_CS, ctxt->vcpu);
2176 ops->set_segment_selector(cs_sel, VCPU_SREG_CS, ctxt->vcpu);
2177 ops->set_cached_descriptor(&ss, VCPU_SREG_SS, ctxt->vcpu);
2178 ops->set_segment_selector(ss_sel, VCPU_SREG_SS, ctxt->vcpu);
2180 c->eip = c->regs[VCPU_REGS_RDX];
2181 c->regs[VCPU_REGS_RSP] = c->regs[VCPU_REGS_RCX];
2183 return X86EMUL_CONTINUE;
2186 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt,
2187 struct x86_emulate_ops *ops)
2190 if (ctxt->mode == X86EMUL_MODE_REAL)
2192 if (ctxt->mode == X86EMUL_MODE_VM86)
2194 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2195 return ops->cpl(ctxt->vcpu) > iopl;
2198 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2199 struct x86_emulate_ops *ops,
2202 struct desc_struct tr_seg;
2205 u8 perm, bit_idx = port & 0x7;
2206 unsigned mask = (1 << len) - 1;
2208 ops->get_cached_descriptor(&tr_seg, VCPU_SREG_TR, ctxt->vcpu);
2211 if (desc_limit_scaled(&tr_seg) < 103)
2213 r = ops->read_std(get_desc_base(&tr_seg) + 102, &io_bitmap_ptr, 2,
2215 if (r != X86EMUL_CONTINUE)
2217 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2219 r = ops->read_std(get_desc_base(&tr_seg) + io_bitmap_ptr + port/8,
2220 &perm, 1, ctxt->vcpu, NULL);
2221 if (r != X86EMUL_CONTINUE)
2223 if ((perm >> bit_idx) & mask)
2228 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2229 struct x86_emulate_ops *ops,
2232 if (emulator_bad_iopl(ctxt, ops))
2233 if (!emulator_io_port_access_allowed(ctxt, ops, port, len))
2238 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2239 struct x86_emulate_ops *ops,
2240 struct tss_segment_16 *tss)
2242 struct decode_cache *c = &ctxt->decode;
2245 tss->flag = ctxt->eflags;
2246 tss->ax = c->regs[VCPU_REGS_RAX];
2247 tss->cx = c->regs[VCPU_REGS_RCX];
2248 tss->dx = c->regs[VCPU_REGS_RDX];
2249 tss->bx = c->regs[VCPU_REGS_RBX];
2250 tss->sp = c->regs[VCPU_REGS_RSP];
2251 tss->bp = c->regs[VCPU_REGS_RBP];
2252 tss->si = c->regs[VCPU_REGS_RSI];
2253 tss->di = c->regs[VCPU_REGS_RDI];
2255 tss->es = ops->get_segment_selector(VCPU_SREG_ES, ctxt->vcpu);
2256 tss->cs = ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
2257 tss->ss = ops->get_segment_selector(VCPU_SREG_SS, ctxt->vcpu);
2258 tss->ds = ops->get_segment_selector(VCPU_SREG_DS, ctxt->vcpu);
2259 tss->ldt = ops->get_segment_selector(VCPU_SREG_LDTR, ctxt->vcpu);
2262 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2263 struct x86_emulate_ops *ops,
2264 struct tss_segment_16 *tss)
2266 struct decode_cache *c = &ctxt->decode;
2270 ctxt->eflags = tss->flag | 2;
2271 c->regs[VCPU_REGS_RAX] = tss->ax;
2272 c->regs[VCPU_REGS_RCX] = tss->cx;
2273 c->regs[VCPU_REGS_RDX] = tss->dx;
2274 c->regs[VCPU_REGS_RBX] = tss->bx;
2275 c->regs[VCPU_REGS_RSP] = tss->sp;
2276 c->regs[VCPU_REGS_RBP] = tss->bp;
2277 c->regs[VCPU_REGS_RSI] = tss->si;
2278 c->regs[VCPU_REGS_RDI] = tss->di;
2281 * SDM says that segment selectors are loaded before segment
2284 ops->set_segment_selector(tss->ldt, VCPU_SREG_LDTR, ctxt->vcpu);
2285 ops->set_segment_selector(tss->es, VCPU_SREG_ES, ctxt->vcpu);
2286 ops->set_segment_selector(tss->cs, VCPU_SREG_CS, ctxt->vcpu);
2287 ops->set_segment_selector(tss->ss, VCPU_SREG_SS, ctxt->vcpu);
2288 ops->set_segment_selector(tss->ds, VCPU_SREG_DS, ctxt->vcpu);
2291 * Now load segment descriptors. If fault happenes at this stage
2292 * it is handled in a context of new task
2294 ret = load_segment_descriptor(ctxt, ops, tss->ldt, VCPU_SREG_LDTR);
2295 if (ret != X86EMUL_CONTINUE)
2297 ret = load_segment_descriptor(ctxt, ops, tss->es, VCPU_SREG_ES);
2298 if (ret != X86EMUL_CONTINUE)
2300 ret = load_segment_descriptor(ctxt, ops, tss->cs, VCPU_SREG_CS);
2301 if (ret != X86EMUL_CONTINUE)
2303 ret = load_segment_descriptor(ctxt, ops, tss->ss, VCPU_SREG_SS);
2304 if (ret != X86EMUL_CONTINUE)
2306 ret = load_segment_descriptor(ctxt, ops, tss->ds, VCPU_SREG_DS);
2307 if (ret != X86EMUL_CONTINUE)
2310 return X86EMUL_CONTINUE;
2313 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2314 struct x86_emulate_ops *ops,
2315 u16 tss_selector, u16 old_tss_sel,
2316 ulong old_tss_base, struct desc_struct *new_desc)
2318 struct tss_segment_16 tss_seg;
2320 u32 err, new_tss_base = get_desc_base(new_desc);
2322 ret = ops->read_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2324 if (ret == X86EMUL_PROPAGATE_FAULT) {
2325 /* FIXME: need to provide precise fault address */
2326 emulate_pf(ctxt, old_tss_base, err);
2330 save_state_to_tss16(ctxt, ops, &tss_seg);
2332 ret = ops->write_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2334 if (ret == X86EMUL_PROPAGATE_FAULT) {
2335 /* FIXME: need to provide precise fault address */
2336 emulate_pf(ctxt, old_tss_base, err);
2340 ret = ops->read_std(new_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2342 if (ret == X86EMUL_PROPAGATE_FAULT) {
2343 /* FIXME: need to provide precise fault address */
2344 emulate_pf(ctxt, new_tss_base, err);
2348 if (old_tss_sel != 0xffff) {
2349 tss_seg.prev_task_link = old_tss_sel;
2351 ret = ops->write_std(new_tss_base,
2352 &tss_seg.prev_task_link,
2353 sizeof tss_seg.prev_task_link,
2355 if (ret == X86EMUL_PROPAGATE_FAULT) {
2356 /* FIXME: need to provide precise fault address */
2357 emulate_pf(ctxt, new_tss_base, err);
2362 return load_state_from_tss16(ctxt, ops, &tss_seg);
2365 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2366 struct x86_emulate_ops *ops,
2367 struct tss_segment_32 *tss)
2369 struct decode_cache *c = &ctxt->decode;
2371 tss->cr3 = ops->get_cr(3, ctxt->vcpu);
2373 tss->eflags = ctxt->eflags;
2374 tss->eax = c->regs[VCPU_REGS_RAX];
2375 tss->ecx = c->regs[VCPU_REGS_RCX];
2376 tss->edx = c->regs[VCPU_REGS_RDX];
2377 tss->ebx = c->regs[VCPU_REGS_RBX];
2378 tss->esp = c->regs[VCPU_REGS_RSP];
2379 tss->ebp = c->regs[VCPU_REGS_RBP];
2380 tss->esi = c->regs[VCPU_REGS_RSI];
2381 tss->edi = c->regs[VCPU_REGS_RDI];
2383 tss->es = ops->get_segment_selector(VCPU_SREG_ES, ctxt->vcpu);
2384 tss->cs = ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
2385 tss->ss = ops->get_segment_selector(VCPU_SREG_SS, ctxt->vcpu);
2386 tss->ds = ops->get_segment_selector(VCPU_SREG_DS, ctxt->vcpu);
2387 tss->fs = ops->get_segment_selector(VCPU_SREG_FS, ctxt->vcpu);
2388 tss->gs = ops->get_segment_selector(VCPU_SREG_GS, ctxt->vcpu);
2389 tss->ldt_selector = ops->get_segment_selector(VCPU_SREG_LDTR, ctxt->vcpu);
2392 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2393 struct x86_emulate_ops *ops,
2394 struct tss_segment_32 *tss)
2396 struct decode_cache *c = &ctxt->decode;
2399 if (ops->set_cr(3, tss->cr3, ctxt->vcpu)) {
2400 emulate_gp(ctxt, 0);
2401 return X86EMUL_PROPAGATE_FAULT;
2404 ctxt->eflags = tss->eflags | 2;
2405 c->regs[VCPU_REGS_RAX] = tss->eax;
2406 c->regs[VCPU_REGS_RCX] = tss->ecx;
2407 c->regs[VCPU_REGS_RDX] = tss->edx;
2408 c->regs[VCPU_REGS_RBX] = tss->ebx;
2409 c->regs[VCPU_REGS_RSP] = tss->esp;
2410 c->regs[VCPU_REGS_RBP] = tss->ebp;
2411 c->regs[VCPU_REGS_RSI] = tss->esi;
2412 c->regs[VCPU_REGS_RDI] = tss->edi;
2415 * SDM says that segment selectors are loaded before segment
2418 ops->set_segment_selector(tss->ldt_selector, VCPU_SREG_LDTR, ctxt->vcpu);
2419 ops->set_segment_selector(tss->es, VCPU_SREG_ES, ctxt->vcpu);
2420 ops->set_segment_selector(tss->cs, VCPU_SREG_CS, ctxt->vcpu);
2421 ops->set_segment_selector(tss->ss, VCPU_SREG_SS, ctxt->vcpu);
2422 ops->set_segment_selector(tss->ds, VCPU_SREG_DS, ctxt->vcpu);
2423 ops->set_segment_selector(tss->fs, VCPU_SREG_FS, ctxt->vcpu);
2424 ops->set_segment_selector(tss->gs, VCPU_SREG_GS, ctxt->vcpu);
2427 * Now load segment descriptors. If fault happenes at this stage
2428 * it is handled in a context of new task
2430 ret = load_segment_descriptor(ctxt, ops, tss->ldt_selector, VCPU_SREG_LDTR);
2431 if (ret != X86EMUL_CONTINUE)
2433 ret = load_segment_descriptor(ctxt, ops, tss->es, VCPU_SREG_ES);
2434 if (ret != X86EMUL_CONTINUE)
2436 ret = load_segment_descriptor(ctxt, ops, tss->cs, VCPU_SREG_CS);
2437 if (ret != X86EMUL_CONTINUE)
2439 ret = load_segment_descriptor(ctxt, ops, tss->ss, VCPU_SREG_SS);
2440 if (ret != X86EMUL_CONTINUE)
2442 ret = load_segment_descriptor(ctxt, ops, tss->ds, VCPU_SREG_DS);
2443 if (ret != X86EMUL_CONTINUE)
2445 ret = load_segment_descriptor(ctxt, ops, tss->fs, VCPU_SREG_FS);
2446 if (ret != X86EMUL_CONTINUE)
2448 ret = load_segment_descriptor(ctxt, ops, tss->gs, VCPU_SREG_GS);
2449 if (ret != X86EMUL_CONTINUE)
2452 return X86EMUL_CONTINUE;
2455 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2456 struct x86_emulate_ops *ops,
2457 u16 tss_selector, u16 old_tss_sel,
2458 ulong old_tss_base, struct desc_struct *new_desc)
2460 struct tss_segment_32 tss_seg;
2462 u32 err, new_tss_base = get_desc_base(new_desc);
2464 ret = ops->read_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2466 if (ret == X86EMUL_PROPAGATE_FAULT) {
2467 /* FIXME: need to provide precise fault address */
2468 emulate_pf(ctxt, old_tss_base, err);
2472 save_state_to_tss32(ctxt, ops, &tss_seg);
2474 ret = ops->write_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2476 if (ret == X86EMUL_PROPAGATE_FAULT) {
2477 /* FIXME: need to provide precise fault address */
2478 emulate_pf(ctxt, old_tss_base, err);
2482 ret = ops->read_std(new_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2484 if (ret == X86EMUL_PROPAGATE_FAULT) {
2485 /* FIXME: need to provide precise fault address */
2486 emulate_pf(ctxt, new_tss_base, err);
2490 if (old_tss_sel != 0xffff) {
2491 tss_seg.prev_task_link = old_tss_sel;
2493 ret = ops->write_std(new_tss_base,
2494 &tss_seg.prev_task_link,
2495 sizeof tss_seg.prev_task_link,
2497 if (ret == X86EMUL_PROPAGATE_FAULT) {
2498 /* FIXME: need to provide precise fault address */
2499 emulate_pf(ctxt, new_tss_base, err);
2504 return load_state_from_tss32(ctxt, ops, &tss_seg);
2507 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2508 struct x86_emulate_ops *ops,
2509 u16 tss_selector, int reason,
2510 bool has_error_code, u32 error_code)
2512 struct desc_struct curr_tss_desc, next_tss_desc;
2514 u16 old_tss_sel = ops->get_segment_selector(VCPU_SREG_TR, ctxt->vcpu);
2515 ulong old_tss_base =
2516 ops->get_cached_segment_base(VCPU_SREG_TR, ctxt->vcpu);
2519 /* FIXME: old_tss_base == ~0 ? */
2521 ret = read_segment_descriptor(ctxt, ops, tss_selector, &next_tss_desc);
2522 if (ret != X86EMUL_CONTINUE)
2524 ret = read_segment_descriptor(ctxt, ops, old_tss_sel, &curr_tss_desc);
2525 if (ret != X86EMUL_CONTINUE)
2528 /* FIXME: check that next_tss_desc is tss */
2530 if (reason != TASK_SWITCH_IRET) {
2531 if ((tss_selector & 3) > next_tss_desc.dpl ||
2532 ops->cpl(ctxt->vcpu) > next_tss_desc.dpl) {
2533 emulate_gp(ctxt, 0);
2534 return X86EMUL_PROPAGATE_FAULT;
2538 desc_limit = desc_limit_scaled(&next_tss_desc);
2539 if (!next_tss_desc.p ||
2540 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2541 desc_limit < 0x2b)) {
2542 emulate_ts(ctxt, tss_selector & 0xfffc);
2543 return X86EMUL_PROPAGATE_FAULT;
2546 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2547 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2548 write_segment_descriptor(ctxt, ops, old_tss_sel,
2552 if (reason == TASK_SWITCH_IRET)
2553 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2555 /* set back link to prev task only if NT bit is set in eflags
2556 note that old_tss_sel is not used afetr this point */
2557 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2558 old_tss_sel = 0xffff;
2560 if (next_tss_desc.type & 8)
2561 ret = task_switch_32(ctxt, ops, tss_selector, old_tss_sel,
2562 old_tss_base, &next_tss_desc);
2564 ret = task_switch_16(ctxt, ops, tss_selector, old_tss_sel,
2565 old_tss_base, &next_tss_desc);
2566 if (ret != X86EMUL_CONTINUE)
2569 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2570 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2572 if (reason != TASK_SWITCH_IRET) {
2573 next_tss_desc.type |= (1 << 1); /* set busy flag */
2574 write_segment_descriptor(ctxt, ops, tss_selector,
2578 ops->set_cr(0, ops->get_cr(0, ctxt->vcpu) | X86_CR0_TS, ctxt->vcpu);
2579 ops->set_cached_descriptor(&next_tss_desc, VCPU_SREG_TR, ctxt->vcpu);
2580 ops->set_segment_selector(tss_selector, VCPU_SREG_TR, ctxt->vcpu);
2582 if (has_error_code) {
2583 struct decode_cache *c = &ctxt->decode;
2585 c->op_bytes = c->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2587 c->src.val = (unsigned long) error_code;
2588 emulate_push(ctxt, ops);
2594 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2595 struct x86_emulate_ops *ops,
2596 u16 tss_selector, int reason,
2597 bool has_error_code, u32 error_code)
2599 struct decode_cache *c = &ctxt->decode;
2603 c->dst.type = OP_NONE;
2605 rc = emulator_do_task_switch(ctxt, ops, tss_selector, reason,
2606 has_error_code, error_code);
2608 if (rc == X86EMUL_CONTINUE) {
2609 rc = writeback(ctxt, ops);
2610 if (rc == X86EMUL_CONTINUE)
2614 return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0;
2617 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned long base,
2618 int reg, struct operand *op)
2620 struct decode_cache *c = &ctxt->decode;
2621 int df = (ctxt->eflags & EFLG_DF) ? -1 : 1;
2623 register_address_increment(c, &c->regs[reg], df * op->bytes);
2624 op->ptr = (unsigned long *)register_address(c, base, c->regs[reg]);
2628 x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
2631 struct decode_cache *c = &ctxt->decode;
2632 int rc = X86EMUL_CONTINUE;
2633 int saved_dst_type = c->dst.type;
2635 ctxt->decode.mem_read.pos = 0;
2637 if (ctxt->mode == X86EMUL_MODE_PROT64 && (c->d & No64)) {
2642 /* LOCK prefix is allowed only with some instructions */
2643 if (c->lock_prefix && (!(c->d & Lock) || c->dst.type != OP_MEM)) {
2648 /* Privileged instruction can be executed only in CPL=0 */
2649 if ((c->d & Priv) && ops->cpl(ctxt->vcpu)) {
2650 emulate_gp(ctxt, 0);
2654 if (c->rep_prefix && (c->d & String)) {
2655 ctxt->restart = true;
2656 /* All REP prefixes have the same first termination condition */
2657 if (address_mask(c, c->regs[VCPU_REGS_RCX]) == 0) {
2659 ctxt->restart = false;
2663 /* The second termination condition only applies for REPE
2664 * and REPNE. Test if the repeat string operation prefix is
2665 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
2666 * corresponding termination condition according to:
2667 * - if REPE/REPZ and ZF = 0 then done
2668 * - if REPNE/REPNZ and ZF = 1 then done
2670 if ((c->b == 0xa6) || (c->b == 0xa7) ||
2671 (c->b == 0xae) || (c->b == 0xaf)) {
2672 if ((c->rep_prefix == REPE_PREFIX) &&
2673 ((ctxt->eflags & EFLG_ZF) == 0))
2675 if ((c->rep_prefix == REPNE_PREFIX) &&
2676 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))
2682 if (c->src.type == OP_MEM) {
2683 rc = read_emulated(ctxt, ops, (unsigned long)c->src.ptr,
2684 c->src.valptr, c->src.bytes);
2685 if (rc != X86EMUL_CONTINUE)
2687 c->src.orig_val64 = c->src.val64;
2690 if (c->src2.type == OP_MEM) {
2691 rc = read_emulated(ctxt, ops, (unsigned long)c->src2.ptr,
2692 &c->src2.val, c->src2.bytes);
2693 if (rc != X86EMUL_CONTINUE)
2697 if ((c->d & DstMask) == ImplicitOps)
2701 if ((c->dst.type == OP_MEM) && !(c->d & Mov)) {
2702 /* optimisation - avoid slow emulated read if Mov */
2703 rc = read_emulated(ctxt, ops, (unsigned long)c->dst.ptr,
2704 &c->dst.val, c->dst.bytes);
2705 if (rc != X86EMUL_CONTINUE)
2708 c->dst.orig_val = c->dst.val;
2718 emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
2720 case 0x06: /* push es */
2721 emulate_push_sreg(ctxt, ops, VCPU_SREG_ES);
2723 case 0x07: /* pop es */
2724 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_ES);
2725 if (rc != X86EMUL_CONTINUE)
2730 emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
2732 case 0x0e: /* push cs */
2733 emulate_push_sreg(ctxt, ops, VCPU_SREG_CS);
2737 emulate_2op_SrcV("adc", c->src, c->dst, ctxt->eflags);
2739 case 0x16: /* push ss */
2740 emulate_push_sreg(ctxt, ops, VCPU_SREG_SS);
2742 case 0x17: /* pop ss */
2743 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_SS);
2744 if (rc != X86EMUL_CONTINUE)
2749 emulate_2op_SrcV("sbb", c->src, c->dst, ctxt->eflags);
2751 case 0x1e: /* push ds */
2752 emulate_push_sreg(ctxt, ops, VCPU_SREG_DS);
2754 case 0x1f: /* pop ds */
2755 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_DS);
2756 if (rc != X86EMUL_CONTINUE)
2761 emulate_2op_SrcV("and", c->src, c->dst, ctxt->eflags);
2765 emulate_2op_SrcV("sub", c->src, c->dst, ctxt->eflags);
2769 emulate_2op_SrcV("xor", c->src, c->dst, ctxt->eflags);
2773 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
2775 case 0x40 ... 0x47: /* inc r16/r32 */
2776 emulate_1op("inc", c->dst, ctxt->eflags);
2778 case 0x48 ... 0x4f: /* dec r16/r32 */
2779 emulate_1op("dec", c->dst, ctxt->eflags);
2781 case 0x50 ... 0x57: /* push reg */
2782 emulate_push(ctxt, ops);
2784 case 0x58 ... 0x5f: /* pop reg */
2786 rc = emulate_pop(ctxt, ops, &c->dst.val, c->op_bytes);
2787 if (rc != X86EMUL_CONTINUE)
2790 case 0x60: /* pusha */
2791 rc = emulate_pusha(ctxt, ops);
2792 if (rc != X86EMUL_CONTINUE)
2795 case 0x61: /* popa */
2796 rc = emulate_popa(ctxt, ops);
2797 if (rc != X86EMUL_CONTINUE)
2800 case 0x63: /* movsxd */
2801 if (ctxt->mode != X86EMUL_MODE_PROT64)
2802 goto cannot_emulate;
2803 c->dst.val = (s32) c->src.val;
2805 case 0x68: /* push imm */
2806 case 0x6a: /* push imm8 */
2807 emulate_push(ctxt, ops);
2809 case 0x6c: /* insb */
2810 case 0x6d: /* insw/insd */
2811 c->dst.bytes = min(c->dst.bytes, 4u);
2812 if (!emulator_io_permited(ctxt, ops, c->regs[VCPU_REGS_RDX],
2814 emulate_gp(ctxt, 0);
2817 if (!pio_in_emulated(ctxt, ops, c->dst.bytes,
2818 c->regs[VCPU_REGS_RDX], &c->dst.val))
2819 goto done; /* IO is needed, skip writeback */
2821 case 0x6e: /* outsb */
2822 case 0x6f: /* outsw/outsd */
2823 c->src.bytes = min(c->src.bytes, 4u);
2824 if (!emulator_io_permited(ctxt, ops, c->regs[VCPU_REGS_RDX],
2826 emulate_gp(ctxt, 0);
2829 ops->pio_out_emulated(c->src.bytes, c->regs[VCPU_REGS_RDX],
2830 &c->src.val, 1, ctxt->vcpu);
2832 c->dst.type = OP_NONE; /* nothing to writeback */
2834 case 0x70 ... 0x7f: /* jcc (short) */
2835 if (test_cc(c->b, ctxt->eflags))
2836 jmp_rel(c, c->src.val);
2838 case 0x80 ... 0x83: /* Grp1 */
2839 switch (c->modrm_reg) {
2860 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
2862 case 0x86 ... 0x87: /* xchg */
2864 /* Write back the register source. */
2865 switch (c->dst.bytes) {
2867 *(u8 *) c->src.ptr = (u8) c->dst.val;
2870 *(u16 *) c->src.ptr = (u16) c->dst.val;
2873 *c->src.ptr = (u32) c->dst.val;
2874 break; /* 64b reg: zero-extend */
2876 *c->src.ptr = c->dst.val;
2880 * Write back the memory destination with implicit LOCK
2883 c->dst.val = c->src.val;
2886 case 0x88 ... 0x8b: /* mov */
2888 case 0x8c: /* mov r/m, sreg */
2889 if (c->modrm_reg > VCPU_SREG_GS) {
2893 c->dst.val = ops->get_segment_selector(c->modrm_reg, ctxt->vcpu);
2895 case 0x8d: /* lea r16/r32, m */
2896 c->dst.val = c->modrm_ea;
2898 case 0x8e: { /* mov seg, r/m16 */
2903 if (c->modrm_reg == VCPU_SREG_CS ||
2904 c->modrm_reg > VCPU_SREG_GS) {
2909 if (c->modrm_reg == VCPU_SREG_SS)
2910 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
2912 rc = load_segment_descriptor(ctxt, ops, sel, c->modrm_reg);
2914 c->dst.type = OP_NONE; /* Disable writeback. */
2917 case 0x8f: /* pop (sole member of Grp1a) */
2918 rc = emulate_grp1a(ctxt, ops);
2919 if (rc != X86EMUL_CONTINUE)
2922 case 0x90: /* nop / xchg r8,rax */
2923 if (c->dst.ptr == (unsigned long *)&c->regs[VCPU_REGS_RAX]) {
2924 c->dst.type = OP_NONE; /* nop */
2927 case 0x91 ... 0x97: /* xchg reg,rax */
2928 c->src.type = OP_REG;
2929 c->src.bytes = c->op_bytes;
2930 c->src.ptr = (unsigned long *) &c->regs[VCPU_REGS_RAX];
2931 c->src.val = *(c->src.ptr);
2933 case 0x9c: /* pushf */
2934 c->src.val = (unsigned long) ctxt->eflags;
2935 emulate_push(ctxt, ops);
2937 case 0x9d: /* popf */
2938 c->dst.type = OP_REG;
2939 c->dst.ptr = (unsigned long *) &ctxt->eflags;
2940 c->dst.bytes = c->op_bytes;
2941 rc = emulate_popf(ctxt, ops, &c->dst.val, c->op_bytes);
2942 if (rc != X86EMUL_CONTINUE)
2945 case 0xa0 ... 0xa3: /* mov */
2946 case 0xa4 ... 0xa5: /* movs */
2948 case 0xa6 ... 0xa7: /* cmps */
2949 c->dst.type = OP_NONE; /* Disable writeback. */
2950 DPRINTF("cmps: mem1=0x%p mem2=0x%p\n", c->src.ptr, c->dst.ptr);
2952 case 0xa8 ... 0xa9: /* test ax, imm */
2954 case 0xaa ... 0xab: /* stos */
2955 c->dst.val = c->regs[VCPU_REGS_RAX];
2957 case 0xac ... 0xad: /* lods */
2959 case 0xae ... 0xaf: /* scas */
2960 DPRINTF("Urk! I don't handle SCAS.\n");
2961 goto cannot_emulate;
2962 case 0xb0 ... 0xbf: /* mov r, imm */
2967 case 0xc3: /* ret */
2968 c->dst.type = OP_REG;
2969 c->dst.ptr = &c->eip;
2970 c->dst.bytes = c->op_bytes;
2971 goto pop_instruction;
2972 case 0xc6 ... 0xc7: /* mov (sole member of Grp11) */
2974 c->dst.val = c->src.val;
2976 case 0xcb: /* ret far */
2977 rc = emulate_ret_far(ctxt, ops);
2978 if (rc != X86EMUL_CONTINUE)
2981 case 0xcf: /* iret */
2982 rc = emulate_iret(ctxt, ops);
2984 if (rc != X86EMUL_CONTINUE)
2987 case 0xd0 ... 0xd1: /* Grp2 */
2991 case 0xd2 ... 0xd3: /* Grp2 */
2992 c->src.val = c->regs[VCPU_REGS_RCX];
2995 case 0xe4: /* inb */
2998 case 0xe6: /* outb */
2999 case 0xe7: /* out */
3001 case 0xe8: /* call (near) */ {
3002 long int rel = c->src.val;
3003 c->src.val = (unsigned long) c->eip;
3005 emulate_push(ctxt, ops);
3008 case 0xe9: /* jmp rel */
3010 case 0xea: { /* jmp far */
3013 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
3015 if (load_segment_descriptor(ctxt, ops, sel, VCPU_SREG_CS))
3019 memcpy(&c->eip, c->src.valptr, c->op_bytes);
3023 jmp: /* jmp rel short */
3024 jmp_rel(c, c->src.val);
3025 c->dst.type = OP_NONE; /* Disable writeback. */
3027 case 0xec: /* in al,dx */
3028 case 0xed: /* in (e/r)ax,dx */
3029 c->src.val = c->regs[VCPU_REGS_RDX];
3031 c->dst.bytes = min(c->dst.bytes, 4u);
3032 if (!emulator_io_permited(ctxt, ops, c->src.val, c->dst.bytes)) {
3033 emulate_gp(ctxt, 0);
3036 if (!pio_in_emulated(ctxt, ops, c->dst.bytes, c->src.val,
3038 goto done; /* IO is needed */
3040 case 0xee: /* out dx,al */
3041 case 0xef: /* out dx,(e/r)ax */
3042 c->src.val = c->regs[VCPU_REGS_RDX];
3044 c->dst.bytes = min(c->dst.bytes, 4u);
3045 if (!emulator_io_permited(ctxt, ops, c->src.val, c->dst.bytes)) {
3046 emulate_gp(ctxt, 0);
3049 ops->pio_out_emulated(c->dst.bytes, c->src.val, &c->dst.val, 1,
3051 c->dst.type = OP_NONE; /* Disable writeback. */
3053 case 0xf4: /* hlt */
3054 ctxt->vcpu->arch.halt_request = 1;
3056 case 0xf5: /* cmc */
3057 /* complement carry flag from eflags reg */
3058 ctxt->eflags ^= EFLG_CF;
3059 c->dst.type = OP_NONE; /* Disable writeback. */
3061 case 0xf6 ... 0xf7: /* Grp3 */
3062 if (!emulate_grp3(ctxt, ops))
3063 goto cannot_emulate;
3065 case 0xf8: /* clc */
3066 ctxt->eflags &= ~EFLG_CF;
3067 c->dst.type = OP_NONE; /* Disable writeback. */
3069 case 0xfa: /* cli */
3070 if (emulator_bad_iopl(ctxt, ops)) {
3071 emulate_gp(ctxt, 0);
3074 ctxt->eflags &= ~X86_EFLAGS_IF;
3075 c->dst.type = OP_NONE; /* Disable writeback. */
3078 case 0xfb: /* sti */
3079 if (emulator_bad_iopl(ctxt, ops)) {
3080 emulate_gp(ctxt, 0);
3083 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3084 ctxt->eflags |= X86_EFLAGS_IF;
3085 c->dst.type = OP_NONE; /* Disable writeback. */
3088 case 0xfc: /* cld */
3089 ctxt->eflags &= ~EFLG_DF;
3090 c->dst.type = OP_NONE; /* Disable writeback. */
3092 case 0xfd: /* std */
3093 ctxt->eflags |= EFLG_DF;
3094 c->dst.type = OP_NONE; /* Disable writeback. */
3096 case 0xfe: /* Grp4 */
3098 rc = emulate_grp45(ctxt, ops);
3099 if (rc != X86EMUL_CONTINUE)
3102 case 0xff: /* Grp5 */
3103 if (c->modrm_reg == 5)
3107 goto cannot_emulate;
3111 rc = writeback(ctxt, ops);
3112 if (rc != X86EMUL_CONTINUE)
3116 * restore dst type in case the decoding will be reused
3117 * (happens for string instruction )
3119 c->dst.type = saved_dst_type;
3121 if ((c->d & SrcMask) == SrcSI)
3122 string_addr_inc(ctxt, seg_override_base(ctxt, ops, c),
3123 VCPU_REGS_RSI, &c->src);
3125 if ((c->d & DstMask) == DstDI)
3126 string_addr_inc(ctxt, es_base(ctxt, ops), VCPU_REGS_RDI,
3129 if (c->rep_prefix && (c->d & String)) {
3130 struct read_cache *rc = &ctxt->decode.io_read;
3131 register_address_increment(c, &c->regs[VCPU_REGS_RCX], -1);
3133 * Re-enter guest when pio read ahead buffer is empty or,
3134 * if it is not used, after each 1024 iteration.
3136 if ((rc->end == 0 && !(c->regs[VCPU_REGS_RCX] & 0x3ff)) ||
3137 (rc->end != 0 && rc->end == rc->pos))
3138 ctxt->restart = false;
3141 * reset read cache here in case string instruction is restared
3144 ctxt->decode.mem_read.end = 0;
3148 return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0;
3152 case 0x01: /* lgdt, lidt, lmsw */
3153 switch (c->modrm_reg) {
3155 unsigned long address;
3157 case 0: /* vmcall */
3158 if (c->modrm_mod != 3 || c->modrm_rm != 1)
3159 goto cannot_emulate;
3161 rc = kvm_fix_hypercall(ctxt->vcpu);
3162 if (rc != X86EMUL_CONTINUE)
3165 /* Let the processor re-execute the fixed hypercall */
3167 /* Disable writeback. */
3168 c->dst.type = OP_NONE;
3171 rc = read_descriptor(ctxt, ops, c->src.ptr,
3172 &size, &address, c->op_bytes);
3173 if (rc != X86EMUL_CONTINUE)
3175 realmode_lgdt(ctxt->vcpu, size, address);
3176 /* Disable writeback. */
3177 c->dst.type = OP_NONE;
3179 case 3: /* lidt/vmmcall */
3180 if (c->modrm_mod == 3) {
3181 switch (c->modrm_rm) {
3183 rc = kvm_fix_hypercall(ctxt->vcpu);
3184 if (rc != X86EMUL_CONTINUE)
3188 goto cannot_emulate;
3191 rc = read_descriptor(ctxt, ops, c->src.ptr,
3194 if (rc != X86EMUL_CONTINUE)
3196 realmode_lidt(ctxt->vcpu, size, address);
3198 /* Disable writeback. */
3199 c->dst.type = OP_NONE;
3203 c->dst.val = ops->get_cr(0, ctxt->vcpu);
3206 ops->set_cr(0, (ops->get_cr(0, ctxt->vcpu) & ~0x0ful) |
3207 (c->src.val & 0x0f), ctxt->vcpu);
3208 c->dst.type = OP_NONE;
3210 case 5: /* not defined */
3214 emulate_invlpg(ctxt->vcpu, c->modrm_ea);
3215 /* Disable writeback. */
3216 c->dst.type = OP_NONE;
3219 goto cannot_emulate;
3222 case 0x05: /* syscall */
3223 rc = emulate_syscall(ctxt, ops);
3224 if (rc != X86EMUL_CONTINUE)
3230 emulate_clts(ctxt->vcpu);
3231 c->dst.type = OP_NONE;
3233 case 0x09: /* wbinvd */
3234 kvm_emulate_wbinvd(ctxt->vcpu);
3235 c->dst.type = OP_NONE;
3237 case 0x08: /* invd */
3238 case 0x0d: /* GrpP (prefetch) */
3239 case 0x18: /* Grp16 (prefetch/nop) */
3240 c->dst.type = OP_NONE;
3242 case 0x20: /* mov cr, reg */
3243 switch (c->modrm_reg) {
3250 c->regs[c->modrm_rm] = ops->get_cr(c->modrm_reg, ctxt->vcpu);
3251 c->dst.type = OP_NONE; /* no writeback */
3253 case 0x21: /* mov from dr to reg */
3254 if ((ops->get_cr(4, ctxt->vcpu) & X86_CR4_DE) &&
3255 (c->modrm_reg == 4 || c->modrm_reg == 5)) {
3259 ops->get_dr(c->modrm_reg, &c->regs[c->modrm_rm], ctxt->vcpu);
3260 c->dst.type = OP_NONE; /* no writeback */
3262 case 0x22: /* mov reg, cr */
3263 if (ops->set_cr(c->modrm_reg, c->modrm_val, ctxt->vcpu)) {
3264 emulate_gp(ctxt, 0);
3267 c->dst.type = OP_NONE;
3269 case 0x23: /* mov from reg to dr */
3270 if ((ops->get_cr(4, ctxt->vcpu) & X86_CR4_DE) &&
3271 (c->modrm_reg == 4 || c->modrm_reg == 5)) {
3276 if (ops->set_dr(c->modrm_reg, c->regs[c->modrm_rm] &
3277 ((ctxt->mode == X86EMUL_MODE_PROT64) ?
3278 ~0ULL : ~0U), ctxt->vcpu) < 0) {
3279 /* #UD condition is already handled by the code above */
3280 emulate_gp(ctxt, 0);
3284 c->dst.type = OP_NONE; /* no writeback */
3288 msr_data = (u32)c->regs[VCPU_REGS_RAX]
3289 | ((u64)c->regs[VCPU_REGS_RDX] << 32);
3290 if (ops->set_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], msr_data)) {
3291 emulate_gp(ctxt, 0);
3294 rc = X86EMUL_CONTINUE;
3295 c->dst.type = OP_NONE;
3299 if (ops->get_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], &msr_data)) {
3300 emulate_gp(ctxt, 0);
3303 c->regs[VCPU_REGS_RAX] = (u32)msr_data;
3304 c->regs[VCPU_REGS_RDX] = msr_data >> 32;
3306 rc = X86EMUL_CONTINUE;
3307 c->dst.type = OP_NONE;
3309 case 0x34: /* sysenter */
3310 rc = emulate_sysenter(ctxt, ops);
3311 if (rc != X86EMUL_CONTINUE)
3316 case 0x35: /* sysexit */
3317 rc = emulate_sysexit(ctxt, ops);
3318 if (rc != X86EMUL_CONTINUE)
3323 case 0x40 ... 0x4f: /* cmov */
3324 c->dst.val = c->dst.orig_val = c->src.val;
3325 if (!test_cc(c->b, ctxt->eflags))
3326 c->dst.type = OP_NONE; /* no writeback */
3328 case 0x80 ... 0x8f: /* jnz rel, etc*/
3329 if (test_cc(c->b, ctxt->eflags))
3330 jmp_rel(c, c->src.val);
3331 c->dst.type = OP_NONE;
3333 case 0xa0: /* push fs */
3334 emulate_push_sreg(ctxt, ops, VCPU_SREG_FS);
3336 case 0xa1: /* pop fs */
3337 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_FS);
3338 if (rc != X86EMUL_CONTINUE)
3343 c->dst.type = OP_NONE;
3344 /* only subword offset */
3345 c->src.val &= (c->dst.bytes << 3) - 1;
3346 emulate_2op_SrcV_nobyte("bt", c->src, c->dst, ctxt->eflags);
3348 case 0xa4: /* shld imm8, r, r/m */
3349 case 0xa5: /* shld cl, r, r/m */
3350 emulate_2op_cl("shld", c->src2, c->src, c->dst, ctxt->eflags);
3352 case 0xa8: /* push gs */
3353 emulate_push_sreg(ctxt, ops, VCPU_SREG_GS);
3355 case 0xa9: /* pop gs */
3356 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_GS);
3357 if (rc != X86EMUL_CONTINUE)
3362 /* only subword offset */
3363 c->src.val &= (c->dst.bytes << 3) - 1;
3364 emulate_2op_SrcV_nobyte("bts", c->src, c->dst, ctxt->eflags);
3366 case 0xac: /* shrd imm8, r, r/m */
3367 case 0xad: /* shrd cl, r, r/m */
3368 emulate_2op_cl("shrd", c->src2, c->src, c->dst, ctxt->eflags);
3370 case 0xae: /* clflush */
3372 case 0xb0 ... 0xb1: /* cmpxchg */
3374 * Save real source value, then compare EAX against
3377 c->src.orig_val = c->src.val;
3378 c->src.val = c->regs[VCPU_REGS_RAX];
3379 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
3380 if (ctxt->eflags & EFLG_ZF) {
3381 /* Success: write back to memory. */
3382 c->dst.val = c->src.orig_val;
3384 /* Failure: write the value we saw to EAX. */
3385 c->dst.type = OP_REG;
3386 c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX];
3391 /* only subword offset */
3392 c->src.val &= (c->dst.bytes << 3) - 1;
3393 emulate_2op_SrcV_nobyte("btr", c->src, c->dst, ctxt->eflags);
3395 case 0xb6 ... 0xb7: /* movzx */
3396 c->dst.bytes = c->op_bytes;
3397 c->dst.val = (c->d & ByteOp) ? (u8) c->src.val
3400 case 0xba: /* Grp8 */
3401 switch (c->modrm_reg & 3) {
3414 /* only subword offset */
3415 c->src.val &= (c->dst.bytes << 3) - 1;
3416 emulate_2op_SrcV_nobyte("btc", c->src, c->dst, ctxt->eflags);
3418 case 0xbe ... 0xbf: /* movsx */
3419 c->dst.bytes = c->op_bytes;
3420 c->dst.val = (c->d & ByteOp) ? (s8) c->src.val :
3423 case 0xc3: /* movnti */
3424 c->dst.bytes = c->op_bytes;
3425 c->dst.val = (c->op_bytes == 4) ? (u32) c->src.val :
3428 case 0xc7: /* Grp9 (cmpxchg8b) */
3429 rc = emulate_grp9(ctxt, ops);
3430 if (rc != X86EMUL_CONTINUE)
3434 goto cannot_emulate;
3439 DPRINTF("Cannot emulate %02x\n", c->b);