net: filter: add "load 64-bit immediate" eBPF instruction

[cascardo/linux.git] / arch / x86 / net / bpf_jit_comp.c

diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index 5c8cb80..06f8c17 100644 (file)
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -393,6 +393,23 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
                        EMIT1_off32(add_1reg(0xB8, dst_reg), imm32);
                        break;
 
+               case BPF_LD | BPF_IMM | BPF_DW:
+                       if (insn[1].code != 0 || insn[1].src_reg != 0 ||
+                           insn[1].dst_reg != 0 || insn[1].off != 0) {
+                               /* verifier must catch invalid insns */
+                               pr_err("invalid BPF_LD_IMM64 insn\n");
+                               return -EINVAL;
+                       }
+
+                       /* movabsq %rax, imm64 */
+                       EMIT2(add_1mod(0x48, dst_reg), add_1reg(0xB8, dst_reg));
+                       EMIT(insn[0].imm, 4);
+                       EMIT(insn[1].imm, 4);
+
+                       insn++;
+                       i++;
+                       break;
+
                        /* dst %= src, dst /= src, dst %= imm32, dst /= imm32 */
                case BPF_ALU | BPF_MOD | BPF_X:
                case BPF_ALU | BPF_DIV | BPF_X:
@@ -515,6 +532,48 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
                        EMIT3(0xC1, add_1reg(b3, dst_reg), imm32);
                        break;
 
+               case BPF_ALU | BPF_LSH | BPF_X:
+               case BPF_ALU | BPF_RSH | BPF_X:
+               case BPF_ALU | BPF_ARSH | BPF_X:
+               case BPF_ALU64 | BPF_LSH | BPF_X:
+               case BPF_ALU64 | BPF_RSH | BPF_X:
+               case BPF_ALU64 | BPF_ARSH | BPF_X:
+
+                       /* check for bad case when dst_reg == rcx */
+                       if (dst_reg == BPF_REG_4) {
+                               /* mov r11, dst_reg */
+                               EMIT_mov(AUX_REG, dst_reg);
+                               dst_reg = AUX_REG;
+                       }
+
+                       if (src_reg != BPF_REG_4) { /* common case */
+                               EMIT1(0x51); /* push rcx */
+
+                               /* mov rcx, src_reg */
+                               EMIT_mov(BPF_REG_4, src_reg);
+                       }
+
+                       /* shl %rax, %cl | shr %rax, %cl | sar %rax, %cl */
+                       if (BPF_CLASS(insn->code) == BPF_ALU64)
+                               EMIT1(add_1mod(0x48, dst_reg));
+                       else if (is_ereg(dst_reg))
+                               EMIT1(add_1mod(0x40, dst_reg));
+
+                       switch (BPF_OP(insn->code)) {
+                       case BPF_LSH: b3 = 0xE0; break;
+                       case BPF_RSH: b3 = 0xE8; break;
+                       case BPF_ARSH: b3 = 0xF8; break;
+                       }
+                       EMIT2(0xD3, add_1reg(b3, dst_reg));
+
+                       if (src_reg != BPF_REG_4)
+                               EMIT1(0x59); /* pop rcx */
+
+                       if (insn->dst_reg == BPF_REG_4)
+                               /* mov dst_reg, r11 */
+                               EMIT_mov(insn->dst_reg, AUX_REG);
+                       break;
+
                case BPF_ALU | BPF_END | BPF_FROM_BE:
                        switch (imm32) {
                        case 16:
@@ -930,23 +989,17 @@ out:
        kfree(addrs);
 }
 
-static void bpf_jit_free_deferred(struct work_struct *work)
+void bpf_jit_free(struct bpf_prog *fp)
 {
-       struct bpf_prog *fp = container_of(work, struct bpf_prog, work);
        unsigned long addr = (unsigned long)fp->bpf_func & PAGE_MASK;
        struct bpf_binary_header *header = (void *)addr;
 
+       if (!fp->jited)
+               goto free_filter;
+
        set_memory_rw(addr, header->pages);
        module_free(NULL, header);
-       kfree(fp);
-}
 
-void bpf_jit_free(struct bpf_prog *fp)
-{
-       if (fp->jited) {
-               INIT_WORK(&fp->work, bpf_jit_free_deferred);
-               schedule_work(&fp->work);
-       } else {
-               kfree(fp);
-       }
+free_filter:
+       bpf_prog_unlock_free(fp);
 }