net/core/filter.c

   1 /*
   2  * Linux Socket Filter - Kernel level socket filtering
   3  *
   4  * Based on the design of the Berkeley Packet Filter. The new
   5  * internal format has been designed by PLUMgrid:
   6  *
   7  *      Copyright (c) 2011 - 2014 PLUMgrid, http://plumgrid.com
   8  *
   9  * Authors:
  10  *
  11  *      Jay Schulist <jschlst@samba.org>
  12  *      Alexei Starovoitov <ast@plumgrid.com>
  13  *      Daniel Borkmann <dborkman@redhat.com>
  14  *
  15  * This program is free software; you can redistribute it and/or
  16  * modify it under the terms of the GNU General Public License
  17  * as published by the Free Software Foundation; either version
  18  * 2 of the License, or (at your option) any later version.
  19  *
  20  * Andi Kleen - Fix a few bad bugs and races.
  21  * Kris Katterjohn - Added many additional checks in sk_chk_filter()
  22  */
  23
  24 #include <linux/module.h>
  25 #include <linux/types.h>
  26 #include <linux/mm.h>
  27 #include <linux/fcntl.h>
  28 #include <linux/socket.h>
  29 #include <linux/in.h>
  30 #include <linux/inet.h>
  31 #include <linux/netdevice.h>
  32 #include <linux/if_packet.h>
  33 #include <linux/gfp.h>
  34 #include <net/ip.h>
  35 #include <net/protocol.h>
  36 #include <net/netlink.h>
  37 #include <linux/skbuff.h>
  38 #include <net/sock.h>
  39 #include <linux/errno.h>
  40 #include <linux/timer.h>
  41 #include <asm/uaccess.h>
  42 #include <asm/unaligned.h>
  43 #include <linux/filter.h>
  44 #include <linux/ratelimit.h>
  45 #include <linux/seccomp.h>
  46 #include <linux/if_vlan.h>
  47
  48 /* No hurry in this branch
  49  *
  50  * Exported for the bpf jit load helper.
  51  */
  52 void *bpf_internal_load_pointer_neg_helper(const struct sk_buff *skb, int k, unsigned int size)
  53 {
  54         u8 *ptr = NULL;
  55
  56         if (k >= SKF_NET_OFF)
  57                 ptr = skb_network_header(skb) + k - SKF_NET_OFF;
  58         else if (k >= SKF_LL_OFF)
  59                 ptr = skb_mac_header(skb) + k - SKF_LL_OFF;
  60
  61         if (ptr >= skb->head && ptr + size <= skb_tail_pointer(skb))
  62                 return ptr;
  63         return NULL;
  64 }
  65
  66 static inline void *load_pointer(const struct sk_buff *skb, int k,
  67                                  unsigned int size, void *buffer)
  68 {
  69         if (k >= 0)
  70                 return skb_header_pointer(skb, k, size, buffer);
  71         return bpf_internal_load_pointer_neg_helper(skb, k, size);
  72 }
  73
  74 /**
  75  *      sk_filter - run a packet through a socket filter
  76  *      @sk: sock associated with &sk_buff
  77  *      @skb: buffer to filter
  78  *
  79  * Run the filter code and then cut skb->data to correct size returned by
  80  * sk_run_filter. If pkt_len is 0 we toss packet. If skb->len is smaller
  81  * than pkt_len we keep whole skb->data. This is the socket level
  82  * wrapper to sk_run_filter. It returns 0 if the packet should
  83  * be accepted or -EPERM if the packet should be tossed.
  84  *
  85  */
  86 int sk_filter(struct sock *sk, struct sk_buff *skb)
  87 {
  88         int err;
  89         struct sk_filter *filter;
  90
  91         /*
  92          * If the skb was allocated from pfmemalloc reserves, only
  93          * allow SOCK_MEMALLOC sockets to use it as this socket is
  94          * helping free memory
  95          */
  96         if (skb_pfmemalloc(skb) && !sock_flag(sk, SOCK_MEMALLOC))
  97                 return -ENOMEM;
  98
  99         err = security_sock_rcv_skb(sk, skb);
 100         if (err)
 101                 return err;
 102
 103         rcu_read_lock();
 104         filter = rcu_dereference(sk->sk_filter);
 105         if (filter) {
 106                 unsigned int pkt_len = SK_RUN_FILTER(filter, skb);
 107
 108                 err = pkt_len ? pskb_trim(skb, pkt_len) : -EPERM;
 109         }
 110         rcu_read_unlock();
 111
 112         return err;
 113 }
 114 EXPORT_SYMBOL(sk_filter);
 115
 116 /* Base function for offset calculation. Needs to go into .text section,
 117  * therefore keeping it non-static as well; will also be used by JITs
 118  * anyway later on, so do not let the compiler omit it.
 119  */
 120 noinline u64 __bpf_call_base(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5)
 121 {
 122         return 0;
 123 }
 124
 125 /**
 126  *      __sk_run_filter - run a filter on a given context
 127  *      @ctx: buffer to run the filter on
 128  *      @insn: filter to apply
 129  *
 130  * Decode and apply filter instructions to the skb->data. Return length to
 131  * keep, 0 for none. @ctx is the data we are operating on, @insn is the
 132  * array of filter instructions.
 133  */
 134 unsigned int __sk_run_filter(void *ctx, const struct sock_filter_int *insn)
 135 {
 136         u64 stack[MAX_BPF_STACK / sizeof(u64)];
 137         u64 regs[MAX_BPF_REG], tmp;
 138         void *ptr;
 139         int off;
 140
 141 #define K  insn->imm
 142 #define A  regs[insn->a_reg]
 143 #define X  regs[insn->x_reg]
 144 #define R0 regs[0]
 145
 146 #define CONT     ({insn++; goto select_insn; })
 147 #define CONT_JMP ({insn++; goto select_insn; })
 148
 149         static const void *jumptable[256] = {
 150                 [0 ... 255] = &&default_label,
 151                 /* Now overwrite non-defaults ... */
 152 #define DL(A, B, C)     [A|B|C] = &&A##_##B##_##C
 153                 DL(BPF_ALU, BPF_ADD, BPF_X),
 154                 DL(BPF_ALU, BPF_ADD, BPF_K),
 155                 DL(BPF_ALU, BPF_SUB, BPF_X),
 156                 DL(BPF_ALU, BPF_SUB, BPF_K),
 157                 DL(BPF_ALU, BPF_AND, BPF_X),
 158                 DL(BPF_ALU, BPF_AND, BPF_K),
 159                 DL(BPF_ALU, BPF_OR, BPF_X),
 160                 DL(BPF_ALU, BPF_OR, BPF_K),
 161                 DL(BPF_ALU, BPF_LSH, BPF_X),
 162                 DL(BPF_ALU, BPF_LSH, BPF_K),
 163                 DL(BPF_ALU, BPF_RSH, BPF_X),
 164                 DL(BPF_ALU, BPF_RSH, BPF_K),
 165                 DL(BPF_ALU, BPF_XOR, BPF_X),
 166                 DL(BPF_ALU, BPF_XOR, BPF_K),
 167                 DL(BPF_ALU, BPF_MUL, BPF_X),
 168                 DL(BPF_ALU, BPF_MUL, BPF_K),
 169                 DL(BPF_ALU, BPF_MOV, BPF_X),
 170                 DL(BPF_ALU, BPF_MOV, BPF_K),
 171                 DL(BPF_ALU, BPF_DIV, BPF_X),
 172                 DL(BPF_ALU, BPF_DIV, BPF_K),
 173                 DL(BPF_ALU, BPF_MOD, BPF_X),
 174                 DL(BPF_ALU, BPF_MOD, BPF_K),
 175                 DL(BPF_ALU, BPF_NEG, 0),
 176                 DL(BPF_ALU, BPF_END, BPF_TO_BE),
 177                 DL(BPF_ALU, BPF_END, BPF_TO_LE),
 178                 DL(BPF_ALU64, BPF_ADD, BPF_X),
 179                 DL(BPF_ALU64, BPF_ADD, BPF_K),
 180                 DL(BPF_ALU64, BPF_SUB, BPF_X),
 181                 DL(BPF_ALU64, BPF_SUB, BPF_K),
 182                 DL(BPF_ALU64, BPF_AND, BPF_X),
 183                 DL(BPF_ALU64, BPF_AND, BPF_K),
 184                 DL(BPF_ALU64, BPF_OR, BPF_X),
 185                 DL(BPF_ALU64, BPF_OR, BPF_K),
 186                 DL(BPF_ALU64, BPF_LSH, BPF_X),
 187                 DL(BPF_ALU64, BPF_LSH, BPF_K),
 188                 DL(BPF_ALU64, BPF_RSH, BPF_X),
 189                 DL(BPF_ALU64, BPF_RSH, BPF_K),
 190                 DL(BPF_ALU64, BPF_XOR, BPF_X),
 191                 DL(BPF_ALU64, BPF_XOR, BPF_K),
 192                 DL(BPF_ALU64, BPF_MUL, BPF_X),
 193                 DL(BPF_ALU64, BPF_MUL, BPF_K),
 194                 DL(BPF_ALU64, BPF_MOV, BPF_X),
 195                 DL(BPF_ALU64, BPF_MOV, BPF_K),
 196                 DL(BPF_ALU64, BPF_ARSH, BPF_X),
 197                 DL(BPF_ALU64, BPF_ARSH, BPF_K),
 198                 DL(BPF_ALU64, BPF_DIV, BPF_X),
 199                 DL(BPF_ALU64, BPF_DIV, BPF_K),
 200                 DL(BPF_ALU64, BPF_MOD, BPF_X),
 201                 DL(BPF_ALU64, BPF_MOD, BPF_K),
 202                 DL(BPF_ALU64, BPF_NEG, 0),
 203                 DL(BPF_JMP, BPF_CALL, 0),
 204                 DL(BPF_JMP, BPF_JA, 0),
 205                 DL(BPF_JMP, BPF_JEQ, BPF_X),
 206                 DL(BPF_JMP, BPF_JEQ, BPF_K),
 207                 DL(BPF_JMP, BPF_JNE, BPF_X),
 208                 DL(BPF_JMP, BPF_JNE, BPF_K),
 209                 DL(BPF_JMP, BPF_JGT, BPF_X),
 210                 DL(BPF_JMP, BPF_JGT, BPF_K),
 211                 DL(BPF_JMP, BPF_JGE, BPF_X),
 212                 DL(BPF_JMP, BPF_JGE, BPF_K),
 213                 DL(BPF_JMP, BPF_JSGT, BPF_X),
 214                 DL(BPF_JMP, BPF_JSGT, BPF_K),
 215                 DL(BPF_JMP, BPF_JSGE, BPF_X),
 216                 DL(BPF_JMP, BPF_JSGE, BPF_K),
 217                 DL(BPF_JMP, BPF_JSET, BPF_X),
 218                 DL(BPF_JMP, BPF_JSET, BPF_K),
 219                 DL(BPF_JMP, BPF_EXIT, 0),
 220                 DL(BPF_STX, BPF_MEM, BPF_B),
 221                 DL(BPF_STX, BPF_MEM, BPF_H),
 222                 DL(BPF_STX, BPF_MEM, BPF_W),
 223                 DL(BPF_STX, BPF_MEM, BPF_DW),
 224                 DL(BPF_STX, BPF_XADD, BPF_W),
 225                 DL(BPF_STX, BPF_XADD, BPF_DW),
 226                 DL(BPF_ST, BPF_MEM, BPF_B),
 227                 DL(BPF_ST, BPF_MEM, BPF_H),
 228                 DL(BPF_ST, BPF_MEM, BPF_W),
 229                 DL(BPF_ST, BPF_MEM, BPF_DW),
 230                 DL(BPF_LDX, BPF_MEM, BPF_B),
 231                 DL(BPF_LDX, BPF_MEM, BPF_H),
 232                 DL(BPF_LDX, BPF_MEM, BPF_W),
 233                 DL(BPF_LDX, BPF_MEM, BPF_DW),
 234                 DL(BPF_LD, BPF_ABS, BPF_W),
 235                 DL(BPF_LD, BPF_ABS, BPF_H),
 236                 DL(BPF_LD, BPF_ABS, BPF_B),
 237                 DL(BPF_LD, BPF_IND, BPF_W),
 238                 DL(BPF_LD, BPF_IND, BPF_H),
 239                 DL(BPF_LD, BPF_IND, BPF_B),
 240 #undef DL
 241         };
 242
 243         regs[FP_REG]  = (u64) (unsigned long) &stack[ARRAY_SIZE(stack)];
 244         regs[ARG1_REG] = (u64) (unsigned long) ctx;
 245
 246 select_insn:
 247         goto *jumptable[insn->code];
 248
 249         /* ALU */
 250 #define ALU(OPCODE, OP)                 \
 251         BPF_ALU64_##OPCODE##_BPF_X:     \
 252                 A = A OP X;             \
 253                 CONT;                   \
 254         BPF_ALU_##OPCODE##_BPF_X:       \
 255                 A = (u32) A OP (u32) X; \
 256                 CONT;                   \
 257         BPF_ALU64_##OPCODE##_BPF_K:     \
 258                 A = A OP K;             \
 259                 CONT;                   \
 260         BPF_ALU_##OPCODE##_BPF_K:       \
 261                 A = (u32) A OP (u32) K; \
 262                 CONT;
 263
 264         ALU(BPF_ADD,  +)
 265         ALU(BPF_SUB,  -)
 266         ALU(BPF_AND,  &)
 267         ALU(BPF_OR,   |)
 268         ALU(BPF_LSH, <<)
 269         ALU(BPF_RSH, >>)
 270         ALU(BPF_XOR,  ^)
 271         ALU(BPF_MUL,  *)
 272 #undef ALU
 273         BPF_ALU_BPF_NEG_0:
 274                 A = (u32) -A;
 275                 CONT;
 276         BPF_ALU64_BPF_NEG_0:
 277                 A = -A;
 278                 CONT;
 279         BPF_ALU_BPF_MOV_BPF_X:
 280                 A = (u32) X;
 281                 CONT;
 282         BPF_ALU_BPF_MOV_BPF_K:
 283                 A = (u32) K;
 284                 CONT;
 285         BPF_ALU64_BPF_MOV_BPF_X:
 286                 A = X;
 287                 CONT;
 288         BPF_ALU64_BPF_MOV_BPF_K:
 289                 A = K;
 290                 CONT;
 291         BPF_ALU64_BPF_ARSH_BPF_X:
 292                 (*(s64 *) &A) >>= X;
 293                 CONT;
 294         BPF_ALU64_BPF_ARSH_BPF_K:
 295                 (*(s64 *) &A) >>= K;
 296                 CONT;
 297         BPF_ALU64_BPF_MOD_BPF_X:
 298                 if (unlikely(X == 0))
 299                         return 0;
 300                 tmp = A;
 301                 A = do_div(tmp, X);
 302                 CONT;
 303         BPF_ALU_BPF_MOD_BPF_X:
 304                 if (unlikely(X == 0))
 305                         return 0;
 306                 tmp = (u32) A;
 307                 A = do_div(tmp, (u32) X);
 308                 CONT;
 309         BPF_ALU64_BPF_MOD_BPF_K:
 310                 tmp = A;
 311                 A = do_div(tmp, K);
 312                 CONT;
 313         BPF_ALU_BPF_MOD_BPF_K:
 314                 tmp = (u32) A;
 315                 A = do_div(tmp, (u32) K);
 316                 CONT;
 317         BPF_ALU64_BPF_DIV_BPF_X:
 318                 if (unlikely(X == 0))
 319                         return 0;
 320                 do_div(A, X);
 321                 CONT;
 322         BPF_ALU_BPF_DIV_BPF_X:
 323                 if (unlikely(X == 0))
 324                         return 0;
 325                 tmp = (u32) A;
 326                 do_div(tmp, (u32) X);
 327                 A = (u32) tmp;
 328                 CONT;
 329         BPF_ALU64_BPF_DIV_BPF_K:
 330                 do_div(A, K);
 331                 CONT;
 332         BPF_ALU_BPF_DIV_BPF_K:
 333                 tmp = (u32) A;
 334                 do_div(tmp, (u32) K);
 335                 A = (u32) tmp;
 336                 CONT;
 337         BPF_ALU_BPF_END_BPF_TO_BE:
 338                 switch (K) {
 339                 case 16:
 340                         A = (__force u16) cpu_to_be16(A);
 341                         break;
 342                 case 32:
 343                         A = (__force u32) cpu_to_be32(A);
 344                         break;
 345                 case 64:
 346                         A = (__force u64) cpu_to_be64(A);
 347                         break;
 348                 }
 349                 CONT;
 350         BPF_ALU_BPF_END_BPF_TO_LE:
 351                 switch (K) {
 352                 case 16:
 353                         A = (__force u16) cpu_to_le16(A);
 354                         break;
 355                 case 32:
 356                         A = (__force u32) cpu_to_le32(A);
 357                         break;
 358                 case 64:
 359                         A = (__force u64) cpu_to_le64(A);
 360                         break;
 361                 }
 362                 CONT;
 363
 364         /* CALL */
 365         BPF_JMP_BPF_CALL_0:
 366                 /* Function call scratches R1-R5 registers, preserves R6-R9,
 367                  * and stores return value into R0.
 368                  */
 369                 R0 = (__bpf_call_base + insn->imm)(regs[1], regs[2], regs[3],
 370                                                    regs[4], regs[5]);
 371                 CONT;
 372
 373         /* JMP */
 374         BPF_JMP_BPF_JA_0:
 375                 insn += insn->off;
 376                 CONT;
 377         BPF_JMP_BPF_JEQ_BPF_X:
 378                 if (A == X) {
 379                         insn += insn->off;
 380                         CONT_JMP;
 381                 }
 382                 CONT;
 383         BPF_JMP_BPF_JEQ_BPF_K:
 384                 if (A == K) {
 385                         insn += insn->off;
 386                         CONT_JMP;
 387                 }
 388                 CONT;
 389         BPF_JMP_BPF_JNE_BPF_X:
 390                 if (A != X) {
 391                         insn += insn->off;
 392                         CONT_JMP;
 393                 }
 394                 CONT;
 395         BPF_JMP_BPF_JNE_BPF_K:
 396                 if (A != K) {
 397                         insn += insn->off;
 398                         CONT_JMP;
 399                 }
 400                 CONT;
 401         BPF_JMP_BPF_JGT_BPF_X:
 402                 if (A > X) {
 403                         insn += insn->off;
 404                         CONT_JMP;
 405                 }
 406                 CONT;
 407         BPF_JMP_BPF_JGT_BPF_K:
 408                 if (A > K) {
 409                         insn += insn->off;
 410                         CONT_JMP;
 411                 }
 412                 CONT;
 413         BPF_JMP_BPF_JGE_BPF_X:
 414                 if (A >= X) {
 415                         insn += insn->off;
 416                         CONT_JMP;
 417                 }
 418                 CONT;
 419         BPF_JMP_BPF_JGE_BPF_K:
 420                 if (A >= K) {
 421                         insn += insn->off;
 422                         CONT_JMP;
 423                 }
 424                 CONT;
 425         BPF_JMP_BPF_JSGT_BPF_X:
 426                 if (((s64)A) > ((s64)X)) {
 427                         insn += insn->off;
 428                         CONT_JMP;
 429                 }
 430                 CONT;
 431         BPF_JMP_BPF_JSGT_BPF_K:
 432                 if (((s64)A) > ((s64)K)) {
 433                         insn += insn->off;
 434                         CONT_JMP;
 435                 }
 436                 CONT;
 437         BPF_JMP_BPF_JSGE_BPF_X:
 438                 if (((s64)A) >= ((s64)X)) {
 439                         insn += insn->off;
 440                         CONT_JMP;
 441                 }
 442                 CONT;
 443         BPF_JMP_BPF_JSGE_BPF_K:
 444                 if (((s64)A) >= ((s64)K)) {
 445                         insn += insn->off;
 446                         CONT_JMP;
 447                 }
 448                 CONT;
 449         BPF_JMP_BPF_JSET_BPF_X:
 450                 if (A & X) {
 451                         insn += insn->off;
 452                         CONT_JMP;
 453                 }
 454                 CONT;
 455         BPF_JMP_BPF_JSET_BPF_K:
 456                 if (A & K) {
 457                         insn += insn->off;
 458                         CONT_JMP;
 459                 }
 460                 CONT;
 461         BPF_JMP_BPF_EXIT_0:
 462                 return R0;
 463
 464         /* STX and ST and LDX*/
 465 #define LDST(SIZEOP, SIZE)                                      \
 466         BPF_STX_BPF_MEM_##SIZEOP:                               \
 467                 *(SIZE *)(unsigned long) (A + insn->off) = X;   \
 468                 CONT;                                           \
 469         BPF_ST_BPF_MEM_##SIZEOP:                                \
 470                 *(SIZE *)(unsigned long) (A + insn->off) = K;   \
 471                 CONT;                                           \
 472         BPF_LDX_BPF_MEM_##SIZEOP:                               \
 473                 A = *(SIZE *)(unsigned long) (X + insn->off);   \
 474                 CONT;
 475
 476         LDST(BPF_B,   u8)
 477         LDST(BPF_H,  u16)
 478         LDST(BPF_W,  u32)
 479         LDST(BPF_DW, u64)
 480 #undef LDST
 481         BPF_STX_BPF_XADD_BPF_W: /* lock xadd *(u32 *)(A + insn->off) += X */
 482                 atomic_add((u32) X, (atomic_t *)(unsigned long)
 483                            (A + insn->off));
 484                 CONT;
 485         BPF_STX_BPF_XADD_BPF_DW: /* lock xadd *(u64 *)(A + insn->off) += X */
 486                 atomic64_add((u64) X, (atomic64_t *)(unsigned long)
 487                              (A + insn->off));
 488                 CONT;
 489         BPF_LD_BPF_ABS_BPF_W: /* R0 = ntohl(*(u32 *) (skb->data + K)) */
 490                 off = K;
 491 load_word:
 492                 /* BPF_LD + BPD_ABS and BPF_LD + BPF_IND insns are only
 493                  * appearing in the programs where ctx == skb. All programs
 494                  * keep 'ctx' in regs[CTX_REG] == R6, sk_convert_filter()
 495                  * saves it in R6, internal BPF verifier will check that
 496                  * R6 == ctx.
 497                  *
 498                  * BPF_ABS and BPF_IND are wrappers of function calls, so
 499                  * they scratch R1-R5 registers, preserve R6-R9, and store
 500                  * return value into R0.
 501                  *
 502                  * Implicit input:
 503                  *   ctx
 504                  *
 505                  * Explicit input:
 506                  *   X == any register
 507                  *   K == 32-bit immediate
 508                  *
 509                  * Output:
 510                  *   R0 - 8/16/32-bit skb data converted to cpu endianness
 511                  */
 512                 ptr = load_pointer((struct sk_buff *) ctx, off, 4, &tmp);
 513                 if (likely(ptr != NULL)) {
 514                         R0 = get_unaligned_be32(ptr);
 515                         CONT;
 516                 }
 517                 return 0;
 518         BPF_LD_BPF_ABS_BPF_H: /* R0 = ntohs(*(u16 *) (skb->data + K)) */
 519                 off = K;
 520 load_half:
 521                 ptr = load_pointer((struct sk_buff *) ctx, off, 2, &tmp);
 522                 if (likely(ptr != NULL)) {
 523                         R0 = get_unaligned_be16(ptr);
 524                         CONT;
 525                 }
 526                 return 0;
 527         BPF_LD_BPF_ABS_BPF_B: /* R0 = *(u8 *) (ctx + K) */
 528                 off = K;
 529 load_byte:
 530                 ptr = load_pointer((struct sk_buff *) ctx, off, 1, &tmp);
 531                 if (likely(ptr != NULL)) {
 532                         R0 = *(u8 *)ptr;
 533                         CONT;
 534                 }
 535                 return 0;
 536         BPF_LD_BPF_IND_BPF_W: /* R0 = ntohl(*(u32 *) (skb->data + X + K)) */
 537                 off = K + X;
 538                 goto load_word;
 539         BPF_LD_BPF_IND_BPF_H: /* R0 = ntohs(*(u16 *) (skb->data + X + K)) */
 540                 off = K + X;
 541                 goto load_half;
 542         BPF_LD_BPF_IND_BPF_B: /* R0 = *(u8 *) (skb->data + X + K) */
 543                 off = K + X;
 544                 goto load_byte;
 545
 546         default_label:
 547                 /* If we ever reach this, we have a bug somewhere. */
 548                 WARN_RATELIMIT(1, "unknown opcode %02x\n", insn->code);
 549                 return 0;
 550 #undef CONT_JMP
 551 #undef CONT
 552
 553 #undef R0
 554 #undef X
 555 #undef A
 556 #undef K
 557 }
 558
 559 u32 sk_run_filter_int_seccomp(const struct seccomp_data *ctx,
 560                               const struct sock_filter_int *insni)
 561     __attribute__ ((alias ("__sk_run_filter")));
 562
 563 u32 sk_run_filter_int_skb(const struct sk_buff *ctx,
 564                           const struct sock_filter_int *insni)
 565     __attribute__ ((alias ("__sk_run_filter")));
 566 EXPORT_SYMBOL_GPL(sk_run_filter_int_skb);
 567
 568 /* Helper to find the offset of pkt_type in sk_buff structure. We want
 569  * to make sure its still a 3bit field starting at a byte boundary;
 570  * taken from arch/x86/net/bpf_jit_comp.c.
 571  */
 572 #define PKT_TYPE_MAX    7
 573 static unsigned int pkt_type_offset(void)
 574 {
 575         struct sk_buff skb_probe = { .pkt_type = ~0, };
 576         u8 *ct = (u8 *) &skb_probe;
 577         unsigned int off;
 578
 579         for (off = 0; off < sizeof(struct sk_buff); off++) {
 580                 if (ct[off] == PKT_TYPE_MAX)
 581                         return off;
 582         }
 583
 584         pr_err_once("Please fix %s, as pkt_type couldn't be found!\n", __func__);
 585         return -1;
 586 }
 587
 588 static u64 __skb_get_pay_offset(u64 ctx, u64 A, u64 X, u64 r4, u64 r5)
 589 {
 590         struct sk_buff *skb = (struct sk_buff *)(long) ctx;
 591
 592         return __skb_get_poff(skb);
 593 }
 594
 595 static u64 __skb_get_nlattr(u64 ctx, u64 A, u64 X, u64 r4, u64 r5)
 596 {
 597         struct sk_buff *skb = (struct sk_buff *)(long) ctx;
 598         struct nlattr *nla;
 599
 600         if (skb_is_nonlinear(skb))
 601                 return 0;
 602
 603         if (skb->len < sizeof(struct nlattr))
 604                 return 0;
 605
 606         if (A > skb->len - sizeof(struct nlattr))
 607                 return 0;
 608
 609         nla = nla_find((struct nlattr *) &skb->data[A], skb->len - A, X);
 610         if (nla)
 611                 return (void *) nla - (void *) skb->data;
 612
 613         return 0;
 614 }
 615
 616 static u64 __skb_get_nlattr_nest(u64 ctx, u64 A, u64 X, u64 r4, u64 r5)
 617 {
 618         struct sk_buff *skb = (struct sk_buff *)(long) ctx;
 619         struct nlattr *nla;
 620
 621         if (skb_is_nonlinear(skb))
 622                 return 0;
 623
 624         if (skb->len < sizeof(struct nlattr))
 625                 return 0;
 626
 627         if (A > skb->len - sizeof(struct nlattr))
 628                 return 0;
 629
 630         nla = (struct nlattr *) &skb->data[A];
 631         if (nla->nla_len > skb->len - A)
 632                 return 0;
 633
 634         nla = nla_find_nested(nla, X);
 635         if (nla)
 636                 return (void *) nla - (void *) skb->data;
 637
 638         return 0;
 639 }
 640
 641 static u64 __get_raw_cpu_id(u64 ctx, u64 A, u64 X, u64 r4, u64 r5)
 642 {
 643         return raw_smp_processor_id();
 644 }
 645
 646 /* Register mappings for user programs. */
 647 #define A_REG           0
 648 #define X_REG           7
 649 #define TMP_REG         8
 650 #define ARG2_REG        2
 651 #define ARG3_REG        3
 652
 653 static bool convert_bpf_extensions(struct sock_filter *fp,
 654                                    struct sock_filter_int **insnp)
 655 {
 656         struct sock_filter_int *insn = *insnp;
 657
 658         switch (fp->k) {
 659         case SKF_AD_OFF + SKF_AD_PROTOCOL:
 660                 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, protocol) != 2);
 661
 662                 insn->code = BPF_LDX | BPF_MEM | BPF_H;
 663                 insn->a_reg = A_REG;
 664                 insn->x_reg = CTX_REG;
 665                 insn->off = offsetof(struct sk_buff, protocol);
 666                 insn++;
 667
 668                 /* A = ntohs(A) [emitting a nop or swap16] */
 669                 insn->code = BPF_ALU | BPF_END | BPF_FROM_BE;
 670                 insn->a_reg = A_REG;
 671                 insn->imm = 16;
 672                 break;
 673
 674         case SKF_AD_OFF + SKF_AD_PKTTYPE:
 675                 insn->code = BPF_LDX | BPF_MEM | BPF_B;
 676                 insn->a_reg = A_REG;
 677                 insn->x_reg = CTX_REG;
 678                 insn->off = pkt_type_offset();
 679                 if (insn->off < 0)
 680                         return false;
 681                 insn++;
 682
 683                 insn->code = BPF_ALU | BPF_AND | BPF_K;
 684                 insn->a_reg = A_REG;
 685                 insn->imm = PKT_TYPE_MAX;
 686                 break;
 687
 688         case SKF_AD_OFF + SKF_AD_IFINDEX:
 689         case SKF_AD_OFF + SKF_AD_HATYPE:
 690                 if (FIELD_SIZEOF(struct sk_buff, dev) == 8)
 691                         insn->code = BPF_LDX | BPF_MEM | BPF_DW;
 692                 else
 693                         insn->code = BPF_LDX | BPF_MEM | BPF_W;
 694                 insn->a_reg = TMP_REG;
 695                 insn->x_reg = CTX_REG;
 696                 insn->off = offsetof(struct sk_buff, dev);
 697                 insn++;
 698
 699                 insn->code = BPF_JMP | BPF_JNE | BPF_K;
 700                 insn->a_reg = TMP_REG;
 701                 insn->imm = 0;
 702                 insn->off = 1;
 703                 insn++;
 704
 705                 insn->code = BPF_JMP | BPF_EXIT;
 706                 insn++;
 707
 708                 BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, ifindex) != 4);
 709                 BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, type) != 2);
 710
 711                 insn->a_reg = A_REG;
 712                 insn->x_reg = TMP_REG;
 713
 714                 if (fp->k == SKF_AD_OFF + SKF_AD_IFINDEX) {
 715                         insn->code = BPF_LDX | BPF_MEM | BPF_W;
 716                         insn->off = offsetof(struct net_device, ifindex);
 717                 } else {
 718                         insn->code = BPF_LDX | BPF_MEM | BPF_H;
 719                         insn->off = offsetof(struct net_device, type);
 720                 }
 721                 break;
 722
 723         case SKF_AD_OFF + SKF_AD_MARK:
 724                 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, mark) != 4);
 725
 726                 insn->code = BPF_LDX | BPF_MEM | BPF_W;
 727                 insn->a_reg = A_REG;
 728                 insn->x_reg = CTX_REG;
 729                 insn->off = offsetof(struct sk_buff, mark);
 730                 break;
 731
 732         case SKF_AD_OFF + SKF_AD_RXHASH:
 733                 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, hash) != 4);
 734
 735                 insn->code = BPF_LDX | BPF_MEM | BPF_W;
 736                 insn->a_reg = A_REG;
 737                 insn->x_reg = CTX_REG;
 738                 insn->off = offsetof(struct sk_buff, hash);
 739                 break;
 740
 741         case SKF_AD_OFF + SKF_AD_QUEUE:
 742                 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, queue_mapping) != 2);
 743
 744                 insn->code = BPF_LDX | BPF_MEM | BPF_H;
 745                 insn->a_reg = A_REG;
 746                 insn->x_reg = CTX_REG;
 747                 insn->off = offsetof(struct sk_buff, queue_mapping);
 748                 break;
 749
 750         case SKF_AD_OFF + SKF_AD_VLAN_TAG:
 751         case SKF_AD_OFF + SKF_AD_VLAN_TAG_PRESENT:
 752                 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, vlan_tci) != 2);
 753
 754                 insn->code = BPF_LDX | BPF_MEM | BPF_H;
 755                 insn->a_reg = A_REG;
 756                 insn->x_reg = CTX_REG;
 757                 insn->off = offsetof(struct sk_buff, vlan_tci);
 758                 insn++;
 759
 760                 BUILD_BUG_ON(VLAN_TAG_PRESENT != 0x1000);
 761
 762                 if (fp->k == SKF_AD_OFF + SKF_AD_VLAN_TAG) {
 763                         insn->code = BPF_ALU | BPF_AND | BPF_K;
 764                         insn->a_reg = A_REG;
 765                         insn->imm = ~VLAN_TAG_PRESENT;
 766                 } else {
 767                         insn->code = BPF_ALU | BPF_RSH | BPF_K;
 768                         insn->a_reg = A_REG;
 769                         insn->imm = 12;
 770                         insn++;
 771
 772                         insn->code = BPF_ALU | BPF_AND | BPF_K;
 773                         insn->a_reg = A_REG;
 774                         insn->imm = 1;
 775                 }
 776                 break;
 777
 778         case SKF_AD_OFF + SKF_AD_PAY_OFFSET:
 779         case SKF_AD_OFF + SKF_AD_NLATTR:
 780         case SKF_AD_OFF + SKF_AD_NLATTR_NEST:
 781         case SKF_AD_OFF + SKF_AD_CPU:
 782                 /* arg1 = ctx */
 783                 insn->code = BPF_ALU64 | BPF_MOV | BPF_X;
 784                 insn->a_reg = ARG1_REG;
 785                 insn->x_reg = CTX_REG;
 786                 insn++;
 787
 788                 /* arg2 = A */
 789                 insn->code = BPF_ALU64 | BPF_MOV | BPF_X;
 790                 insn->a_reg = ARG2_REG;
 791                 insn->x_reg = A_REG;
 792                 insn++;
 793
 794                 /* arg3 = X */
 795                 insn->code = BPF_ALU64 | BPF_MOV | BPF_X;
 796                 insn->a_reg = ARG3_REG;
 797                 insn->x_reg = X_REG;
 798                 insn++;
 799
 800                 /* Emit call(ctx, arg2=A, arg3=X) */
 801                 insn->code = BPF_JMP | BPF_CALL;
 802                 switch (fp->k) {
 803                 case SKF_AD_OFF + SKF_AD_PAY_OFFSET:
 804                         insn->imm = __skb_get_pay_offset - __bpf_call_base;
 805                         break;
 806                 case SKF_AD_OFF + SKF_AD_NLATTR:
 807                         insn->imm = __skb_get_nlattr - __bpf_call_base;
 808                         break;
 809                 case SKF_AD_OFF + SKF_AD_NLATTR_NEST:
 810                         insn->imm = __skb_get_nlattr_nest - __bpf_call_base;
 811                         break;
 812                 case SKF_AD_OFF + SKF_AD_CPU:
 813                         insn->imm = __get_raw_cpu_id - __bpf_call_base;
 814                         break;
 815                 }
 816                 break;
 817
 818         case SKF_AD_OFF + SKF_AD_ALU_XOR_X:
 819                 insn->code = BPF_ALU | BPF_XOR | BPF_X;
 820                 insn->a_reg = A_REG;
 821                 insn->x_reg = X_REG;
 822                 break;
 823
 824         default:
 825                 /* This is just a dummy call to avoid letting the compiler
 826                  * evict __bpf_call_base() as an optimization. Placed here
 827                  * where no-one bothers.
 828                  */
 829                 BUG_ON(__bpf_call_base(0, 0, 0, 0, 0) != 0);
 830                 return false;
 831         }
 832
 833         *insnp = insn;
 834         return true;
 835 }
 836
 837 /**
 838  *      sk_convert_filter - convert filter program
 839  *      @prog: the user passed filter program
 840  *      @len: the length of the user passed filter program
 841  *      @new_prog: buffer where converted program will be stored
 842  *      @new_len: pointer to store length of converted program
 843  *
 844  * Remap 'sock_filter' style BPF instruction set to 'sock_filter_ext' style.
 845  * Conversion workflow:
 846  *
 847  * 1) First pass for calculating the new program length:
 848  *   sk_convert_filter(old_prog, old_len, NULL, &new_len)
 849  *
 850  * 2) 2nd pass to remap in two passes: 1st pass finds new
 851  *    jump offsets, 2nd pass remapping:
 852  *   new_prog = kmalloc(sizeof(struct sock_filter_int) * new_len);
 853  *   sk_convert_filter(old_prog, old_len, new_prog, &new_len);
 854  *
 855  * User BPF's register A is mapped to our BPF register 6, user BPF
 856  * register X is mapped to BPF register 7; frame pointer is always
 857  * register 10; Context 'void *ctx' is stored in register 1, that is,
 858  * for socket filters: ctx == 'struct sk_buff *', for seccomp:
 859  * ctx == 'struct seccomp_data *'.
 860  */
 861 int sk_convert_filter(struct sock_filter *prog, int len,
 862                       struct sock_filter_int *new_prog, int *new_len)
 863 {
 864         int new_flen = 0, pass = 0, target, i;
 865         struct sock_filter_int *new_insn;
 866         struct sock_filter *fp;
 867         int *addrs = NULL;
 868         u8 bpf_src;
 869
 870         BUILD_BUG_ON(BPF_MEMWORDS * sizeof(u32) > MAX_BPF_STACK);
 871         BUILD_BUG_ON(FP_REG + 1 != MAX_BPF_REG);
 872
 873         if (len <= 0 || len >= BPF_MAXINSNS)
 874                 return -EINVAL;
 875
 876         if (new_prog) {
 877                 addrs = kzalloc(len * sizeof(*addrs), GFP_KERNEL);
 878                 if (!addrs)
 879                         return -ENOMEM;
 880         }
 881
 882 do_pass:
 883         new_insn = new_prog;
 884         fp = prog;
 885
 886         if (new_insn) {
 887                 new_insn->code = BPF_ALU64 | BPF_MOV | BPF_X;
 888                 new_insn->a_reg = CTX_REG;
 889                 new_insn->x_reg = ARG1_REG;
 890         }
 891         new_insn++;
 892
 893         for (i = 0; i < len; fp++, i++) {
 894                 struct sock_filter_int tmp_insns[6] = { };
 895                 struct sock_filter_int *insn = tmp_insns;
 896
 897                 if (addrs)
 898                         addrs[i] = new_insn - new_prog;
 899
 900                 switch (fp->code) {
 901                 /* All arithmetic insns and skb loads map as-is. */
 902                 case BPF_ALU | BPF_ADD | BPF_X:
 903                 case BPF_ALU | BPF_ADD | BPF_K:
 904                 case BPF_ALU | BPF_SUB | BPF_X:
 905                 case BPF_ALU | BPF_SUB | BPF_K:
 906                 case BPF_ALU | BPF_AND | BPF_X:
 907                 case BPF_ALU | BPF_AND | BPF_K:
 908                 case BPF_ALU | BPF_OR | BPF_X:
 909                 case BPF_ALU | BPF_OR | BPF_K:
 910                 case BPF_ALU | BPF_LSH | BPF_X:
 911                 case BPF_ALU | BPF_LSH | BPF_K:
 912                 case BPF_ALU | BPF_RSH | BPF_X:
 913                 case BPF_ALU | BPF_RSH | BPF_K:
 914                 case BPF_ALU | BPF_XOR | BPF_X:
 915                 case BPF_ALU | BPF_XOR | BPF_K:
 916                 case BPF_ALU | BPF_MUL | BPF_X:
 917                 case BPF_ALU | BPF_MUL | BPF_K:
 918                 case BPF_ALU | BPF_DIV | BPF_X:
 919                 case BPF_ALU | BPF_DIV | BPF_K:
 920                 case BPF_ALU | BPF_MOD | BPF_X:
 921                 case BPF_ALU | BPF_MOD | BPF_K:
 922                 case BPF_ALU | BPF_NEG:
 923                 case BPF_LD | BPF_ABS | BPF_W:
 924                 case BPF_LD | BPF_ABS | BPF_H:
 925                 case BPF_LD | BPF_ABS | BPF_B:
 926                 case BPF_LD | BPF_IND | BPF_W:
 927                 case BPF_LD | BPF_IND | BPF_H:
 928                 case BPF_LD | BPF_IND | BPF_B:
 929                         /* Check for overloaded BPF extension and
 930                          * directly convert it if found, otherwise
 931                          * just move on with mapping.
 932                          */
 933                         if (BPF_CLASS(fp->code) == BPF_LD &&
 934                             BPF_MODE(fp->code) == BPF_ABS &&
 935                             convert_bpf_extensions(fp, &insn))
 936                                 break;
 937
 938                         insn->code = fp->code;
 939                         insn->a_reg = A_REG;
 940                         insn->x_reg = X_REG;
 941                         insn->imm = fp->k;
 942                         break;
 943
 944                 /* Jump opcodes map as-is, but offsets need adjustment. */
 945                 case BPF_JMP | BPF_JA:
 946                         target = i + fp->k + 1;
 947                         insn->code = fp->code;
 948 #define EMIT_JMP                                                        \
 949         do {                                                            \
 950                 if (target >= len || target < 0)                        \
 951                         goto err;                                       \
 952                 insn->off = addrs ? addrs[target] - addrs[i] - 1 : 0;   \
 953                 /* Adjust pc relative offset for 2nd or 3rd insn. */    \
 954                 insn->off -= insn - tmp_insns;                          \
 955         } while (0)
 956
 957                         EMIT_JMP;
 958                         break;
 959
 960                 case BPF_JMP | BPF_JEQ | BPF_K:
 961                 case BPF_JMP | BPF_JEQ | BPF_X:
 962                 case BPF_JMP | BPF_JSET | BPF_K:
 963                 case BPF_JMP | BPF_JSET | BPF_X:
 964                 case BPF_JMP | BPF_JGT | BPF_K:
 965                 case BPF_JMP | BPF_JGT | BPF_X:
 966                 case BPF_JMP | BPF_JGE | BPF_K:
 967                 case BPF_JMP | BPF_JGE | BPF_X:
 968                         if (BPF_SRC(fp->code) == BPF_K && (int) fp->k < 0) {
 969                                 /* BPF immediates are signed, zero extend
 970                                  * immediate into tmp register and use it
 971                                  * in compare insn.
 972                                  */
 973                                 insn->code = BPF_ALU | BPF_MOV | BPF_K;
 974                                 insn->a_reg = TMP_REG;
 975                                 insn->imm = fp->k;
 976                                 insn++;
 977
 978                                 insn->a_reg = A_REG;
 979                                 insn->x_reg = TMP_REG;
 980                                 bpf_src = BPF_X;
 981                         } else {
 982                                 insn->a_reg = A_REG;
 983                                 insn->x_reg = X_REG;
 984                                 insn->imm = fp->k;
 985                                 bpf_src = BPF_SRC(fp->code);
 986                         }
 987
 988                         /* Common case where 'jump_false' is next insn. */
 989                         if (fp->jf == 0) {
 990                                 insn->code = BPF_JMP | BPF_OP(fp->code) | bpf_src;
 991                                 target = i + fp->jt + 1;
 992                                 EMIT_JMP;
 993                                 break;
 994                         }
 995
 996                         /* Convert JEQ into JNE when 'jump_true' is next insn. */
 997                         if (fp->jt == 0 && BPF_OP(fp->code) == BPF_JEQ) {
 998                                 insn->code = BPF_JMP | BPF_JNE | bpf_src;
 999                                 target = i + fp->jf + 1;
1000                                 EMIT_JMP;
1001                                 break;
1002                         }
1003
1004                         /* Other jumps are mapped into two insns: Jxx and JA. */
1005                         target = i + fp->jt + 1;
1006                         insn->code = BPF_JMP | BPF_OP(fp->code) | bpf_src;
1007                         EMIT_JMP;
1008                         insn++;
1009
1010                         insn->code = BPF_JMP | BPF_JA;
1011                         target = i + fp->jf + 1;
1012                         EMIT_JMP;
1013                         break;
1014
1015                 /* ldxb 4 * ([14] & 0xf) is remaped into 6 insns. */
1016                 case BPF_LDX | BPF_MSH | BPF_B:
1017                         insn->code = BPF_ALU64 | BPF_MOV | BPF_X;
1018                         insn->a_reg = TMP_REG;
1019                         insn->x_reg = A_REG;
1020                         insn++;
1021
1022                         insn->code = BPF_LD | BPF_ABS | BPF_B;
1023                         insn->a_reg = A_REG;
1024                         insn->imm = fp->k;
1025                         insn++;
1026
1027                         insn->code = BPF_ALU | BPF_AND | BPF_K;
1028                         insn->a_reg = A_REG;
1029                         insn->imm = 0xf;
1030                         insn++;
1031
1032                         insn->code = BPF_ALU | BPF_LSH | BPF_K;
1033                         insn->a_reg = A_REG;
1034                         insn->imm = 2;
1035                         insn++;
1036
1037                         insn->code = BPF_ALU64 | BPF_MOV | BPF_X;
1038                         insn->a_reg = X_REG;
1039                         insn->x_reg = A_REG;
1040                         insn++;
1041
1042                         insn->code = BPF_ALU64 | BPF_MOV | BPF_X;
1043                         insn->a_reg = A_REG;
1044                         insn->x_reg = TMP_REG;
1045                         break;
1046
1047                 /* RET_K, RET_A are remaped into 2 insns. */
1048                 case BPF_RET | BPF_A:
1049                 case BPF_RET | BPF_K:
1050                         insn->code = BPF_ALU | BPF_MOV |
1051                                      (BPF_RVAL(fp->code) == BPF_K ?
1052                                       BPF_K : BPF_X);
1053                         insn->a_reg = 0;
1054                         insn->x_reg = A_REG;
1055                         insn->imm = fp->k;
1056                         insn++;
1057
1058                         insn->code = BPF_JMP | BPF_EXIT;
1059                         break;
1060
1061                 /* Store to stack. */
1062                 case BPF_ST:
1063                 case BPF_STX:
1064                         insn->code = BPF_STX | BPF_MEM | BPF_W;
1065                         insn->a_reg = FP_REG;
1066                         insn->x_reg = fp->code == BPF_ST ? A_REG : X_REG;
1067                         insn->off = -(BPF_MEMWORDS - fp->k) * 4;
1068                         break;
1069
1070                 /* Load from stack. */
1071                 case BPF_LD | BPF_MEM:
1072                 case BPF_LDX | BPF_MEM:
1073                         insn->code = BPF_LDX | BPF_MEM | BPF_W;
1074                         insn->a_reg = BPF_CLASS(fp->code) == BPF_LD ?
1075                                       A_REG : X_REG;
1076                         insn->x_reg = FP_REG;
1077                         insn->off = -(BPF_MEMWORDS - fp->k) * 4;
1078                         break;
1079
1080                 /* A = K or X = K */
1081                 case BPF_LD | BPF_IMM:
1082                 case BPF_LDX | BPF_IMM:
1083                         insn->code = BPF_ALU | BPF_MOV | BPF_K;
1084                         insn->a_reg = BPF_CLASS(fp->code) == BPF_LD ?
1085                                       A_REG : X_REG;
1086                         insn->imm = fp->k;
1087                         break;
1088
1089                 /* X = A */
1090                 case BPF_MISC | BPF_TAX:
1091                         insn->code = BPF_ALU64 | BPF_MOV | BPF_X;
1092                         insn->a_reg = X_REG;
1093                         insn->x_reg = A_REG;
1094                         break;
1095
1096                 /* A = X */
1097                 case BPF_MISC | BPF_TXA:
1098                         insn->code = BPF_ALU64 | BPF_MOV | BPF_X;
1099                         insn->a_reg = A_REG;
1100                         insn->x_reg = X_REG;
1101                         break;
1102
1103                 /* A = skb->len or X = skb->len */
1104                 case BPF_LD | BPF_W | BPF_LEN:
1105                 case BPF_LDX | BPF_W | BPF_LEN:
1106                         insn->code = BPF_LDX | BPF_MEM | BPF_W;
1107                         insn->a_reg = BPF_CLASS(fp->code) == BPF_LD ?
1108                                       A_REG : X_REG;
1109                         insn->x_reg = CTX_REG;
1110                         insn->off = offsetof(struct sk_buff, len);
1111                         break;
1112
1113                 /* access seccomp_data fields */
1114                 case BPF_LDX | BPF_ABS | BPF_W:
1115                         insn->code = BPF_LDX | BPF_MEM | BPF_W;
1116                         insn->a_reg = A_REG;
1117                         insn->x_reg = CTX_REG;
1118                         insn->off = fp->k;
1119                         break;
1120
1121                 default:
1122                         goto err;
1123                 }
1124
1125                 insn++;
1126                 if (new_prog)
1127                         memcpy(new_insn, tmp_insns,
1128                                sizeof(*insn) * (insn - tmp_insns));
1129
1130                 new_insn += insn - tmp_insns;
1131         }
1132
1133         if (!new_prog) {
1134                 /* Only calculating new length. */
1135                 *new_len = new_insn - new_prog;
1136                 return 0;
1137         }
1138
1139         pass++;
1140         if (new_flen != new_insn - new_prog) {
1141                 new_flen = new_insn - new_prog;
1142                 if (pass > 2)
1143                         goto err;
1144
1145                 goto do_pass;
1146         }
1147
1148         kfree(addrs);
1149         BUG_ON(*new_len != new_flen);
1150         return 0;
1151 err:
1152         kfree(addrs);
1153         return -EINVAL;
1154 }
1155
1156 /* Security:
1157  *
1158  * A BPF program is able to use 16 cells of memory to store intermediate
1159  * values (check u32 mem[BPF_MEMWORDS] in sk_run_filter()).
1160  *
1161  * As we dont want to clear mem[] array for each packet going through
1162  * sk_run_filter(), we check that filter loaded by user never try to read
1163  * a cell if not previously written, and we check all branches to be sure
1164  * a malicious user doesn't try to abuse us.
1165  */
1166 static int check_load_and_stores(struct sock_filter *filter, int flen)
1167 {
1168         u16 *masks, memvalid = 0; /* one bit per cell, 16 cells */
1169         int pc, ret = 0;
1170
1171         BUILD_BUG_ON(BPF_MEMWORDS > 16);
1172         masks = kmalloc(flen * sizeof(*masks), GFP_KERNEL);
1173         if (!masks)
1174                 return -ENOMEM;
1175         memset(masks, 0xff, flen * sizeof(*masks));
1176
1177         for (pc = 0; pc < flen; pc++) {
1178                 memvalid &= masks[pc];
1179
1180                 switch (filter[pc].code) {
1181                 case BPF_S_ST:
1182                 case BPF_S_STX:
1183                         memvalid |= (1 << filter[pc].k);
1184                         break;
1185                 case BPF_S_LD_MEM:
1186                 case BPF_S_LDX_MEM:
1187                         if (!(memvalid & (1 << filter[pc].k))) {
1188                                 ret = -EINVAL;
1189                                 goto error;
1190                         }
1191                         break;
1192                 case BPF_S_JMP_JA:
1193                         /* a jump must set masks on target */
1194                         masks[pc + 1 + filter[pc].k] &= memvalid;
1195                         memvalid = ~0;
1196                         break;
1197                 case BPF_S_JMP_JEQ_K:
1198                 case BPF_S_JMP_JEQ_X:
1199                 case BPF_S_JMP_JGE_K:
1200                 case BPF_S_JMP_JGE_X:
1201                 case BPF_S_JMP_JGT_K:
1202                 case BPF_S_JMP_JGT_X:
1203                 case BPF_S_JMP_JSET_X:
1204                 case BPF_S_JMP_JSET_K:
1205                         /* a jump must set masks on targets */
1206                         masks[pc + 1 + filter[pc].jt] &= memvalid;
1207                         masks[pc + 1 + filter[pc].jf] &= memvalid;
1208                         memvalid = ~0;
1209                         break;
1210                 }
1211         }
1212 error:
1213         kfree(masks);
1214         return ret;
1215 }
1216
1217 /**
1218  *      sk_chk_filter - verify socket filter code
1219  *      @filter: filter to verify
1220  *      @flen: length of filter
1221  *
1222  * Check the user's filter code. If we let some ugly
1223  * filter code slip through kaboom! The filter must contain
1224  * no references or jumps that are out of range, no illegal
1225  * instructions, and must end with a RET instruction.
1226  *
1227  * All jumps are forward as they are not signed.
1228  *
1229  * Returns 0 if the rule set is legal or -EINVAL if not.
1230  */
1231 int sk_chk_filter(struct sock_filter *filter, unsigned int flen)
1232 {
1233         /*
1234          * Valid instructions are initialized to non-0.
1235          * Invalid instructions are initialized to 0.
1236          */
1237         static const u8 codes[] = {
1238                 [BPF_ALU|BPF_ADD|BPF_K]  = BPF_S_ALU_ADD_K,
1239                 [BPF_ALU|BPF_ADD|BPF_X]  = BPF_S_ALU_ADD_X,
1240                 [BPF_ALU|BPF_SUB|BPF_K]  = BPF_S_ALU_SUB_K,
1241                 [BPF_ALU|BPF_SUB|BPF_X]  = BPF_S_ALU_SUB_X,
1242                 [BPF_ALU|BPF_MUL|BPF_K]  = BPF_S_ALU_MUL_K,
1243                 [BPF_ALU|BPF_MUL|BPF_X]  = BPF_S_ALU_MUL_X,
1244                 [BPF_ALU|BPF_DIV|BPF_X]  = BPF_S_ALU_DIV_X,
1245                 [BPF_ALU|BPF_MOD|BPF_K]  = BPF_S_ALU_MOD_K,
1246                 [BPF_ALU|BPF_MOD|BPF_X]  = BPF_S_ALU_MOD_X,
1247                 [BPF_ALU|BPF_AND|BPF_K]  = BPF_S_ALU_AND_K,
1248                 [BPF_ALU|BPF_AND|BPF_X]  = BPF_S_ALU_AND_X,
1249                 [BPF_ALU|BPF_OR|BPF_K]   = BPF_S_ALU_OR_K,
1250                 [BPF_ALU|BPF_OR|BPF_X]   = BPF_S_ALU_OR_X,
1251                 [BPF_ALU|BPF_XOR|BPF_K]  = BPF_S_ALU_XOR_K,
1252                 [BPF_ALU|BPF_XOR|BPF_X]  = BPF_S_ALU_XOR_X,
1253                 [BPF_ALU|BPF_LSH|BPF_K]  = BPF_S_ALU_LSH_K,
1254                 [BPF_ALU|BPF_LSH|BPF_X]  = BPF_S_ALU_LSH_X,
1255                 [BPF_ALU|BPF_RSH|BPF_K]  = BPF_S_ALU_RSH_K,
1256                 [BPF_ALU|BPF_RSH|BPF_X]  = BPF_S_ALU_RSH_X,
1257                 [BPF_ALU|BPF_NEG]        = BPF_S_ALU_NEG,
1258                 [BPF_LD|BPF_W|BPF_ABS]   = BPF_S_LD_W_ABS,
1259                 [BPF_LD|BPF_H|BPF_ABS]   = BPF_S_LD_H_ABS,
1260                 [BPF_LD|BPF_B|BPF_ABS]   = BPF_S_LD_B_ABS,
1261                 [BPF_LD|BPF_W|BPF_LEN]   = BPF_S_LD_W_LEN,
1262                 [BPF_LD|BPF_W|BPF_IND]   = BPF_S_LD_W_IND,
1263                 [BPF_LD|BPF_H|BPF_IND]   = BPF_S_LD_H_IND,
1264                 [BPF_LD|BPF_B|BPF_IND]   = BPF_S_LD_B_IND,
1265                 [BPF_LD|BPF_IMM]         = BPF_S_LD_IMM,
1266                 [BPF_LDX|BPF_W|BPF_LEN]  = BPF_S_LDX_W_LEN,
1267                 [BPF_LDX|BPF_B|BPF_MSH]  = BPF_S_LDX_B_MSH,
1268                 [BPF_LDX|BPF_IMM]        = BPF_S_LDX_IMM,
1269                 [BPF_MISC|BPF_TAX]       = BPF_S_MISC_TAX,
1270                 [BPF_MISC|BPF_TXA]       = BPF_S_MISC_TXA,
1271                 [BPF_RET|BPF_K]          = BPF_S_RET_K,
1272                 [BPF_RET|BPF_A]          = BPF_S_RET_A,
1273                 [BPF_ALU|BPF_DIV|BPF_K]  = BPF_S_ALU_DIV_K,
1274                 [BPF_LD|BPF_MEM]         = BPF_S_LD_MEM,
1275                 [BPF_LDX|BPF_MEM]        = BPF_S_LDX_MEM,
1276                 [BPF_ST]                 = BPF_S_ST,
1277                 [BPF_STX]                = BPF_S_STX,
1278                 [BPF_JMP|BPF_JA]         = BPF_S_JMP_JA,
1279                 [BPF_JMP|BPF_JEQ|BPF_K]  = BPF_S_JMP_JEQ_K,
1280                 [BPF_JMP|BPF_JEQ|BPF_X]  = BPF_S_JMP_JEQ_X,
1281                 [BPF_JMP|BPF_JGE|BPF_K]  = BPF_S_JMP_JGE_K,
1282                 [BPF_JMP|BPF_JGE|BPF_X]  = BPF_S_JMP_JGE_X,
1283                 [BPF_JMP|BPF_JGT|BPF_K]  = BPF_S_JMP_JGT_K,
1284                 [BPF_JMP|BPF_JGT|BPF_X]  = BPF_S_JMP_JGT_X,
1285                 [BPF_JMP|BPF_JSET|BPF_K] = BPF_S_JMP_JSET_K,
1286                 [BPF_JMP|BPF_JSET|BPF_X] = BPF_S_JMP_JSET_X,
1287         };
1288         int pc;
1289         bool anc_found;
1290
1291         if (flen == 0 || flen > BPF_MAXINSNS)
1292                 return -EINVAL;
1293
1294         /* check the filter code now */
1295         for (pc = 0; pc < flen; pc++) {
1296                 struct sock_filter *ftest = &filter[pc];
1297                 u16 code = ftest->code;
1298
1299                 if (code >= ARRAY_SIZE(codes))
1300                         return -EINVAL;
1301                 code = codes[code];
1302                 if (!code)
1303                         return -EINVAL;
1304                 /* Some instructions need special checks */
1305                 switch (code) {
1306                 case BPF_S_ALU_DIV_K:
1307                 case BPF_S_ALU_MOD_K:
1308                         /* check for division by zero */
1309                         if (ftest->k == 0)
1310                                 return -EINVAL;
1311                         break;
1312                 case BPF_S_LD_MEM:
1313                 case BPF_S_LDX_MEM:
1314                 case BPF_S_ST:
1315                 case BPF_S_STX:
1316                         /* check for invalid memory addresses */
1317                         if (ftest->k >= BPF_MEMWORDS)
1318                                 return -EINVAL;
1319                         break;
1320                 case BPF_S_JMP_JA:
1321                         /*
1322                          * Note, the large ftest->k might cause loops.
1323                          * Compare this with conditional jumps below,
1324                          * where offsets are limited. --ANK (981016)
1325                          */
1326                         if (ftest->k >= (unsigned int)(flen-pc-1))
1327                                 return -EINVAL;
1328                         break;
1329                 case BPF_S_JMP_JEQ_K:
1330                 case BPF_S_JMP_JEQ_X:
1331                 case BPF_S_JMP_JGE_K:
1332                 case BPF_S_JMP_JGE_X:
1333                 case BPF_S_JMP_JGT_K:
1334                 case BPF_S_JMP_JGT_X:
1335                 case BPF_S_JMP_JSET_X:
1336                 case BPF_S_JMP_JSET_K:
1337                         /* for conditionals both must be safe */
1338                         if (pc + ftest->jt + 1 >= flen ||
1339                             pc + ftest->jf + 1 >= flen)
1340                                 return -EINVAL;
1341                         break;
1342                 case BPF_S_LD_W_ABS:
1343                 case BPF_S_LD_H_ABS:
1344                 case BPF_S_LD_B_ABS:
1345                         anc_found = false;
1346 #define ANCILLARY(CODE) case SKF_AD_OFF + SKF_AD_##CODE:        \
1347                                 code = BPF_S_ANC_##CODE;        \
1348                                 anc_found = true;               \
1349                                 break
1350                         switch (ftest->k) {
1351                         ANCILLARY(PROTOCOL);
1352                         ANCILLARY(PKTTYPE);
1353                         ANCILLARY(IFINDEX);
1354                         ANCILLARY(NLATTR);
1355                         ANCILLARY(NLATTR_NEST);
1356                         ANCILLARY(MARK);
1357                         ANCILLARY(QUEUE);
1358                         ANCILLARY(HATYPE);
1359                         ANCILLARY(RXHASH);
1360                         ANCILLARY(CPU);
1361                         ANCILLARY(ALU_XOR_X);
1362                         ANCILLARY(VLAN_TAG);
1363                         ANCILLARY(VLAN_TAG_PRESENT);
1364                         ANCILLARY(PAY_OFFSET);
1365                         }
1366
1367                         /* ancillary operation unknown or unsupported */
1368                         if (anc_found == false && ftest->k >= SKF_AD_OFF)
1369                                 return -EINVAL;
1370                 }
1371                 ftest->code = code;
1372         }
1373
1374         /* last instruction must be a RET code */
1375         switch (filter[flen - 1].code) {
1376         case BPF_S_RET_K:
1377         case BPF_S_RET_A:
1378                 return check_load_and_stores(filter, flen);
1379         }
1380         return -EINVAL;
1381 }
1382 EXPORT_SYMBOL(sk_chk_filter);
1383
1384 static int sk_store_orig_filter(struct sk_filter *fp,
1385                                 const struct sock_fprog *fprog)
1386 {
1387         unsigned int fsize = sk_filter_proglen(fprog);
1388         struct sock_fprog_kern *fkprog;
1389
1390         fp->orig_prog = kmalloc(sizeof(*fkprog), GFP_KERNEL);
1391         if (!fp->orig_prog)
1392                 return -ENOMEM;
1393
1394         fkprog = fp->orig_prog;
1395         fkprog->len = fprog->len;
1396         fkprog->filter = kmemdup(fp->insns, fsize, GFP_KERNEL);
1397         if (!fkprog->filter) {
1398                 kfree(fp->orig_prog);
1399                 return -ENOMEM;
1400         }
1401
1402         return 0;
1403 }
1404
1405 static void sk_release_orig_filter(struct sk_filter *fp)
1406 {
1407         struct sock_fprog_kern *fprog = fp->orig_prog;
1408
1409         if (fprog) {
1410                 kfree(fprog->filter);
1411                 kfree(fprog);
1412         }
1413 }
1414
1415 /**
1416  *      sk_filter_release_rcu - Release a socket filter by rcu_head
1417  *      @rcu: rcu_head that contains the sk_filter to free
1418  */
1419 static void sk_filter_release_rcu(struct rcu_head *rcu)
1420 {
1421         struct sk_filter *fp = container_of(rcu, struct sk_filter, rcu);
1422
1423         sk_release_orig_filter(fp);
1424         bpf_jit_free(fp);
1425 }
1426
1427 /**
1428  *      sk_filter_release - release a socket filter
1429  *      @fp: filter to remove
1430  *
1431  *      Remove a filter from a socket and release its resources.
1432  */
1433 static void sk_filter_release(struct sk_filter *fp)
1434 {
1435         if (atomic_dec_and_test(&fp->refcnt))
1436                 call_rcu(&fp->rcu, sk_filter_release_rcu);
1437 }
1438
1439 void sk_filter_uncharge(struct sock *sk, struct sk_filter *fp)
1440 {
1441         atomic_sub(sk_filter_size(fp->len), &sk->sk_omem_alloc);
1442         sk_filter_release(fp);
1443 }
1444
1445 void sk_filter_charge(struct sock *sk, struct sk_filter *fp)
1446 {
1447         atomic_inc(&fp->refcnt);
1448         atomic_add(sk_filter_size(fp->len), &sk->sk_omem_alloc);
1449 }
1450
1451 static struct sk_filter *__sk_migrate_realloc(struct sk_filter *fp,
1452                                               struct sock *sk,
1453                                               unsigned int len)
1454 {
1455         struct sk_filter *fp_new;
1456
1457         if (sk == NULL)
1458                 return krealloc(fp, len, GFP_KERNEL);
1459
1460         fp_new = sock_kmalloc(sk, len, GFP_KERNEL);
1461         if (fp_new) {
1462                 memcpy(fp_new, fp, sizeof(struct sk_filter));
1463                 /* As we're kepping orig_prog in fp_new along,
1464                  * we need to make sure we're not evicting it
1465                  * from the old fp.
1466                  */
1467                 fp->orig_prog = NULL;
1468                 sk_filter_uncharge(sk, fp);
1469         }
1470
1471         return fp_new;
1472 }
1473
1474 static struct sk_filter *__sk_migrate_filter(struct sk_filter *fp,
1475                                              struct sock *sk)
1476 {
1477         struct sock_filter *old_prog;
1478         struct sk_filter *old_fp;
1479         int i, err, new_len, old_len = fp->len;
1480
1481         /* We are free to overwrite insns et al right here as it
1482          * won't be used at this point in time anymore internally
1483          * after the migration to the internal BPF instruction
1484          * representation.
1485          */
1486         BUILD_BUG_ON(sizeof(struct sock_filter) !=
1487                      sizeof(struct sock_filter_int));
1488
1489         /* For now, we need to unfiddle BPF_S_* identifiers in place.
1490          * This can sooner or later on be subject to removal, e.g. when
1491          * JITs have been converted.
1492          */
1493         for (i = 0; i < fp->len; i++)
1494                 sk_decode_filter(&fp->insns[i], &fp->insns[i]);
1495
1496         /* Conversion cannot happen on overlapping memory areas,
1497          * so we need to keep the user BPF around until the 2nd
1498          * pass. At this time, the user BPF is stored in fp->insns.
1499          */
1500         old_prog = kmemdup(fp->insns, old_len * sizeof(struct sock_filter),
1501                            GFP_KERNEL);
1502         if (!old_prog) {
1503                 err = -ENOMEM;
1504                 goto out_err;
1505         }
1506
1507         /* 1st pass: calculate the new program length. */
1508         err = sk_convert_filter(old_prog, old_len, NULL, &new_len);
1509         if (err)
1510                 goto out_err_free;
1511
1512         /* Expand fp for appending the new filter representation. */
1513         old_fp = fp;
1514         fp = __sk_migrate_realloc(old_fp, sk, sk_filter_size(new_len));
1515         if (!fp) {
1516                 /* The old_fp is still around in case we couldn't
1517                  * allocate new memory, so uncharge on that one.
1518                  */
1519                 fp = old_fp;
1520                 err = -ENOMEM;
1521                 goto out_err_free;
1522         }
1523
1524         fp->bpf_func = sk_run_filter_int_skb;
1525         fp->len = new_len;
1526
1527         /* 2nd pass: remap sock_filter insns into sock_filter_int insns. */
1528         err = sk_convert_filter(old_prog, old_len, fp->insnsi, &new_len);
1529         if (err)
1530                 /* 2nd sk_convert_filter() can fail only if it fails
1531                  * to allocate memory, remapping must succeed. Note,
1532                  * that at this time old_fp has already been released
1533                  * by __sk_migrate_realloc().
1534                  */
1535                 goto out_err_free;
1536
1537         kfree(old_prog);
1538         return fp;
1539
1540 out_err_free:
1541         kfree(old_prog);
1542 out_err:
1543         /* Rollback filter setup. */
1544         if (sk != NULL)
1545                 sk_filter_uncharge(sk, fp);
1546         else
1547                 kfree(fp);
1548         return ERR_PTR(err);
1549 }
1550
1551 static struct sk_filter *__sk_prepare_filter(struct sk_filter *fp,
1552                                              struct sock *sk)
1553 {
1554         int err;
1555
1556         fp->bpf_func = NULL;
1557         fp->jited = 0;
1558
1559         err = sk_chk_filter(fp->insns, fp->len);
1560         if (err)
1561                 return ERR_PTR(err);
1562
1563         /* Probe if we can JIT compile the filter and if so, do
1564          * the compilation of the filter.
1565          */
1566         bpf_jit_compile(fp);
1567
1568         /* JIT compiler couldn't process this filter, so do the
1569          * internal BPF translation for the optimized interpreter.
1570          */
1571         if (!fp->jited)
1572                 fp = __sk_migrate_filter(fp, sk);
1573
1574         return fp;
1575 }
1576
1577 /**
1578  *      sk_unattached_filter_create - create an unattached filter
1579  *      @fprog: the filter program
1580  *      @pfp: the unattached filter that is created
1581  *
1582  * Create a filter independent of any socket. We first run some
1583  * sanity checks on it to make sure it does not explode on us later.
1584  * If an error occurs or there is insufficient memory for the filter
1585  * a negative errno code is returned. On success the return is zero.
1586  */
1587 int sk_unattached_filter_create(struct sk_filter **pfp,
1588                                 struct sock_fprog *fprog)
1589 {
1590         unsigned int fsize = sk_filter_proglen(fprog);
1591         struct sk_filter *fp;
1592
1593         /* Make sure new filter is there and in the right amounts. */
1594         if (fprog->filter == NULL)
1595                 return -EINVAL;
1596
1597         fp = kmalloc(sk_filter_size(fprog->len), GFP_KERNEL);
1598         if (!fp)
1599                 return -ENOMEM;
1600
1601         memcpy(fp->insns, fprog->filter, fsize);
1602
1603         atomic_set(&fp->refcnt, 1);
1604         fp->len = fprog->len;
1605         /* Since unattached filters are not copied back to user
1606          * space through sk_get_filter(), we do not need to hold
1607          * a copy here, and can spare us the work.
1608          */
1609         fp->orig_prog = NULL;
1610
1611         /* __sk_prepare_filter() already takes care of uncharging
1612          * memory in case something goes wrong.
1613          */
1614         fp = __sk_prepare_filter(fp, NULL);
1615         if (IS_ERR(fp))
1616                 return PTR_ERR(fp);
1617
1618         *pfp = fp;
1619         return 0;
1620 }
1621 EXPORT_SYMBOL_GPL(sk_unattached_filter_create);
1622
1623 void sk_unattached_filter_destroy(struct sk_filter *fp)
1624 {
1625         sk_filter_release(fp);
1626 }
1627 EXPORT_SYMBOL_GPL(sk_unattached_filter_destroy);
1628
1629 /**
1630  *      sk_attach_filter - attach a socket filter
1631  *      @fprog: the filter program
1632  *      @sk: the socket to use
1633  *
1634  * Attach the user's filter code. We first run some sanity checks on
1635  * it to make sure it does not explode on us later. If an error
1636  * occurs or there is insufficient memory for the filter a negative
1637  * errno code is returned. On success the return is zero.
1638  */
1639 int sk_attach_filter(struct sock_fprog *fprog, struct sock *sk)
1640 {
1641         struct sk_filter *fp, *old_fp;
1642         unsigned int fsize = sk_filter_proglen(fprog);
1643         unsigned int sk_fsize = sk_filter_size(fprog->len);
1644         int err;
1645
1646         if (sock_flag(sk, SOCK_FILTER_LOCKED))
1647                 return -EPERM;
1648
1649         /* Make sure new filter is there and in the right amounts. */
1650         if (fprog->filter == NULL)
1651                 return -EINVAL;
1652
1653         fp = sock_kmalloc(sk, sk_fsize, GFP_KERNEL);
1654         if (!fp)
1655                 return -ENOMEM;
1656
1657         if (copy_from_user(fp->insns, fprog->filter, fsize)) {
1658                 sock_kfree_s(sk, fp, sk_fsize);
1659                 return -EFAULT;
1660         }
1661
1662         atomic_set(&fp->refcnt, 1);
1663         fp->len = fprog->len;
1664
1665         err = sk_store_orig_filter(fp, fprog);
1666         if (err) {
1667                 sk_filter_uncharge(sk, fp);
1668                 return -ENOMEM;
1669         }
1670
1671         /* __sk_prepare_filter() already takes care of uncharging
1672          * memory in case something goes wrong.
1673          */
1674         fp = __sk_prepare_filter(fp, sk);
1675         if (IS_ERR(fp))
1676                 return PTR_ERR(fp);
1677
1678         old_fp = rcu_dereference_protected(sk->sk_filter,
1679                                            sock_owned_by_user(sk));
1680         rcu_assign_pointer(sk->sk_filter, fp);
1681
1682         if (old_fp)
1683                 sk_filter_uncharge(sk, old_fp);
1684
1685         return 0;
1686 }
1687 EXPORT_SYMBOL_GPL(sk_attach_filter);
1688
1689 int sk_detach_filter(struct sock *sk)
1690 {
1691         int ret = -ENOENT;
1692         struct sk_filter *filter;
1693
1694         if (sock_flag(sk, SOCK_FILTER_LOCKED))
1695                 return -EPERM;
1696
1697         filter = rcu_dereference_protected(sk->sk_filter,
1698                                            sock_owned_by_user(sk));
1699         if (filter) {
1700                 RCU_INIT_POINTER(sk->sk_filter, NULL);
1701                 sk_filter_uncharge(sk, filter);
1702                 ret = 0;
1703         }
1704
1705         return ret;
1706 }
1707 EXPORT_SYMBOL_GPL(sk_detach_filter);
1708
1709 void sk_decode_filter(struct sock_filter *filt, struct sock_filter *to)
1710 {
1711         static const u16 decodes[] = {
1712                 [BPF_S_ALU_ADD_K]       = BPF_ALU|BPF_ADD|BPF_K,
1713                 [BPF_S_ALU_ADD_X]       = BPF_ALU|BPF_ADD|BPF_X,
1714                 [BPF_S_ALU_SUB_K]       = BPF_ALU|BPF_SUB|BPF_K,
1715                 [BPF_S_ALU_SUB_X]       = BPF_ALU|BPF_SUB|BPF_X,
1716                 [BPF_S_ALU_MUL_K]       = BPF_ALU|BPF_MUL|BPF_K,
1717                 [BPF_S_ALU_MUL_X]       = BPF_ALU|BPF_MUL|BPF_X,
1718                 [BPF_S_ALU_DIV_X]       = BPF_ALU|BPF_DIV|BPF_X,
1719                 [BPF_S_ALU_MOD_K]       = BPF_ALU|BPF_MOD|BPF_K,
1720                 [BPF_S_ALU_MOD_X]       = BPF_ALU|BPF_MOD|BPF_X,
1721                 [BPF_S_ALU_AND_K]       = BPF_ALU|BPF_AND|BPF_K,
1722                 [BPF_S_ALU_AND_X]       = BPF_ALU|BPF_AND|BPF_X,
1723                 [BPF_S_ALU_OR_K]        = BPF_ALU|BPF_OR|BPF_K,
1724                 [BPF_S_ALU_OR_X]        = BPF_ALU|BPF_OR|BPF_X,
1725                 [BPF_S_ALU_XOR_K]       = BPF_ALU|BPF_XOR|BPF_K,
1726                 [BPF_S_ALU_XOR_X]       = BPF_ALU|BPF_XOR|BPF_X,
1727                 [BPF_S_ALU_LSH_K]       = BPF_ALU|BPF_LSH|BPF_K,
1728                 [BPF_S_ALU_LSH_X]       = BPF_ALU|BPF_LSH|BPF_X,
1729                 [BPF_S_ALU_RSH_K]       = BPF_ALU|BPF_RSH|BPF_K,
1730                 [BPF_S_ALU_RSH_X]       = BPF_ALU|BPF_RSH|BPF_X,
1731                 [BPF_S_ALU_NEG]         = BPF_ALU|BPF_NEG,
1732                 [BPF_S_LD_W_ABS]        = BPF_LD|BPF_W|BPF_ABS,
1733                 [BPF_S_LD_H_ABS]        = BPF_LD|BPF_H|BPF_ABS,
1734                 [BPF_S_LD_B_ABS]        = BPF_LD|BPF_B|BPF_ABS,
1735                 [BPF_S_ANC_PROTOCOL]    = BPF_LD|BPF_B|BPF_ABS,
1736                 [BPF_S_ANC_PKTTYPE]     = BPF_LD|BPF_B|BPF_ABS,
1737                 [BPF_S_ANC_IFINDEX]     = BPF_LD|BPF_B|BPF_ABS,
1738                 [BPF_S_ANC_NLATTR]      = BPF_LD|BPF_B|BPF_ABS,
1739                 [BPF_S_ANC_NLATTR_NEST] = BPF_LD|BPF_B|BPF_ABS,
1740                 [BPF_S_ANC_MARK]        = BPF_LD|BPF_B|BPF_ABS,
1741                 [BPF_S_ANC_QUEUE]       = BPF_LD|BPF_B|BPF_ABS,
1742                 [BPF_S_ANC_HATYPE]      = BPF_LD|BPF_B|BPF_ABS,
1743                 [BPF_S_ANC_RXHASH]      = BPF_LD|BPF_B|BPF_ABS,
1744                 [BPF_S_ANC_CPU]         = BPF_LD|BPF_B|BPF_ABS,
1745                 [BPF_S_ANC_ALU_XOR_X]   = BPF_LD|BPF_B|BPF_ABS,
1746                 [BPF_S_ANC_VLAN_TAG]    = BPF_LD|BPF_B|BPF_ABS,
1747                 [BPF_S_ANC_VLAN_TAG_PRESENT] = BPF_LD|BPF_B|BPF_ABS,
1748                 [BPF_S_ANC_PAY_OFFSET]  = BPF_LD|BPF_B|BPF_ABS,
1749                 [BPF_S_LD_W_LEN]        = BPF_LD|BPF_W|BPF_LEN,
1750                 [BPF_S_LD_W_IND]        = BPF_LD|BPF_W|BPF_IND,
1751                 [BPF_S_LD_H_IND]        = BPF_LD|BPF_H|BPF_IND,
1752                 [BPF_S_LD_B_IND]        = BPF_LD|BPF_B|BPF_IND,
1753                 [BPF_S_LD_IMM]          = BPF_LD|BPF_IMM,
1754                 [BPF_S_LDX_W_LEN]       = BPF_LDX|BPF_W|BPF_LEN,
1755                 [BPF_S_LDX_B_MSH]       = BPF_LDX|BPF_B|BPF_MSH,
1756                 [BPF_S_LDX_IMM]         = BPF_LDX|BPF_IMM,
1757                 [BPF_S_MISC_TAX]        = BPF_MISC|BPF_TAX,
1758                 [BPF_S_MISC_TXA]        = BPF_MISC|BPF_TXA,
1759                 [BPF_S_RET_K]           = BPF_RET|BPF_K,
1760                 [BPF_S_RET_A]           = BPF_RET|BPF_A,
1761                 [BPF_S_ALU_DIV_K]       = BPF_ALU|BPF_DIV|BPF_K,
1762                 [BPF_S_LD_MEM]          = BPF_LD|BPF_MEM,
1763                 [BPF_S_LDX_MEM]         = BPF_LDX|BPF_MEM,
1764                 [BPF_S_ST]              = BPF_ST,
1765                 [BPF_S_STX]             = BPF_STX,
1766                 [BPF_S_JMP_JA]          = BPF_JMP|BPF_JA,
1767                 [BPF_S_JMP_JEQ_K]       = BPF_JMP|BPF_JEQ|BPF_K,
1768                 [BPF_S_JMP_JEQ_X]       = BPF_JMP|BPF_JEQ|BPF_X,
1769                 [BPF_S_JMP_JGE_K]       = BPF_JMP|BPF_JGE|BPF_K,
1770                 [BPF_S_JMP_JGE_X]       = BPF_JMP|BPF_JGE|BPF_X,
1771                 [BPF_S_JMP_JGT_K]       = BPF_JMP|BPF_JGT|BPF_K,
1772                 [BPF_S_JMP_JGT_X]       = BPF_JMP|BPF_JGT|BPF_X,
1773                 [BPF_S_JMP_JSET_K]      = BPF_JMP|BPF_JSET|BPF_K,
1774                 [BPF_S_JMP_JSET_X]      = BPF_JMP|BPF_JSET|BPF_X,
1775         };
1776         u16 code;
1777
1778         code = filt->code;
1779
1780         to->code = decodes[code];
1781         to->jt = filt->jt;
1782         to->jf = filt->jf;
1783         to->k = filt->k;
1784 }
1785
1786 int sk_get_filter(struct sock *sk, struct sock_filter __user *ubuf,
1787                   unsigned int len)
1788 {
1789         struct sock_fprog_kern *fprog;
1790         struct sk_filter *filter;
1791         int ret = 0;
1792
1793         lock_sock(sk);
1794         filter = rcu_dereference_protected(sk->sk_filter,
1795                                            sock_owned_by_user(sk));
1796         if (!filter)
1797                 goto out;
1798
1799         /* We're copying the filter that has been originally attached,
1800          * so no conversion/decode needed anymore.
1801          */
1802         fprog = filter->orig_prog;
1803
1804         ret = fprog->len;
1805         if (!len)
1806                 /* User space only enquires number of filter blocks. */
1807                 goto out;
1808
1809         ret = -EINVAL;
1810         if (len < fprog->len)
1811                 goto out;
1812
1813         ret = -EFAULT;
1814         if (copy_to_user(ubuf, fprog->filter, sk_filter_proglen(fprog)))
1815                 goto out;
1816
1817         /* Instead of bytes, the API requests to return the number
1818          * of filter blocks.
1819          */
1820         ret = fprog->len;
1821 out:
1822         release_sock(sk);
1823         return ret;
1824 }