net/core/filter.c

   1 /*
   2  * Linux Socket Filter - Kernel level socket filtering
   3  *
   4  * Based on the design of the Berkeley Packet Filter. The new
   5  * internal format has been designed by PLUMgrid:
   6  *
   7  *      Copyright (c) 2011 - 2014 PLUMgrid, http://plumgrid.com
   8  *
   9  * Authors:
  10  *
  11  *      Jay Schulist <jschlst@samba.org>
  12  *      Alexei Starovoitov <ast@plumgrid.com>
  13  *      Daniel Borkmann <dborkman@redhat.com>
  14  *
  15  * This program is free software; you can redistribute it and/or
  16  * modify it under the terms of the GNU General Public License
  17  * as published by the Free Software Foundation; either version
  18  * 2 of the License, or (at your option) any later version.
  19  *
  20  * Andi Kleen - Fix a few bad bugs and races.
  21  * Kris Katterjohn - Added many additional checks in sk_chk_filter()
  22  */
  23
  24 #include <linux/module.h>
  25 #include <linux/types.h>
  26 #include <linux/mm.h>
  27 #include <linux/fcntl.h>
  28 #include <linux/socket.h>
  29 #include <linux/in.h>
  30 #include <linux/inet.h>
  31 #include <linux/netdevice.h>
  32 #include <linux/if_packet.h>
  33 #include <linux/gfp.h>
  34 #include <net/ip.h>
  35 #include <net/protocol.h>
  36 #include <net/netlink.h>
  37 #include <linux/skbuff.h>
  38 #include <net/sock.h>
  39 #include <linux/errno.h>
  40 #include <linux/timer.h>
  41 #include <asm/uaccess.h>
  42 #include <asm/unaligned.h>
  43 #include <linux/filter.h>
  44 #include <linux/ratelimit.h>
  45 #include <linux/seccomp.h>
  46 #include <linux/if_vlan.h>
  47
  48 /* No hurry in this branch
  49  *
  50  * Exported for the bpf jit load helper.
  51  */
  52 void *bpf_internal_load_pointer_neg_helper(const struct sk_buff *skb, int k, unsigned int size)
  53 {
  54         u8 *ptr = NULL;
  55
  56         if (k >= SKF_NET_OFF)
  57                 ptr = skb_network_header(skb) + k - SKF_NET_OFF;
  58         else if (k >= SKF_LL_OFF)
  59                 ptr = skb_mac_header(skb) + k - SKF_LL_OFF;
  60
  61         if (ptr >= skb->head && ptr + size <= skb_tail_pointer(skb))
  62                 return ptr;
  63         return NULL;
  64 }
  65
  66 static inline void *load_pointer(const struct sk_buff *skb, int k,
  67                                  unsigned int size, void *buffer)
  68 {
  69         if (k >= 0)
  70                 return skb_header_pointer(skb, k, size, buffer);
  71         return bpf_internal_load_pointer_neg_helper(skb, k, size);
  72 }
  73
  74 /**
  75  *      sk_filter - run a packet through a socket filter
  76  *      @sk: sock associated with &sk_buff
  77  *      @skb: buffer to filter
  78  *
  79  * Run the filter code and then cut skb->data to correct size returned by
  80  * sk_run_filter. If pkt_len is 0 we toss packet. If skb->len is smaller
  81  * than pkt_len we keep whole skb->data. This is the socket level
  82  * wrapper to sk_run_filter. It returns 0 if the packet should
  83  * be accepted or -EPERM if the packet should be tossed.
  84  *
  85  */
  86 int sk_filter(struct sock *sk, struct sk_buff *skb)
  87 {
  88         int err;
  89         struct sk_filter *filter;
  90
  91         /*
  92          * If the skb was allocated from pfmemalloc reserves, only
  93          * allow SOCK_MEMALLOC sockets to use it as this socket is
  94          * helping free memory
  95          */
  96         if (skb_pfmemalloc(skb) && !sock_flag(sk, SOCK_MEMALLOC))
  97                 return -ENOMEM;
  98
  99         err = security_sock_rcv_skb(sk, skb);
 100         if (err)
 101                 return err;
 102
 103         rcu_read_lock();
 104         filter = rcu_dereference(sk->sk_filter);
 105         if (filter) {
 106                 unsigned int pkt_len = SK_RUN_FILTER(filter, skb);
 107
 108                 err = pkt_len ? pskb_trim(skb, pkt_len) : -EPERM;
 109         }
 110         rcu_read_unlock();
 111
 112         return err;
 113 }
 114 EXPORT_SYMBOL(sk_filter);
 115
 116 /* Base function for offset calculation. Needs to go into .text section,
 117  * therefore keeping it non-static as well; will also be used by JITs
 118  * anyway later on, so do not let the compiler omit it.
 119  */
 120 noinline u64 __bpf_call_base(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5)
 121 {
 122         return 0;
 123 }
 124
 125 /**
 126  *      __sk_run_filter - run a filter on a given context
 127  *      @ctx: buffer to run the filter on
 128  *      @insn: filter to apply
 129  *
 130  * Decode and apply filter instructions to the skb->data. Return length to
 131  * keep, 0 for none. @ctx is the data we are operating on, @insn is the
 132  * array of filter instructions.
 133  */
 134 unsigned int __sk_run_filter(void *ctx, const struct sock_filter_int *insn)
 135 {
 136         u64 stack[MAX_BPF_STACK / sizeof(u64)];
 137         u64 regs[MAX_BPF_REG], tmp;
 138         void *ptr;
 139         int off;
 140
 141 #define K  insn->imm
 142 #define A  regs[insn->a_reg]
 143 #define X  regs[insn->x_reg]
 144 #define R0 regs[0]
 145
 146 #define CONT     ({insn++; goto select_insn; })
 147 #define CONT_JMP ({insn++; goto select_insn; })
 148
 149         static const void *jumptable[256] = {
 150                 [0 ... 255] = &&default_label,
 151                 /* Now overwrite non-defaults ... */
 152 #define DL(A, B, C)     [A|B|C] = &&A##_##B##_##C
 153                 DL(BPF_ALU, BPF_ADD, BPF_X),
 154                 DL(BPF_ALU, BPF_ADD, BPF_K),
 155                 DL(BPF_ALU, BPF_SUB, BPF_X),
 156                 DL(BPF_ALU, BPF_SUB, BPF_K),
 157                 DL(BPF_ALU, BPF_AND, BPF_X),
 158                 DL(BPF_ALU, BPF_AND, BPF_K),
 159                 DL(BPF_ALU, BPF_OR, BPF_X),
 160                 DL(BPF_ALU, BPF_OR, BPF_K),
 161                 DL(BPF_ALU, BPF_LSH, BPF_X),
 162                 DL(BPF_ALU, BPF_LSH, BPF_K),
 163                 DL(BPF_ALU, BPF_RSH, BPF_X),
 164                 DL(BPF_ALU, BPF_RSH, BPF_K),
 165                 DL(BPF_ALU, BPF_XOR, BPF_X),
 166                 DL(BPF_ALU, BPF_XOR, BPF_K),
 167                 DL(BPF_ALU, BPF_MUL, BPF_X),
 168                 DL(BPF_ALU, BPF_MUL, BPF_K),
 169                 DL(BPF_ALU, BPF_MOV, BPF_X),
 170                 DL(BPF_ALU, BPF_MOV, BPF_K),
 171                 DL(BPF_ALU, BPF_DIV, BPF_X),
 172                 DL(BPF_ALU, BPF_DIV, BPF_K),
 173                 DL(BPF_ALU, BPF_MOD, BPF_X),
 174                 DL(BPF_ALU, BPF_MOD, BPF_K),
 175                 DL(BPF_ALU, BPF_NEG, 0),
 176                 DL(BPF_ALU, BPF_END, BPF_TO_BE),
 177                 DL(BPF_ALU, BPF_END, BPF_TO_LE),
 178                 DL(BPF_ALU64, BPF_ADD, BPF_X),
 179                 DL(BPF_ALU64, BPF_ADD, BPF_K),
 180                 DL(BPF_ALU64, BPF_SUB, BPF_X),
 181                 DL(BPF_ALU64, BPF_SUB, BPF_K),
 182                 DL(BPF_ALU64, BPF_AND, BPF_X),
 183                 DL(BPF_ALU64, BPF_AND, BPF_K),
 184                 DL(BPF_ALU64, BPF_OR, BPF_X),
 185                 DL(BPF_ALU64, BPF_OR, BPF_K),
 186                 DL(BPF_ALU64, BPF_LSH, BPF_X),
 187                 DL(BPF_ALU64, BPF_LSH, BPF_K),
 188                 DL(BPF_ALU64, BPF_RSH, BPF_X),
 189                 DL(BPF_ALU64, BPF_RSH, BPF_K),
 190                 DL(BPF_ALU64, BPF_XOR, BPF_X),
 191                 DL(BPF_ALU64, BPF_XOR, BPF_K),
 192                 DL(BPF_ALU64, BPF_MUL, BPF_X),
 193                 DL(BPF_ALU64, BPF_MUL, BPF_K),
 194                 DL(BPF_ALU64, BPF_MOV, BPF_X),
 195                 DL(BPF_ALU64, BPF_MOV, BPF_K),
 196                 DL(BPF_ALU64, BPF_ARSH, BPF_X),
 197                 DL(BPF_ALU64, BPF_ARSH, BPF_K),
 198                 DL(BPF_ALU64, BPF_DIV, BPF_X),
 199                 DL(BPF_ALU64, BPF_DIV, BPF_K),
 200                 DL(BPF_ALU64, BPF_MOD, BPF_X),
 201                 DL(BPF_ALU64, BPF_MOD, BPF_K),
 202                 DL(BPF_ALU64, BPF_NEG, 0),
 203                 DL(BPF_JMP, BPF_CALL, 0),
 204                 DL(BPF_JMP, BPF_JA, 0),
 205                 DL(BPF_JMP, BPF_JEQ, BPF_X),
 206                 DL(BPF_JMP, BPF_JEQ, BPF_K),
 207                 DL(BPF_JMP, BPF_JNE, BPF_X),
 208                 DL(BPF_JMP, BPF_JNE, BPF_K),
 209                 DL(BPF_JMP, BPF_JGT, BPF_X),
 210                 DL(BPF_JMP, BPF_JGT, BPF_K),
 211                 DL(BPF_JMP, BPF_JGE, BPF_X),
 212                 DL(BPF_JMP, BPF_JGE, BPF_K),
 213                 DL(BPF_JMP, BPF_JSGT, BPF_X),
 214                 DL(BPF_JMP, BPF_JSGT, BPF_K),
 215                 DL(BPF_JMP, BPF_JSGE, BPF_X),
 216                 DL(BPF_JMP, BPF_JSGE, BPF_K),
 217                 DL(BPF_JMP, BPF_JSET, BPF_X),
 218                 DL(BPF_JMP, BPF_JSET, BPF_K),
 219                 DL(BPF_JMP, BPF_EXIT, 0),
 220                 DL(BPF_STX, BPF_MEM, BPF_B),
 221                 DL(BPF_STX, BPF_MEM, BPF_H),
 222                 DL(BPF_STX, BPF_MEM, BPF_W),
 223                 DL(BPF_STX, BPF_MEM, BPF_DW),
 224                 DL(BPF_STX, BPF_XADD, BPF_W),
 225                 DL(BPF_STX, BPF_XADD, BPF_DW),
 226                 DL(BPF_ST, BPF_MEM, BPF_B),
 227                 DL(BPF_ST, BPF_MEM, BPF_H),
 228                 DL(BPF_ST, BPF_MEM, BPF_W),
 229                 DL(BPF_ST, BPF_MEM, BPF_DW),
 230                 DL(BPF_LDX, BPF_MEM, BPF_B),
 231                 DL(BPF_LDX, BPF_MEM, BPF_H),
 232                 DL(BPF_LDX, BPF_MEM, BPF_W),
 233                 DL(BPF_LDX, BPF_MEM, BPF_DW),
 234                 DL(BPF_LD, BPF_ABS, BPF_W),
 235                 DL(BPF_LD, BPF_ABS, BPF_H),
 236                 DL(BPF_LD, BPF_ABS, BPF_B),
 237                 DL(BPF_LD, BPF_IND, BPF_W),
 238                 DL(BPF_LD, BPF_IND, BPF_H),
 239                 DL(BPF_LD, BPF_IND, BPF_B),
 240 #undef DL
 241         };
 242
 243         regs[FP_REG]  = (u64) (unsigned long) &stack[ARRAY_SIZE(stack)];
 244         regs[ARG1_REG] = (u64) (unsigned long) ctx;
 245
 246 select_insn:
 247         goto *jumptable[insn->code];
 248
 249         /* ALU */
 250 #define ALU(OPCODE, OP)                 \
 251         BPF_ALU64_##OPCODE##_BPF_X:     \
 252                 A = A OP X;             \
 253                 CONT;                   \
 254         BPF_ALU_##OPCODE##_BPF_X:       \
 255                 A = (u32) A OP (u32) X; \
 256                 CONT;                   \
 257         BPF_ALU64_##OPCODE##_BPF_K:     \
 258                 A = A OP K;             \
 259                 CONT;                   \
 260         BPF_ALU_##OPCODE##_BPF_K:       \
 261                 A = (u32) A OP (u32) K; \
 262                 CONT;
 263
 264         ALU(BPF_ADD,  +)
 265         ALU(BPF_SUB,  -)
 266         ALU(BPF_AND,  &)
 267         ALU(BPF_OR,   |)
 268         ALU(BPF_LSH, <<)
 269         ALU(BPF_RSH, >>)
 270         ALU(BPF_XOR,  ^)
 271         ALU(BPF_MUL,  *)
 272 #undef ALU
 273         BPF_ALU_BPF_NEG_0:
 274                 A = (u32) -A;
 275                 CONT;
 276         BPF_ALU64_BPF_NEG_0:
 277                 A = -A;
 278                 CONT;
 279         BPF_ALU_BPF_MOV_BPF_X:
 280                 A = (u32) X;
 281                 CONT;
 282         BPF_ALU_BPF_MOV_BPF_K:
 283                 A = (u32) K;
 284                 CONT;
 285         BPF_ALU64_BPF_MOV_BPF_X:
 286                 A = X;
 287                 CONT;
 288         BPF_ALU64_BPF_MOV_BPF_K:
 289                 A = K;
 290                 CONT;
 291         BPF_ALU64_BPF_ARSH_BPF_X:
 292                 (*(s64 *) &A) >>= X;
 293                 CONT;
 294         BPF_ALU64_BPF_ARSH_BPF_K:
 295                 (*(s64 *) &A) >>= K;
 296                 CONT;
 297         BPF_ALU64_BPF_MOD_BPF_X:
 298                 tmp = A;
 299                 if (X)
 300                         A = do_div(tmp, X);
 301                 CONT;
 302         BPF_ALU_BPF_MOD_BPF_X:
 303                 tmp = (u32) A;
 304                 if (X)
 305                         A = do_div(tmp, (u32) X);
 306                 CONT;
 307         BPF_ALU64_BPF_MOD_BPF_K:
 308                 tmp = A;
 309                 if (K)
 310                         A = do_div(tmp, K);
 311                 CONT;
 312         BPF_ALU_BPF_MOD_BPF_K:
 313                 tmp = (u32) A;
 314                 if (K)
 315                         A = do_div(tmp, (u32) K);
 316                 CONT;
 317         BPF_ALU64_BPF_DIV_BPF_X:
 318                 if (X)
 319                         do_div(A, X);
 320                 CONT;
 321         BPF_ALU_BPF_DIV_BPF_X:
 322                 tmp = (u32) A;
 323                 if (X)
 324                         do_div(tmp, (u32) X);
 325                 A = (u32) tmp;
 326                 CONT;
 327         BPF_ALU64_BPF_DIV_BPF_K:
 328                 if (K)
 329                         do_div(A, K);
 330                 CONT;
 331         BPF_ALU_BPF_DIV_BPF_K:
 332                 tmp = (u32) A;
 333                 if (K)
 334                         do_div(tmp, (u32) K);
 335                 A = (u32) tmp;
 336                 CONT;
 337         BPF_ALU_BPF_END_BPF_TO_BE:
 338                 switch (K) {
 339                 case 16:
 340                         A = (__force u16) cpu_to_be16(A);
 341                         break;
 342                 case 32:
 343                         A = (__force u32) cpu_to_be32(A);
 344                         break;
 345                 case 64:
 346                         A = (__force u64) cpu_to_be64(A);
 347                         break;
 348                 }
 349                 CONT;
 350         BPF_ALU_BPF_END_BPF_TO_LE:
 351                 switch (K) {
 352                 case 16:
 353                         A = (__force u16) cpu_to_le16(A);
 354                         break;
 355                 case 32:
 356                         A = (__force u32) cpu_to_le32(A);
 357                         break;
 358                 case 64:
 359                         A = (__force u64) cpu_to_le64(A);
 360                         break;
 361                 }
 362                 CONT;
 363
 364         /* CALL */
 365         BPF_JMP_BPF_CALL_0:
 366                 /* Function call scratches R1-R5 registers, preserves R6-R9,
 367                  * and stores return value into R0.
 368                  */
 369                 R0 = (__bpf_call_base + insn->imm)(regs[1], regs[2], regs[3],
 370                                                    regs[4], regs[5]);
 371                 CONT;
 372
 373         /* JMP */
 374         BPF_JMP_BPF_JA_0:
 375                 insn += insn->off;
 376                 CONT;
 377         BPF_JMP_BPF_JEQ_BPF_X:
 378                 if (A == X) {
 379                         insn += insn->off;
 380                         CONT_JMP;
 381                 }
 382                 CONT;
 383         BPF_JMP_BPF_JEQ_BPF_K:
 384                 if (A == K) {
 385                         insn += insn->off;
 386                         CONT_JMP;
 387                 }
 388                 CONT;
 389         BPF_JMP_BPF_JNE_BPF_X:
 390                 if (A != X) {
 391                         insn += insn->off;
 392                         CONT_JMP;
 393                 }
 394                 CONT;
 395         BPF_JMP_BPF_JNE_BPF_K:
 396                 if (A != K) {
 397                         insn += insn->off;
 398                         CONT_JMP;
 399                 }
 400                 CONT;
 401         BPF_JMP_BPF_JGT_BPF_X:
 402                 if (A > X) {
 403                         insn += insn->off;
 404                         CONT_JMP;
 405                 }
 406                 CONT;
 407         BPF_JMP_BPF_JGT_BPF_K:
 408                 if (A > K) {
 409                         insn += insn->off;
 410                         CONT_JMP;
 411                 }
 412                 CONT;
 413         BPF_JMP_BPF_JGE_BPF_X:
 414                 if (A >= X) {
 415                         insn += insn->off;
 416                         CONT_JMP;
 417                 }
 418                 CONT;
 419         BPF_JMP_BPF_JGE_BPF_K:
 420                 if (A >= K) {
 421                         insn += insn->off;
 422                         CONT_JMP;
 423                 }
 424                 CONT;
 425         BPF_JMP_BPF_JSGT_BPF_X:
 426                 if (((s64)A) > ((s64)X)) {
 427                         insn += insn->off;
 428                         CONT_JMP;
 429                 }
 430                 CONT;
 431         BPF_JMP_BPF_JSGT_BPF_K:
 432                 if (((s64)A) > ((s64)K)) {
 433                         insn += insn->off;
 434                         CONT_JMP;
 435                 }
 436                 CONT;
 437         BPF_JMP_BPF_JSGE_BPF_X:
 438                 if (((s64)A) >= ((s64)X)) {
 439                         insn += insn->off;
 440                         CONT_JMP;
 441                 }
 442                 CONT;
 443         BPF_JMP_BPF_JSGE_BPF_K:
 444                 if (((s64)A) >= ((s64)K)) {
 445                         insn += insn->off;
 446                         CONT_JMP;
 447                 }
 448                 CONT;
 449         BPF_JMP_BPF_JSET_BPF_X:
 450                 if (A & X) {
 451                         insn += insn->off;
 452                         CONT_JMP;
 453                 }
 454                 CONT;
 455         BPF_JMP_BPF_JSET_BPF_K:
 456                 if (A & K) {
 457                         insn += insn->off;
 458                         CONT_JMP;
 459                 }
 460                 CONT;
 461         BPF_JMP_BPF_EXIT_0:
 462                 return R0;
 463
 464         /* STX and ST and LDX*/
 465 #define LDST(SIZEOP, SIZE)                                      \
 466         BPF_STX_BPF_MEM_##SIZEOP:                               \
 467                 *(SIZE *)(unsigned long) (A + insn->off) = X;   \
 468                 CONT;                                           \
 469         BPF_ST_BPF_MEM_##SIZEOP:                                \
 470                 *(SIZE *)(unsigned long) (A + insn->off) = K;   \
 471                 CONT;                                           \
 472         BPF_LDX_BPF_MEM_##SIZEOP:                               \
 473                 A = *(SIZE *)(unsigned long) (X + insn->off);   \
 474                 CONT;
 475
 476         LDST(BPF_B,   u8)
 477         LDST(BPF_H,  u16)
 478         LDST(BPF_W,  u32)
 479         LDST(BPF_DW, u64)
 480 #undef LDST
 481         BPF_STX_BPF_XADD_BPF_W: /* lock xadd *(u32 *)(A + insn->off) += X */
 482                 atomic_add((u32) X, (atomic_t *)(unsigned long)
 483                            (A + insn->off));
 484                 CONT;
 485         BPF_STX_BPF_XADD_BPF_DW: /* lock xadd *(u64 *)(A + insn->off) += X */
 486                 atomic64_add((u64) X, (atomic64_t *)(unsigned long)
 487                              (A + insn->off));
 488                 CONT;
 489         BPF_LD_BPF_ABS_BPF_W: /* R0 = ntohl(*(u32 *) (skb->data + K)) */
 490                 off = K;
 491 load_word:
 492                 /* BPF_LD + BPD_ABS and BPF_LD + BPF_IND insns are only
 493                  * appearing in the programs where ctx == skb. All programs
 494                  * keep 'ctx' in regs[CTX_REG] == R6, sk_convert_filter()
 495                  * saves it in R6, internal BPF verifier will check that
 496                  * R6 == ctx.
 497                  *
 498                  * BPF_ABS and BPF_IND are wrappers of function calls, so
 499                  * they scratch R1-R5 registers, preserve R6-R9, and store
 500                  * return value into R0.
 501                  *
 502                  * Implicit input:
 503                  *   ctx
 504                  *
 505                  * Explicit input:
 506                  *   X == any register
 507                  *   K == 32-bit immediate
 508                  *
 509                  * Output:
 510                  *   R0 - 8/16/32-bit skb data converted to cpu endianness
 511                  */
 512                 ptr = load_pointer((struct sk_buff *) ctx, off, 4, &tmp);
 513                 if (likely(ptr != NULL)) {
 514                         R0 = get_unaligned_be32(ptr);
 515                         CONT;
 516                 }
 517                 return 0;
 518         BPF_LD_BPF_ABS_BPF_H: /* R0 = ntohs(*(u16 *) (skb->data + K)) */
 519                 off = K;
 520 load_half:
 521                 ptr = load_pointer((struct sk_buff *) ctx, off, 2, &tmp);
 522                 if (likely(ptr != NULL)) {
 523                         R0 = get_unaligned_be16(ptr);
 524                         CONT;
 525                 }
 526                 return 0;
 527         BPF_LD_BPF_ABS_BPF_B: /* R0 = *(u8 *) (ctx + K) */
 528                 off = K;
 529 load_byte:
 530                 ptr = load_pointer((struct sk_buff *) ctx, off, 1, &tmp);
 531                 if (likely(ptr != NULL)) {
 532                         R0 = *(u8 *)ptr;
 533                         CONT;
 534                 }
 535                 return 0;
 536         BPF_LD_BPF_IND_BPF_W: /* R0 = ntohl(*(u32 *) (skb->data + X + K)) */
 537                 off = K + X;
 538                 goto load_word;
 539         BPF_LD_BPF_IND_BPF_H: /* R0 = ntohs(*(u16 *) (skb->data + X + K)) */
 540                 off = K + X;
 541                 goto load_half;
 542         BPF_LD_BPF_IND_BPF_B: /* R0 = *(u8 *) (skb->data + X + K) */
 543                 off = K + X;
 544                 goto load_byte;
 545
 546         default_label:
 547                 /* If we ever reach this, we have a bug somewhere. */
 548                 WARN_RATELIMIT(1, "unknown opcode %02x\n", insn->code);
 549                 return 0;
 550 #undef CONT_JMP
 551 #undef CONT
 552
 553 #undef R0
 554 #undef X
 555 #undef A
 556 #undef K
 557 }
 558
 559 u32 sk_run_filter_int_seccomp(const struct seccomp_data *ctx,
 560                               const struct sock_filter_int *insni)
 561     __attribute__ ((alias ("__sk_run_filter")));
 562
 563 u32 sk_run_filter_int_skb(const struct sk_buff *ctx,
 564                           const struct sock_filter_int *insni)
 565     __attribute__ ((alias ("__sk_run_filter")));
 566 EXPORT_SYMBOL_GPL(sk_run_filter_int_skb);
 567
 568 /* Helper to find the offset of pkt_type in sk_buff structure. We want
 569  * to make sure its still a 3bit field starting at a byte boundary;
 570  * taken from arch/x86/net/bpf_jit_comp.c.
 571  */
 572 #define PKT_TYPE_MAX    7
 573 static unsigned int pkt_type_offset(void)
 574 {
 575         struct sk_buff skb_probe = { .pkt_type = ~0, };
 576         u8 *ct = (u8 *) &skb_probe;
 577         unsigned int off;
 578
 579         for (off = 0; off < sizeof(struct sk_buff); off++) {
 580                 if (ct[off] == PKT_TYPE_MAX)
 581                         return off;
 582         }
 583
 584         pr_err_once("Please fix %s, as pkt_type couldn't be found!\n", __func__);
 585         return -1;
 586 }
 587
 588 static u64 __skb_get_pay_offset(u64 ctx, u64 A, u64 X, u64 r4, u64 r5)
 589 {
 590         struct sk_buff *skb = (struct sk_buff *)(long) ctx;
 591
 592         return __skb_get_poff(skb);
 593 }
 594
 595 static u64 __skb_get_nlattr(u64 ctx, u64 A, u64 X, u64 r4, u64 r5)
 596 {
 597         struct sk_buff *skb = (struct sk_buff *)(long) ctx;
 598         struct nlattr *nla;
 599
 600         if (skb_is_nonlinear(skb))
 601                 return 0;
 602
 603         if (A > skb->len - sizeof(struct nlattr))
 604                 return 0;
 605
 606         nla = nla_find((struct nlattr *) &skb->data[A], skb->len - A, X);
 607         if (nla)
 608                 return (void *) nla - (void *) skb->data;
 609
 610         return 0;
 611 }
 612
 613 static u64 __skb_get_nlattr_nest(u64 ctx, u64 A, u64 X, u64 r4, u64 r5)
 614 {
 615         struct sk_buff *skb = (struct sk_buff *)(long) ctx;
 616         struct nlattr *nla;
 617
 618         if (skb_is_nonlinear(skb))
 619                 return 0;
 620
 621         if (A > skb->len - sizeof(struct nlattr))
 622                 return 0;
 623
 624         nla = (struct nlattr *) &skb->data[A];
 625         if (nla->nla_len > A - skb->len)
 626                 return 0;
 627
 628         nla = nla_find_nested(nla, X);
 629         if (nla)
 630                 return (void *) nla - (void *) skb->data;
 631
 632         return 0;
 633 }
 634
 635 static u64 __get_raw_cpu_id(u64 ctx, u64 A, u64 X, u64 r4, u64 r5)
 636 {
 637         return raw_smp_processor_id();
 638 }
 639
 640 /* Register mappings for user programs. */
 641 #define A_REG           0
 642 #define X_REG           7
 643 #define TMP_REG         8
 644 #define ARG2_REG        2
 645 #define ARG3_REG        3
 646
 647 static bool convert_bpf_extensions(struct sock_filter *fp,
 648                                    struct sock_filter_int **insnp)
 649 {
 650         struct sock_filter_int *insn = *insnp;
 651
 652         switch (fp->k) {
 653         case SKF_AD_OFF + SKF_AD_PROTOCOL:
 654                 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, protocol) != 2);
 655
 656                 insn->code = BPF_LDX | BPF_MEM | BPF_H;
 657                 insn->a_reg = A_REG;
 658                 insn->x_reg = CTX_REG;
 659                 insn->off = offsetof(struct sk_buff, protocol);
 660                 insn++;
 661
 662                 /* A = ntohs(A) [emitting a nop or swap16] */
 663                 insn->code = BPF_ALU | BPF_END | BPF_FROM_BE;
 664                 insn->a_reg = A_REG;
 665                 insn->imm = 16;
 666                 break;
 667
 668         case SKF_AD_OFF + SKF_AD_PKTTYPE:
 669                 insn->code = BPF_LDX | BPF_MEM | BPF_B;
 670                 insn->a_reg = A_REG;
 671                 insn->x_reg = CTX_REG;
 672                 insn->off = pkt_type_offset();
 673                 if (insn->off < 0)
 674                         return false;
 675                 insn++;
 676
 677                 insn->code = BPF_ALU | BPF_AND | BPF_K;
 678                 insn->a_reg = A_REG;
 679                 insn->imm = PKT_TYPE_MAX;
 680                 break;
 681
 682         case SKF_AD_OFF + SKF_AD_IFINDEX:
 683         case SKF_AD_OFF + SKF_AD_HATYPE:
 684                 if (FIELD_SIZEOF(struct sk_buff, dev) == 8)
 685                         insn->code = BPF_LDX | BPF_MEM | BPF_DW;
 686                 else
 687                         insn->code = BPF_LDX | BPF_MEM | BPF_W;
 688                 insn->a_reg = TMP_REG;
 689                 insn->x_reg = CTX_REG;
 690                 insn->off = offsetof(struct sk_buff, dev);
 691                 insn++;
 692
 693                 insn->code = BPF_JMP | BPF_JNE | BPF_K;
 694                 insn->a_reg = TMP_REG;
 695                 insn->imm = 0;
 696                 insn->off = 1;
 697                 insn++;
 698
 699                 insn->code = BPF_JMP | BPF_EXIT;
 700                 insn++;
 701
 702                 BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, ifindex) != 4);
 703                 BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, type) != 2);
 704
 705                 insn->a_reg = A_REG;
 706                 insn->x_reg = TMP_REG;
 707
 708                 if (fp->k == SKF_AD_OFF + SKF_AD_IFINDEX) {
 709                         insn->code = BPF_LDX | BPF_MEM | BPF_W;
 710                         insn->off = offsetof(struct net_device, ifindex);
 711                 } else {
 712                         insn->code = BPF_LDX | BPF_MEM | BPF_H;
 713                         insn->off = offsetof(struct net_device, type);
 714                 }
 715                 break;
 716
 717         case SKF_AD_OFF + SKF_AD_MARK:
 718                 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, mark) != 4);
 719
 720                 insn->code = BPF_LDX | BPF_MEM | BPF_W;
 721                 insn->a_reg = A_REG;
 722                 insn->x_reg = CTX_REG;
 723                 insn->off = offsetof(struct sk_buff, mark);
 724                 break;
 725
 726         case SKF_AD_OFF + SKF_AD_RXHASH:
 727                 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, hash) != 4);
 728
 729                 insn->code = BPF_LDX | BPF_MEM | BPF_W;
 730                 insn->a_reg = A_REG;
 731                 insn->x_reg = CTX_REG;
 732                 insn->off = offsetof(struct sk_buff, hash);
 733                 break;
 734
 735         case SKF_AD_OFF + SKF_AD_QUEUE:
 736                 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, queue_mapping) != 2);
 737
 738                 insn->code = BPF_LDX | BPF_MEM | BPF_H;
 739                 insn->a_reg = A_REG;
 740                 insn->x_reg = CTX_REG;
 741                 insn->off = offsetof(struct sk_buff, queue_mapping);
 742                 break;
 743
 744         case SKF_AD_OFF + SKF_AD_VLAN_TAG:
 745         case SKF_AD_OFF + SKF_AD_VLAN_TAG_PRESENT:
 746                 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, vlan_tci) != 2);
 747
 748                 insn->code = BPF_LDX | BPF_MEM | BPF_H;
 749                 insn->a_reg = A_REG;
 750                 insn->x_reg = CTX_REG;
 751                 insn->off = offsetof(struct sk_buff, vlan_tci);
 752                 insn++;
 753
 754                 BUILD_BUG_ON(VLAN_TAG_PRESENT != 0x1000);
 755
 756                 if (fp->k == SKF_AD_OFF + SKF_AD_VLAN_TAG) {
 757                         insn->code = BPF_ALU | BPF_AND | BPF_K;
 758                         insn->a_reg = A_REG;
 759                         insn->imm = ~VLAN_TAG_PRESENT;
 760                 } else {
 761                         insn->code = BPF_ALU | BPF_RSH | BPF_K;
 762                         insn->a_reg = A_REG;
 763                         insn->imm = 12;
 764                         insn++;
 765
 766                         insn->code = BPF_ALU | BPF_AND | BPF_K;
 767                         insn->a_reg = A_REG;
 768                         insn->imm = 1;
 769                 }
 770                 break;
 771
 772         case SKF_AD_OFF + SKF_AD_PAY_OFFSET:
 773         case SKF_AD_OFF + SKF_AD_NLATTR:
 774         case SKF_AD_OFF + SKF_AD_NLATTR_NEST:
 775         case SKF_AD_OFF + SKF_AD_CPU:
 776                 /* arg1 = ctx */
 777                 insn->code = BPF_ALU64 | BPF_MOV | BPF_X;
 778                 insn->a_reg = ARG1_REG;
 779                 insn->x_reg = CTX_REG;
 780                 insn++;
 781
 782                 /* arg2 = A */
 783                 insn->code = BPF_ALU64 | BPF_MOV | BPF_X;
 784                 insn->a_reg = ARG2_REG;
 785                 insn->x_reg = A_REG;
 786                 insn++;
 787
 788                 /* arg3 = X */
 789                 insn->code = BPF_ALU64 | BPF_MOV | BPF_X;
 790                 insn->a_reg = ARG3_REG;
 791                 insn->x_reg = X_REG;
 792                 insn++;
 793
 794                 /* Emit call(ctx, arg2=A, arg3=X) */
 795                 insn->code = BPF_JMP | BPF_CALL;
 796                 switch (fp->k) {
 797                 case SKF_AD_OFF + SKF_AD_PAY_OFFSET:
 798                         insn->imm = __skb_get_pay_offset - __bpf_call_base;
 799                         break;
 800                 case SKF_AD_OFF + SKF_AD_NLATTR:
 801                         insn->imm = __skb_get_nlattr - __bpf_call_base;
 802                         break;
 803                 case SKF_AD_OFF + SKF_AD_NLATTR_NEST:
 804                         insn->imm = __skb_get_nlattr_nest - __bpf_call_base;
 805                         break;
 806                 case SKF_AD_OFF + SKF_AD_CPU:
 807                         insn->imm = __get_raw_cpu_id - __bpf_call_base;
 808                         break;
 809                 }
 810                 break;
 811
 812         case SKF_AD_OFF + SKF_AD_ALU_XOR_X:
 813                 insn->code = BPF_ALU | BPF_XOR | BPF_X;
 814                 insn->a_reg = A_REG;
 815                 insn->x_reg = X_REG;
 816                 break;
 817
 818         default:
 819                 /* This is just a dummy call to avoid letting the compiler
 820                  * evict __bpf_call_base() as an optimization. Placed here
 821                  * where no-one bothers.
 822                  */
 823                 BUG_ON(__bpf_call_base(0, 0, 0, 0, 0) != 0);
 824                 return false;
 825         }
 826
 827         *insnp = insn;
 828         return true;
 829 }
 830
 831 /**
 832  *      sk_convert_filter - convert filter program
 833  *      @prog: the user passed filter program
 834  *      @len: the length of the user passed filter program
 835  *      @new_prog: buffer where converted program will be stored
 836  *      @new_len: pointer to store length of converted program
 837  *
 838  * Remap 'sock_filter' style BPF instruction set to 'sock_filter_ext' style.
 839  * Conversion workflow:
 840  *
 841  * 1) First pass for calculating the new program length:
 842  *   sk_convert_filter(old_prog, old_len, NULL, &new_len)
 843  *
 844  * 2) 2nd pass to remap in two passes: 1st pass finds new
 845  *    jump offsets, 2nd pass remapping:
 846  *   new_prog = kmalloc(sizeof(struct sock_filter_int) * new_len);
 847  *   sk_convert_filter(old_prog, old_len, new_prog, &new_len);
 848  *
 849  * User BPF's register A is mapped to our BPF register 6, user BPF
 850  * register X is mapped to BPF register 7; frame pointer is always
 851  * register 10; Context 'void *ctx' is stored in register 1, that is,
 852  * for socket filters: ctx == 'struct sk_buff *', for seccomp:
 853  * ctx == 'struct seccomp_data *'.
 854  */
 855 int sk_convert_filter(struct sock_filter *prog, int len,
 856                       struct sock_filter_int *new_prog, int *new_len)
 857 {
 858         int new_flen = 0, pass = 0, target, i;
 859         struct sock_filter_int *new_insn;
 860         struct sock_filter *fp;
 861         int *addrs = NULL;
 862         u8 bpf_src;
 863
 864         BUILD_BUG_ON(BPF_MEMWORDS * sizeof(u32) > MAX_BPF_STACK);
 865         BUILD_BUG_ON(FP_REG + 1 != MAX_BPF_REG);
 866
 867         if (len <= 0 || len >= BPF_MAXINSNS)
 868                 return -EINVAL;
 869
 870         if (new_prog) {
 871                 addrs = kzalloc(len * sizeof(*addrs), GFP_KERNEL);
 872                 if (!addrs)
 873                         return -ENOMEM;
 874         }
 875
 876 do_pass:
 877         new_insn = new_prog;
 878         fp = prog;
 879
 880         if (new_insn) {
 881                 new_insn->code = BPF_ALU64 | BPF_MOV | BPF_X;
 882                 new_insn->a_reg = CTX_REG;
 883                 new_insn->x_reg = ARG1_REG;
 884         }
 885         new_insn++;
 886
 887         for (i = 0; i < len; fp++, i++) {
 888                 struct sock_filter_int tmp_insns[6] = { };
 889                 struct sock_filter_int *insn = tmp_insns;
 890
 891                 if (addrs)
 892                         addrs[i] = new_insn - new_prog;
 893
 894                 switch (fp->code) {
 895                 /* All arithmetic insns and skb loads map as-is. */
 896                 case BPF_ALU | BPF_ADD | BPF_X:
 897                 case BPF_ALU | BPF_ADD | BPF_K:
 898                 case BPF_ALU | BPF_SUB | BPF_X:
 899                 case BPF_ALU | BPF_SUB | BPF_K:
 900                 case BPF_ALU | BPF_AND | BPF_X:
 901                 case BPF_ALU | BPF_AND | BPF_K:
 902                 case BPF_ALU | BPF_OR | BPF_X:
 903                 case BPF_ALU | BPF_OR | BPF_K:
 904                 case BPF_ALU | BPF_LSH | BPF_X:
 905                 case BPF_ALU | BPF_LSH | BPF_K:
 906                 case BPF_ALU | BPF_RSH | BPF_X:
 907                 case BPF_ALU | BPF_RSH | BPF_K:
 908                 case BPF_ALU | BPF_XOR | BPF_X:
 909                 case BPF_ALU | BPF_XOR | BPF_K:
 910                 case BPF_ALU | BPF_MUL | BPF_X:
 911                 case BPF_ALU | BPF_MUL | BPF_K:
 912                 case BPF_ALU | BPF_DIV | BPF_X:
 913                 case BPF_ALU | BPF_DIV | BPF_K:
 914                 case BPF_ALU | BPF_MOD | BPF_X:
 915                 case BPF_ALU | BPF_MOD | BPF_K:
 916                 case BPF_ALU | BPF_NEG:
 917                 case BPF_LD | BPF_ABS | BPF_W:
 918                 case BPF_LD | BPF_ABS | BPF_H:
 919                 case BPF_LD | BPF_ABS | BPF_B:
 920                 case BPF_LD | BPF_IND | BPF_W:
 921                 case BPF_LD | BPF_IND | BPF_H:
 922                 case BPF_LD | BPF_IND | BPF_B:
 923                         /* Check for overloaded BPF extension and
 924                          * directly convert it if found, otherwise
 925                          * just move on with mapping.
 926                          */
 927                         if (BPF_CLASS(fp->code) == BPF_LD &&
 928                             BPF_MODE(fp->code) == BPF_ABS &&
 929                             convert_bpf_extensions(fp, &insn))
 930                                 break;
 931
 932                         insn->code = fp->code;
 933                         insn->a_reg = A_REG;
 934                         insn->x_reg = X_REG;
 935                         insn->imm = fp->k;
 936                         break;
 937
 938                 /* Jump opcodes map as-is, but offsets need adjustment. */
 939                 case BPF_JMP | BPF_JA:
 940                         target = i + fp->k + 1;
 941                         insn->code = fp->code;
 942 #define EMIT_JMP                                                        \
 943         do {                                                            \
 944                 if (target >= len || target < 0)                        \
 945                         goto err;                                       \
 946                 insn->off = addrs ? addrs[target] - addrs[i] - 1 : 0;   \
 947                 /* Adjust pc relative offset for 2nd or 3rd insn. */    \
 948                 insn->off -= insn - tmp_insns;                          \
 949         } while (0)
 950
 951                         EMIT_JMP;
 952                         break;
 953
 954                 case BPF_JMP | BPF_JEQ | BPF_K:
 955                 case BPF_JMP | BPF_JEQ | BPF_X:
 956                 case BPF_JMP | BPF_JSET | BPF_K:
 957                 case BPF_JMP | BPF_JSET | BPF_X:
 958                 case BPF_JMP | BPF_JGT | BPF_K:
 959                 case BPF_JMP | BPF_JGT | BPF_X:
 960                 case BPF_JMP | BPF_JGE | BPF_K:
 961                 case BPF_JMP | BPF_JGE | BPF_X:
 962                         if (BPF_SRC(fp->code) == BPF_K && (int) fp->k < 0) {
 963                                 /* BPF immediates are signed, zero extend
 964                                  * immediate into tmp register and use it
 965                                  * in compare insn.
 966                                  */
 967                                 insn->code = BPF_ALU | BPF_MOV | BPF_K;
 968                                 insn->a_reg = TMP_REG;
 969                                 insn->imm = fp->k;
 970                                 insn++;
 971
 972                                 insn->a_reg = A_REG;
 973                                 insn->x_reg = TMP_REG;
 974                                 bpf_src = BPF_X;
 975                         } else {
 976                                 insn->a_reg = A_REG;
 977                                 insn->x_reg = X_REG;
 978                                 insn->imm = fp->k;
 979                                 bpf_src = BPF_SRC(fp->code);
 980                         }
 981
 982                         /* Common case where 'jump_false' is next insn. */
 983                         if (fp->jf == 0) {
 984                                 insn->code = BPF_JMP | BPF_OP(fp->code) | bpf_src;
 985                                 target = i + fp->jt + 1;
 986                                 EMIT_JMP;
 987                                 break;
 988                         }
 989
 990                         /* Convert JEQ into JNE when 'jump_true' is next insn. */
 991                         if (fp->jt == 0 && BPF_OP(fp->code) == BPF_JEQ) {
 992                                 insn->code = BPF_JMP | BPF_JNE | bpf_src;
 993                                 target = i + fp->jf + 1;
 994                                 EMIT_JMP;
 995                                 break;
 996                         }
 997
 998                         /* Other jumps are mapped into two insns: Jxx and JA. */
 999                         target = i + fp->jt + 1;
1000                         insn->code = BPF_JMP | BPF_OP(fp->code) | bpf_src;
1001                         EMIT_JMP;
1002                         insn++;
1003
1004                         insn->code = BPF_JMP | BPF_JA;
1005                         target = i + fp->jf + 1;
1006                         EMIT_JMP;
1007                         break;
1008
1009                 /* ldxb 4 * ([14] & 0xf) is remaped into 6 insns. */
1010                 case BPF_LDX | BPF_MSH | BPF_B:
1011                         insn->code = BPF_ALU64 | BPF_MOV | BPF_X;
1012                         insn->a_reg = TMP_REG;
1013                         insn->x_reg = A_REG;
1014                         insn++;
1015
1016                         insn->code = BPF_LD | BPF_ABS | BPF_B;
1017                         insn->a_reg = A_REG;
1018                         insn->imm = fp->k;
1019                         insn++;
1020
1021                         insn->code = BPF_ALU | BPF_AND | BPF_K;
1022                         insn->a_reg = A_REG;
1023                         insn->imm = 0xf;
1024                         insn++;
1025
1026                         insn->code = BPF_ALU | BPF_LSH | BPF_K;
1027                         insn->a_reg = A_REG;
1028                         insn->imm = 2;
1029                         insn++;
1030
1031                         insn->code = BPF_ALU64 | BPF_MOV | BPF_X;
1032                         insn->a_reg = X_REG;
1033                         insn->x_reg = A_REG;
1034                         insn++;
1035
1036                         insn->code = BPF_ALU64 | BPF_MOV | BPF_X;
1037                         insn->a_reg = A_REG;
1038                         insn->x_reg = TMP_REG;
1039                         break;
1040
1041                 /* RET_K, RET_A are remaped into 2 insns. */
1042                 case BPF_RET | BPF_A:
1043                 case BPF_RET | BPF_K:
1044                         insn->code = BPF_ALU | BPF_MOV |
1045                                      (BPF_RVAL(fp->code) == BPF_K ?
1046                                       BPF_K : BPF_X);
1047                         insn->a_reg = 0;
1048                         insn->x_reg = A_REG;
1049                         insn->imm = fp->k;
1050                         insn++;
1051
1052                         insn->code = BPF_JMP | BPF_EXIT;
1053                         break;
1054
1055                 /* Store to stack. */
1056                 case BPF_ST:
1057                 case BPF_STX:
1058                         insn->code = BPF_STX | BPF_MEM | BPF_W;
1059                         insn->a_reg = FP_REG;
1060                         insn->x_reg = fp->code == BPF_ST ? A_REG : X_REG;
1061                         insn->off = -(BPF_MEMWORDS - fp->k) * 4;
1062                         break;
1063
1064                 /* Load from stack. */
1065                 case BPF_LD | BPF_MEM:
1066                 case BPF_LDX | BPF_MEM:
1067                         insn->code = BPF_LDX | BPF_MEM | BPF_W;
1068                         insn->a_reg = BPF_CLASS(fp->code) == BPF_LD ?
1069                                       A_REG : X_REG;
1070                         insn->x_reg = FP_REG;
1071                         insn->off = -(BPF_MEMWORDS - fp->k) * 4;
1072                         break;
1073
1074                 /* A = K or X = K */
1075                 case BPF_LD | BPF_IMM:
1076                 case BPF_LDX | BPF_IMM:
1077                         insn->code = BPF_ALU | BPF_MOV | BPF_K;
1078                         insn->a_reg = BPF_CLASS(fp->code) == BPF_LD ?
1079                                       A_REG : X_REG;
1080                         insn->imm = fp->k;
1081                         break;
1082
1083                 /* X = A */
1084                 case BPF_MISC | BPF_TAX:
1085                         insn->code = BPF_ALU64 | BPF_MOV | BPF_X;
1086                         insn->a_reg = X_REG;
1087                         insn->x_reg = A_REG;
1088                         break;
1089
1090                 /* A = X */
1091                 case BPF_MISC | BPF_TXA:
1092                         insn->code = BPF_ALU64 | BPF_MOV | BPF_X;
1093                         insn->a_reg = A_REG;
1094                         insn->x_reg = X_REG;
1095                         break;
1096
1097                 /* A = skb->len or X = skb->len */
1098                 case BPF_LD | BPF_W | BPF_LEN:
1099                 case BPF_LDX | BPF_W | BPF_LEN:
1100                         insn->code = BPF_LDX | BPF_MEM | BPF_W;
1101                         insn->a_reg = BPF_CLASS(fp->code) == BPF_LD ?
1102                                       A_REG : X_REG;
1103                         insn->x_reg = CTX_REG;
1104                         insn->off = offsetof(struct sk_buff, len);
1105                         break;
1106
1107                 /* access seccomp_data fields */
1108                 case BPF_LDX | BPF_ABS | BPF_W:
1109                         insn->code = BPF_LDX | BPF_MEM | BPF_W;
1110                         insn->a_reg = A_REG;
1111                         insn->x_reg = CTX_REG;
1112                         insn->off = fp->k;
1113                         break;
1114
1115                 default:
1116                         goto err;
1117                 }
1118
1119                 insn++;
1120                 if (new_prog)
1121                         memcpy(new_insn, tmp_insns,
1122                                sizeof(*insn) * (insn - tmp_insns));
1123
1124                 new_insn += insn - tmp_insns;
1125         }
1126
1127         if (!new_prog) {
1128                 /* Only calculating new length. */
1129                 *new_len = new_insn - new_prog;
1130                 return 0;
1131         }
1132
1133         pass++;
1134         if (new_flen != new_insn - new_prog) {
1135                 new_flen = new_insn - new_prog;
1136                 if (pass > 2)
1137                         goto err;
1138
1139                 goto do_pass;
1140         }
1141
1142         kfree(addrs);
1143         BUG_ON(*new_len != new_flen);
1144         return 0;
1145 err:
1146         kfree(addrs);
1147         return -EINVAL;
1148 }
1149
1150 /* Security:
1151  *
1152  * A BPF program is able to use 16 cells of memory to store intermediate
1153  * values (check u32 mem[BPF_MEMWORDS] in sk_run_filter()).
1154  *
1155  * As we dont want to clear mem[] array for each packet going through
1156  * sk_run_filter(), we check that filter loaded by user never try to read
1157  * a cell if not previously written, and we check all branches to be sure
1158  * a malicious user doesn't try to abuse us.
1159  */
1160 static int check_load_and_stores(struct sock_filter *filter, int flen)
1161 {
1162         u16 *masks, memvalid = 0; /* one bit per cell, 16 cells */
1163         int pc, ret = 0;
1164
1165         BUILD_BUG_ON(BPF_MEMWORDS > 16);
1166         masks = kmalloc(flen * sizeof(*masks), GFP_KERNEL);
1167         if (!masks)
1168                 return -ENOMEM;
1169         memset(masks, 0xff, flen * sizeof(*masks));
1170
1171         for (pc = 0; pc < flen; pc++) {
1172                 memvalid &= masks[pc];
1173
1174                 switch (filter[pc].code) {
1175                 case BPF_S_ST:
1176                 case BPF_S_STX:
1177                         memvalid |= (1 << filter[pc].k);
1178                         break;
1179                 case BPF_S_LD_MEM:
1180                 case BPF_S_LDX_MEM:
1181                         if (!(memvalid & (1 << filter[pc].k))) {
1182                                 ret = -EINVAL;
1183                                 goto error;
1184                         }
1185                         break;
1186                 case BPF_S_JMP_JA:
1187                         /* a jump must set masks on target */
1188                         masks[pc + 1 + filter[pc].k] &= memvalid;
1189                         memvalid = ~0;
1190                         break;
1191                 case BPF_S_JMP_JEQ_K:
1192                 case BPF_S_JMP_JEQ_X:
1193                 case BPF_S_JMP_JGE_K:
1194                 case BPF_S_JMP_JGE_X:
1195                 case BPF_S_JMP_JGT_K:
1196                 case BPF_S_JMP_JGT_X:
1197                 case BPF_S_JMP_JSET_X:
1198                 case BPF_S_JMP_JSET_K:
1199                         /* a jump must set masks on targets */
1200                         masks[pc + 1 + filter[pc].jt] &= memvalid;
1201                         masks[pc + 1 + filter[pc].jf] &= memvalid;
1202                         memvalid = ~0;
1203                         break;
1204                 }
1205         }
1206 error:
1207         kfree(masks);
1208         return ret;
1209 }
1210
1211 /**
1212  *      sk_chk_filter - verify socket filter code
1213  *      @filter: filter to verify
1214  *      @flen: length of filter
1215  *
1216  * Check the user's filter code. If we let some ugly
1217  * filter code slip through kaboom! The filter must contain
1218  * no references or jumps that are out of range, no illegal
1219  * instructions, and must end with a RET instruction.
1220  *
1221  * All jumps are forward as they are not signed.
1222  *
1223  * Returns 0 if the rule set is legal or -EINVAL if not.
1224  */
1225 int sk_chk_filter(struct sock_filter *filter, unsigned int flen)
1226 {
1227         /*
1228          * Valid instructions are initialized to non-0.
1229          * Invalid instructions are initialized to 0.
1230          */
1231         static const u8 codes[] = {
1232                 [BPF_ALU|BPF_ADD|BPF_K]  = BPF_S_ALU_ADD_K,
1233                 [BPF_ALU|BPF_ADD|BPF_X]  = BPF_S_ALU_ADD_X,
1234                 [BPF_ALU|BPF_SUB|BPF_K]  = BPF_S_ALU_SUB_K,
1235                 [BPF_ALU|BPF_SUB|BPF_X]  = BPF_S_ALU_SUB_X,
1236                 [BPF_ALU|BPF_MUL|BPF_K]  = BPF_S_ALU_MUL_K,
1237                 [BPF_ALU|BPF_MUL|BPF_X]  = BPF_S_ALU_MUL_X,
1238                 [BPF_ALU|BPF_DIV|BPF_X]  = BPF_S_ALU_DIV_X,
1239                 [BPF_ALU|BPF_MOD|BPF_K]  = BPF_S_ALU_MOD_K,
1240                 [BPF_ALU|BPF_MOD|BPF_X]  = BPF_S_ALU_MOD_X,
1241                 [BPF_ALU|BPF_AND|BPF_K]  = BPF_S_ALU_AND_K,
1242                 [BPF_ALU|BPF_AND|BPF_X]  = BPF_S_ALU_AND_X,
1243                 [BPF_ALU|BPF_OR|BPF_K]   = BPF_S_ALU_OR_K,
1244                 [BPF_ALU|BPF_OR|BPF_X]   = BPF_S_ALU_OR_X,
1245                 [BPF_ALU|BPF_XOR|BPF_K]  = BPF_S_ALU_XOR_K,
1246                 [BPF_ALU|BPF_XOR|BPF_X]  = BPF_S_ALU_XOR_X,
1247                 [BPF_ALU|BPF_LSH|BPF_K]  = BPF_S_ALU_LSH_K,
1248                 [BPF_ALU|BPF_LSH|BPF_X]  = BPF_S_ALU_LSH_X,
1249                 [BPF_ALU|BPF_RSH|BPF_K]  = BPF_S_ALU_RSH_K,
1250                 [BPF_ALU|BPF_RSH|BPF_X]  = BPF_S_ALU_RSH_X,
1251                 [BPF_ALU|BPF_NEG]        = BPF_S_ALU_NEG,
1252                 [BPF_LD|BPF_W|BPF_ABS]   = BPF_S_LD_W_ABS,
1253                 [BPF_LD|BPF_H|BPF_ABS]   = BPF_S_LD_H_ABS,
1254                 [BPF_LD|BPF_B|BPF_ABS]   = BPF_S_LD_B_ABS,
1255                 [BPF_LD|BPF_W|BPF_LEN]   = BPF_S_LD_W_LEN,
1256                 [BPF_LD|BPF_W|BPF_IND]   = BPF_S_LD_W_IND,
1257                 [BPF_LD|BPF_H|BPF_IND]   = BPF_S_LD_H_IND,
1258                 [BPF_LD|BPF_B|BPF_IND]   = BPF_S_LD_B_IND,
1259                 [BPF_LD|BPF_IMM]         = BPF_S_LD_IMM,
1260                 [BPF_LDX|BPF_W|BPF_LEN]  = BPF_S_LDX_W_LEN,
1261                 [BPF_LDX|BPF_B|BPF_MSH]  = BPF_S_LDX_B_MSH,
1262                 [BPF_LDX|BPF_IMM]        = BPF_S_LDX_IMM,
1263                 [BPF_MISC|BPF_TAX]       = BPF_S_MISC_TAX,
1264                 [BPF_MISC|BPF_TXA]       = BPF_S_MISC_TXA,
1265                 [BPF_RET|BPF_K]          = BPF_S_RET_K,
1266                 [BPF_RET|BPF_A]          = BPF_S_RET_A,
1267                 [BPF_ALU|BPF_DIV|BPF_K]  = BPF_S_ALU_DIV_K,
1268                 [BPF_LD|BPF_MEM]         = BPF_S_LD_MEM,
1269                 [BPF_LDX|BPF_MEM]        = BPF_S_LDX_MEM,
1270                 [BPF_ST]                 = BPF_S_ST,
1271                 [BPF_STX]                = BPF_S_STX,
1272                 [BPF_JMP|BPF_JA]         = BPF_S_JMP_JA,
1273                 [BPF_JMP|BPF_JEQ|BPF_K]  = BPF_S_JMP_JEQ_K,
1274                 [BPF_JMP|BPF_JEQ|BPF_X]  = BPF_S_JMP_JEQ_X,
1275                 [BPF_JMP|BPF_JGE|BPF_K]  = BPF_S_JMP_JGE_K,
1276                 [BPF_JMP|BPF_JGE|BPF_X]  = BPF_S_JMP_JGE_X,
1277                 [BPF_JMP|BPF_JGT|BPF_K]  = BPF_S_JMP_JGT_K,
1278                 [BPF_JMP|BPF_JGT|BPF_X]  = BPF_S_JMP_JGT_X,
1279                 [BPF_JMP|BPF_JSET|BPF_K] = BPF_S_JMP_JSET_K,
1280                 [BPF_JMP|BPF_JSET|BPF_X] = BPF_S_JMP_JSET_X,
1281         };
1282         int pc;
1283         bool anc_found;
1284
1285         if (flen == 0 || flen > BPF_MAXINSNS)
1286                 return -EINVAL;
1287
1288         /* check the filter code now */
1289         for (pc = 0; pc < flen; pc++) {
1290                 struct sock_filter *ftest = &filter[pc];
1291                 u16 code = ftest->code;
1292
1293                 if (code >= ARRAY_SIZE(codes))
1294                         return -EINVAL;
1295                 code = codes[code];
1296                 if (!code)
1297                         return -EINVAL;
1298                 /* Some instructions need special checks */
1299                 switch (code) {
1300                 case BPF_S_ALU_DIV_K:
1301                 case BPF_S_ALU_MOD_K:
1302                         /* check for division by zero */
1303                         if (ftest->k == 0)
1304                                 return -EINVAL;
1305                         break;
1306                 case BPF_S_LD_MEM:
1307                 case BPF_S_LDX_MEM:
1308                 case BPF_S_ST:
1309                 case BPF_S_STX:
1310                         /* check for invalid memory addresses */
1311                         if (ftest->k >= BPF_MEMWORDS)
1312                                 return -EINVAL;
1313                         break;
1314                 case BPF_S_JMP_JA:
1315                         /*
1316                          * Note, the large ftest->k might cause loops.
1317                          * Compare this with conditional jumps below,
1318                          * where offsets are limited. --ANK (981016)
1319                          */
1320                         if (ftest->k >= (unsigned int)(flen-pc-1))
1321                                 return -EINVAL;
1322                         break;
1323                 case BPF_S_JMP_JEQ_K:
1324                 case BPF_S_JMP_JEQ_X:
1325                 case BPF_S_JMP_JGE_K:
1326                 case BPF_S_JMP_JGE_X:
1327                 case BPF_S_JMP_JGT_K:
1328                 case BPF_S_JMP_JGT_X:
1329                 case BPF_S_JMP_JSET_X:
1330                 case BPF_S_JMP_JSET_K:
1331                         /* for conditionals both must be safe */
1332                         if (pc + ftest->jt + 1 >= flen ||
1333                             pc + ftest->jf + 1 >= flen)
1334                                 return -EINVAL;
1335                         break;
1336                 case BPF_S_LD_W_ABS:
1337                 case BPF_S_LD_H_ABS:
1338                 case BPF_S_LD_B_ABS:
1339                         anc_found = false;
1340 #define ANCILLARY(CODE) case SKF_AD_OFF + SKF_AD_##CODE:        \
1341                                 code = BPF_S_ANC_##CODE;        \
1342                                 anc_found = true;               \
1343                                 break
1344                         switch (ftest->k) {
1345                         ANCILLARY(PROTOCOL);
1346                         ANCILLARY(PKTTYPE);
1347                         ANCILLARY(IFINDEX);
1348                         ANCILLARY(NLATTR);
1349                         ANCILLARY(NLATTR_NEST);
1350                         ANCILLARY(MARK);
1351                         ANCILLARY(QUEUE);
1352                         ANCILLARY(HATYPE);
1353                         ANCILLARY(RXHASH);
1354                         ANCILLARY(CPU);
1355                         ANCILLARY(ALU_XOR_X);
1356                         ANCILLARY(VLAN_TAG);
1357                         ANCILLARY(VLAN_TAG_PRESENT);
1358                         ANCILLARY(PAY_OFFSET);
1359                         }
1360
1361                         /* ancillary operation unknown or unsupported */
1362                         if (anc_found == false && ftest->k >= SKF_AD_OFF)
1363                                 return -EINVAL;
1364                 }
1365                 ftest->code = code;
1366         }
1367
1368         /* last instruction must be a RET code */
1369         switch (filter[flen - 1].code) {
1370         case BPF_S_RET_K:
1371         case BPF_S_RET_A:
1372                 return check_load_and_stores(filter, flen);
1373         }
1374         return -EINVAL;
1375 }
1376 EXPORT_SYMBOL(sk_chk_filter);
1377
1378 static int sk_store_orig_filter(struct sk_filter *fp,
1379                                 const struct sock_fprog *fprog)
1380 {
1381         unsigned int fsize = sk_filter_proglen(fprog);
1382         struct sock_fprog_kern *fkprog;
1383
1384         fp->orig_prog = kmalloc(sizeof(*fkprog), GFP_KERNEL);
1385         if (!fp->orig_prog)
1386                 return -ENOMEM;
1387
1388         fkprog = fp->orig_prog;
1389         fkprog->len = fprog->len;
1390         fkprog->filter = kmemdup(fp->insns, fsize, GFP_KERNEL);
1391         if (!fkprog->filter) {
1392                 kfree(fp->orig_prog);
1393                 return -ENOMEM;
1394         }
1395
1396         return 0;
1397 }
1398
1399 static void sk_release_orig_filter(struct sk_filter *fp)
1400 {
1401         struct sock_fprog_kern *fprog = fp->orig_prog;
1402
1403         if (fprog) {
1404                 kfree(fprog->filter);
1405                 kfree(fprog);
1406         }
1407 }
1408
1409 /**
1410  *      sk_filter_release_rcu - Release a socket filter by rcu_head
1411  *      @rcu: rcu_head that contains the sk_filter to free
1412  */
1413 static void sk_filter_release_rcu(struct rcu_head *rcu)
1414 {
1415         struct sk_filter *fp = container_of(rcu, struct sk_filter, rcu);
1416
1417         sk_release_orig_filter(fp);
1418         bpf_jit_free(fp);
1419 }
1420
1421 /**
1422  *      sk_filter_release - release a socket filter
1423  *      @fp: filter to remove
1424  *
1425  *      Remove a filter from a socket and release its resources.
1426  */
1427 static void sk_filter_release(struct sk_filter *fp)
1428 {
1429         if (atomic_dec_and_test(&fp->refcnt))
1430                 call_rcu(&fp->rcu, sk_filter_release_rcu);
1431 }
1432
1433 void sk_filter_uncharge(struct sock *sk, struct sk_filter *fp)
1434 {
1435         atomic_sub(sk_filter_size(fp->len), &sk->sk_omem_alloc);
1436         sk_filter_release(fp);
1437 }
1438
1439 void sk_filter_charge(struct sock *sk, struct sk_filter *fp)
1440 {
1441         atomic_inc(&fp->refcnt);
1442         atomic_add(sk_filter_size(fp->len), &sk->sk_omem_alloc);
1443 }
1444
1445 static struct sk_filter *__sk_migrate_realloc(struct sk_filter *fp,
1446                                               struct sock *sk,
1447                                               unsigned int len)
1448 {
1449         struct sk_filter *fp_new;
1450
1451         if (sk == NULL)
1452                 return krealloc(fp, len, GFP_KERNEL);
1453
1454         fp_new = sock_kmalloc(sk, len, GFP_KERNEL);
1455         if (fp_new) {
1456                 memcpy(fp_new, fp, sizeof(struct sk_filter));
1457                 /* As we're kepping orig_prog in fp_new along,
1458                  * we need to make sure we're not evicting it
1459                  * from the old fp.
1460                  */
1461                 fp->orig_prog = NULL;
1462                 sk_filter_uncharge(sk, fp);
1463         }
1464
1465         return fp_new;
1466 }
1467
1468 static struct sk_filter *__sk_migrate_filter(struct sk_filter *fp,
1469                                              struct sock *sk)
1470 {
1471         struct sock_filter *old_prog;
1472         struct sk_filter *old_fp;
1473         int i, err, new_len, old_len = fp->len;
1474
1475         /* We are free to overwrite insns et al right here as it
1476          * won't be used at this point in time anymore internally
1477          * after the migration to the internal BPF instruction
1478          * representation.
1479          */
1480         BUILD_BUG_ON(sizeof(struct sock_filter) !=
1481                      sizeof(struct sock_filter_int));
1482
1483         /* For now, we need to unfiddle BPF_S_* identifiers in place.
1484          * This can sooner or later on be subject to removal, e.g. when
1485          * JITs have been converted.
1486          */
1487         for (i = 0; i < fp->len; i++)
1488                 sk_decode_filter(&fp->insns[i], &fp->insns[i]);
1489
1490         /* Conversion cannot happen on overlapping memory areas,
1491          * so we need to keep the user BPF around until the 2nd
1492          * pass. At this time, the user BPF is stored in fp->insns.
1493          */
1494         old_prog = kmemdup(fp->insns, old_len * sizeof(struct sock_filter),
1495                            GFP_KERNEL);
1496         if (!old_prog) {
1497                 err = -ENOMEM;
1498                 goto out_err;
1499         }
1500
1501         /* 1st pass: calculate the new program length. */
1502         err = sk_convert_filter(old_prog, old_len, NULL, &new_len);
1503         if (err)
1504                 goto out_err_free;
1505
1506         /* Expand fp for appending the new filter representation. */
1507         old_fp = fp;
1508         fp = __sk_migrate_realloc(old_fp, sk, sk_filter_size(new_len));
1509         if (!fp) {
1510                 /* The old_fp is still around in case we couldn't
1511                  * allocate new memory, so uncharge on that one.
1512                  */
1513                 fp = old_fp;
1514                 err = -ENOMEM;
1515                 goto out_err_free;
1516         }
1517
1518         fp->bpf_func = sk_run_filter_int_skb;
1519         fp->len = new_len;
1520
1521         /* 2nd pass: remap sock_filter insns into sock_filter_int insns. */
1522         err = sk_convert_filter(old_prog, old_len, fp->insnsi, &new_len);
1523         if (err)
1524                 /* 2nd sk_convert_filter() can fail only if it fails
1525                  * to allocate memory, remapping must succeed. Note,
1526                  * that at this time old_fp has already been released
1527                  * by __sk_migrate_realloc().
1528                  */
1529                 goto out_err_free;
1530
1531         kfree(old_prog);
1532         return fp;
1533
1534 out_err_free:
1535         kfree(old_prog);
1536 out_err:
1537         /* Rollback filter setup. */
1538         if (sk != NULL)
1539                 sk_filter_uncharge(sk, fp);
1540         else
1541                 kfree(fp);
1542         return ERR_PTR(err);
1543 }
1544
1545 static struct sk_filter *__sk_prepare_filter(struct sk_filter *fp,
1546                                              struct sock *sk)
1547 {
1548         int err;
1549
1550         fp->bpf_func = NULL;
1551         fp->jited = 0;
1552
1553         err = sk_chk_filter(fp->insns, fp->len);
1554         if (err)
1555                 return ERR_PTR(err);
1556
1557         /* Probe if we can JIT compile the filter and if so, do
1558          * the compilation of the filter.
1559          */
1560         bpf_jit_compile(fp);
1561
1562         /* JIT compiler couldn't process this filter, so do the
1563          * internal BPF translation for the optimized interpreter.
1564          */
1565         if (!fp->jited)
1566                 fp = __sk_migrate_filter(fp, sk);
1567
1568         return fp;
1569 }
1570
1571 /**
1572  *      sk_unattached_filter_create - create an unattached filter
1573  *      @fprog: the filter program
1574  *      @pfp: the unattached filter that is created
1575  *
1576  * Create a filter independent of any socket. We first run some
1577  * sanity checks on it to make sure it does not explode on us later.
1578  * If an error occurs or there is insufficient memory for the filter
1579  * a negative errno code is returned. On success the return is zero.
1580  */
1581 int sk_unattached_filter_create(struct sk_filter **pfp,
1582                                 struct sock_fprog *fprog)
1583 {
1584         unsigned int fsize = sk_filter_proglen(fprog);
1585         struct sk_filter *fp;
1586
1587         /* Make sure new filter is there and in the right amounts. */
1588         if (fprog->filter == NULL)
1589                 return -EINVAL;
1590
1591         fp = kmalloc(sk_filter_size(fprog->len), GFP_KERNEL);
1592         if (!fp)
1593                 return -ENOMEM;
1594
1595         memcpy(fp->insns, fprog->filter, fsize);
1596
1597         atomic_set(&fp->refcnt, 1);
1598         fp->len = fprog->len;
1599         /* Since unattached filters are not copied back to user
1600          * space through sk_get_filter(), we do not need to hold
1601          * a copy here, and can spare us the work.
1602          */
1603         fp->orig_prog = NULL;
1604
1605         /* __sk_prepare_filter() already takes care of uncharging
1606          * memory in case something goes wrong.
1607          */
1608         fp = __sk_prepare_filter(fp, NULL);
1609         if (IS_ERR(fp))
1610                 return PTR_ERR(fp);
1611
1612         *pfp = fp;
1613         return 0;
1614 }
1615 EXPORT_SYMBOL_GPL(sk_unattached_filter_create);
1616
1617 void sk_unattached_filter_destroy(struct sk_filter *fp)
1618 {
1619         sk_filter_release(fp);
1620 }
1621 EXPORT_SYMBOL_GPL(sk_unattached_filter_destroy);
1622
1623 /**
1624  *      sk_attach_filter - attach a socket filter
1625  *      @fprog: the filter program
1626  *      @sk: the socket to use
1627  *
1628  * Attach the user's filter code. We first run some sanity checks on
1629  * it to make sure it does not explode on us later. If an error
1630  * occurs or there is insufficient memory for the filter a negative
1631  * errno code is returned. On success the return is zero.
1632  */
1633 int sk_attach_filter(struct sock_fprog *fprog, struct sock *sk)
1634 {
1635         struct sk_filter *fp, *old_fp;
1636         unsigned int fsize = sk_filter_proglen(fprog);
1637         unsigned int sk_fsize = sk_filter_size(fprog->len);
1638         int err;
1639
1640         if (sock_flag(sk, SOCK_FILTER_LOCKED))
1641                 return -EPERM;
1642
1643         /* Make sure new filter is there and in the right amounts. */
1644         if (fprog->filter == NULL)
1645                 return -EINVAL;
1646
1647         fp = sock_kmalloc(sk, sk_fsize, GFP_KERNEL);
1648         if (!fp)
1649                 return -ENOMEM;
1650
1651         if (copy_from_user(fp->insns, fprog->filter, fsize)) {
1652                 sock_kfree_s(sk, fp, sk_fsize);
1653                 return -EFAULT;
1654         }
1655
1656         atomic_set(&fp->refcnt, 1);
1657         fp->len = fprog->len;
1658
1659         err = sk_store_orig_filter(fp, fprog);
1660         if (err) {
1661                 sk_filter_uncharge(sk, fp);
1662                 return -ENOMEM;
1663         }
1664
1665         /* __sk_prepare_filter() already takes care of uncharging
1666          * memory in case something goes wrong.
1667          */
1668         fp = __sk_prepare_filter(fp, sk);
1669         if (IS_ERR(fp))
1670                 return PTR_ERR(fp);
1671
1672         old_fp = rcu_dereference_protected(sk->sk_filter,
1673                                            sock_owned_by_user(sk));
1674         rcu_assign_pointer(sk->sk_filter, fp);
1675
1676         if (old_fp)
1677                 sk_filter_uncharge(sk, old_fp);
1678
1679         return 0;
1680 }
1681 EXPORT_SYMBOL_GPL(sk_attach_filter);
1682
1683 int sk_detach_filter(struct sock *sk)
1684 {
1685         int ret = -ENOENT;
1686         struct sk_filter *filter;
1687
1688         if (sock_flag(sk, SOCK_FILTER_LOCKED))
1689                 return -EPERM;
1690
1691         filter = rcu_dereference_protected(sk->sk_filter,
1692                                            sock_owned_by_user(sk));
1693         if (filter) {
1694                 RCU_INIT_POINTER(sk->sk_filter, NULL);
1695                 sk_filter_uncharge(sk, filter);
1696                 ret = 0;
1697         }
1698
1699         return ret;
1700 }
1701 EXPORT_SYMBOL_GPL(sk_detach_filter);
1702
1703 void sk_decode_filter(struct sock_filter *filt, struct sock_filter *to)
1704 {
1705         static const u16 decodes[] = {
1706                 [BPF_S_ALU_ADD_K]       = BPF_ALU|BPF_ADD|BPF_K,
1707                 [BPF_S_ALU_ADD_X]       = BPF_ALU|BPF_ADD|BPF_X,
1708                 [BPF_S_ALU_SUB_K]       = BPF_ALU|BPF_SUB|BPF_K,
1709                 [BPF_S_ALU_SUB_X]       = BPF_ALU|BPF_SUB|BPF_X,
1710                 [BPF_S_ALU_MUL_K]       = BPF_ALU|BPF_MUL|BPF_K,
1711                 [BPF_S_ALU_MUL_X]       = BPF_ALU|BPF_MUL|BPF_X,
1712                 [BPF_S_ALU_DIV_X]       = BPF_ALU|BPF_DIV|BPF_X,
1713                 [BPF_S_ALU_MOD_K]       = BPF_ALU|BPF_MOD|BPF_K,
1714                 [BPF_S_ALU_MOD_X]       = BPF_ALU|BPF_MOD|BPF_X,
1715                 [BPF_S_ALU_AND_K]       = BPF_ALU|BPF_AND|BPF_K,
1716                 [BPF_S_ALU_AND_X]       = BPF_ALU|BPF_AND|BPF_X,
1717                 [BPF_S_ALU_OR_K]        = BPF_ALU|BPF_OR|BPF_K,
1718                 [BPF_S_ALU_OR_X]        = BPF_ALU|BPF_OR|BPF_X,
1719                 [BPF_S_ALU_XOR_K]       = BPF_ALU|BPF_XOR|BPF_K,
1720                 [BPF_S_ALU_XOR_X]       = BPF_ALU|BPF_XOR|BPF_X,
1721                 [BPF_S_ALU_LSH_K]       = BPF_ALU|BPF_LSH|BPF_K,
1722                 [BPF_S_ALU_LSH_X]       = BPF_ALU|BPF_LSH|BPF_X,
1723                 [BPF_S_ALU_RSH_K]       = BPF_ALU|BPF_RSH|BPF_K,
1724                 [BPF_S_ALU_RSH_X]       = BPF_ALU|BPF_RSH|BPF_X,
1725                 [BPF_S_ALU_NEG]         = BPF_ALU|BPF_NEG,
1726                 [BPF_S_LD_W_ABS]        = BPF_LD|BPF_W|BPF_ABS,
1727                 [BPF_S_LD_H_ABS]        = BPF_LD|BPF_H|BPF_ABS,
1728                 [BPF_S_LD_B_ABS]        = BPF_LD|BPF_B|BPF_ABS,
1729                 [BPF_S_ANC_PROTOCOL]    = BPF_LD|BPF_B|BPF_ABS,
1730                 [BPF_S_ANC_PKTTYPE]     = BPF_LD|BPF_B|BPF_ABS,
1731                 [BPF_S_ANC_IFINDEX]     = BPF_LD|BPF_B|BPF_ABS,
1732                 [BPF_S_ANC_NLATTR]      = BPF_LD|BPF_B|BPF_ABS,
1733                 [BPF_S_ANC_NLATTR_NEST] = BPF_LD|BPF_B|BPF_ABS,
1734                 [BPF_S_ANC_MARK]        = BPF_LD|BPF_B|BPF_ABS,
1735                 [BPF_S_ANC_QUEUE]       = BPF_LD|BPF_B|BPF_ABS,
1736                 [BPF_S_ANC_HATYPE]      = BPF_LD|BPF_B|BPF_ABS,
1737                 [BPF_S_ANC_RXHASH]      = BPF_LD|BPF_B|BPF_ABS,
1738                 [BPF_S_ANC_CPU]         = BPF_LD|BPF_B|BPF_ABS,
1739                 [BPF_S_ANC_ALU_XOR_X]   = BPF_LD|BPF_B|BPF_ABS,
1740                 [BPF_S_ANC_SECCOMP_LD_W] = BPF_LD|BPF_B|BPF_ABS,
1741                 [BPF_S_ANC_VLAN_TAG]    = BPF_LD|BPF_B|BPF_ABS,
1742                 [BPF_S_ANC_VLAN_TAG_PRESENT] = BPF_LD|BPF_B|BPF_ABS,
1743                 [BPF_S_ANC_PAY_OFFSET]  = BPF_LD|BPF_B|BPF_ABS,
1744                 [BPF_S_LD_W_LEN]        = BPF_LD|BPF_W|BPF_LEN,
1745                 [BPF_S_LD_W_IND]        = BPF_LD|BPF_W|BPF_IND,
1746                 [BPF_S_LD_H_IND]        = BPF_LD|BPF_H|BPF_IND,
1747                 [BPF_S_LD_B_IND]        = BPF_LD|BPF_B|BPF_IND,
1748                 [BPF_S_LD_IMM]          = BPF_LD|BPF_IMM,
1749                 [BPF_S_LDX_W_LEN]       = BPF_LDX|BPF_W|BPF_LEN,
1750                 [BPF_S_LDX_B_MSH]       = BPF_LDX|BPF_B|BPF_MSH,
1751                 [BPF_S_LDX_IMM]         = BPF_LDX|BPF_IMM,
1752                 [BPF_S_MISC_TAX]        = BPF_MISC|BPF_TAX,
1753                 [BPF_S_MISC_TXA]        = BPF_MISC|BPF_TXA,
1754                 [BPF_S_RET_K]           = BPF_RET|BPF_K,
1755                 [BPF_S_RET_A]           = BPF_RET|BPF_A,
1756                 [BPF_S_ALU_DIV_K]       = BPF_ALU|BPF_DIV|BPF_K,
1757                 [BPF_S_LD_MEM]          = BPF_LD|BPF_MEM,
1758                 [BPF_S_LDX_MEM]         = BPF_LDX|BPF_MEM,
1759                 [BPF_S_ST]              = BPF_ST,
1760                 [BPF_S_STX]             = BPF_STX,
1761                 [BPF_S_JMP_JA]          = BPF_JMP|BPF_JA,
1762                 [BPF_S_JMP_JEQ_K]       = BPF_JMP|BPF_JEQ|BPF_K,
1763                 [BPF_S_JMP_JEQ_X]       = BPF_JMP|BPF_JEQ|BPF_X,
1764                 [BPF_S_JMP_JGE_K]       = BPF_JMP|BPF_JGE|BPF_K,
1765                 [BPF_S_JMP_JGE_X]       = BPF_JMP|BPF_JGE|BPF_X,
1766                 [BPF_S_JMP_JGT_K]       = BPF_JMP|BPF_JGT|BPF_K,
1767                 [BPF_S_JMP_JGT_X]       = BPF_JMP|BPF_JGT|BPF_X,
1768                 [BPF_S_JMP_JSET_K]      = BPF_JMP|BPF_JSET|BPF_K,
1769                 [BPF_S_JMP_JSET_X]      = BPF_JMP|BPF_JSET|BPF_X,
1770         };
1771         u16 code;
1772
1773         code = filt->code;
1774
1775         to->code = decodes[code];
1776         to->jt = filt->jt;
1777         to->jf = filt->jf;
1778         to->k = filt->k;
1779 }
1780
1781 int sk_get_filter(struct sock *sk, struct sock_filter __user *ubuf,
1782                   unsigned int len)
1783 {
1784         struct sock_fprog_kern *fprog;
1785         struct sk_filter *filter;
1786         int ret = 0;
1787
1788         lock_sock(sk);
1789         filter = rcu_dereference_protected(sk->sk_filter,
1790                                            sock_owned_by_user(sk));
1791         if (!filter)
1792                 goto out;
1793
1794         /* We're copying the filter that has been originally attached,
1795          * so no conversion/decode needed anymore.
1796          */
1797         fprog = filter->orig_prog;
1798
1799         ret = fprog->len;
1800         if (!len)
1801                 /* User space only enquires number of filter blocks. */
1802                 goto out;
1803
1804         ret = -EINVAL;
1805         if (len < fprog->len)
1806                 goto out;
1807
1808         ret = -EFAULT;
1809         if (copy_to_user(ubuf, fprog->filter, sk_filter_proglen(fprog)))
1810                 goto out;
1811
1812         /* Instead of bytes, the API requests to return the number
1813          * of filter blocks.
1814          */
1815         ret = fprog->len;
1816 out:
1817         release_sock(sk);
1818         return ret;
1819 }