net/ipv6/ila/ila_xlat.c

   1 #include <linux/jhash.h>
   2 #include <linux/netfilter.h>
   3 #include <linux/rcupdate.h>
   4 #include <linux/rhashtable.h>
   5 #include <linux/vmalloc.h>
   6 #include <net/genetlink.h>
   7 #include <net/ila.h>
   8 #include <net/netns/generic.h>
   9 #include <uapi/linux/genetlink.h>
  10 #include "ila.h"
  11
  12 struct ila_xlat_params {
  13         struct ila_params ip;
  14         __be64 identifier;
  15         int ifindex;
  16         unsigned int dir;
  17 };
  18
  19 struct ila_map {
  20         struct ila_xlat_params p;
  21         struct rhash_head node;
  22         struct ila_map __rcu *next;
  23         struct rcu_head rcu;
  24 };
  25
  26 static unsigned int ila_net_id;
  27
  28 struct ila_net {
  29         struct rhashtable rhash_table;
  30         spinlock_t *locks; /* Bucket locks for entry manipulation */
  31         unsigned int locks_mask;
  32         bool hooks_registered;
  33 };
  34
  35 #define LOCKS_PER_CPU 10
  36
  37 static int alloc_ila_locks(struct ila_net *ilan)
  38 {
  39         unsigned int i, size;
  40         unsigned int nr_pcpus = num_possible_cpus();
  41
  42         nr_pcpus = min_t(unsigned int, nr_pcpus, 32UL);
  43         size = roundup_pow_of_two(nr_pcpus * LOCKS_PER_CPU);
  44
  45         if (sizeof(spinlock_t) != 0) {
  46 #ifdef CONFIG_NUMA
  47                 if (size * sizeof(spinlock_t) > PAGE_SIZE)
  48                         ilan->locks = vmalloc(size * sizeof(spinlock_t));
  49                 else
  50 #endif
  51                 ilan->locks = kmalloc_array(size, sizeof(spinlock_t),
  52                                             GFP_KERNEL);
  53                 if (!ilan->locks)
  54                         return -ENOMEM;
  55                 for (i = 0; i < size; i++)
  56                         spin_lock_init(&ilan->locks[i]);
  57         }
  58         ilan->locks_mask = size - 1;
  59
  60         return 0;
  61 }
  62
  63 static u32 hashrnd __read_mostly;
  64 static __always_inline void __ila_hash_secret_init(void)
  65 {
  66         net_get_random_once(&hashrnd, sizeof(hashrnd));
  67 }
  68
  69 static inline u32 ila_identifier_hash(__be64 identifier)
  70 {
  71         u32 *v = (u32 *)&identifier;
  72
  73         return jhash_2words(v[0], v[1], hashrnd);
  74 }
  75
  76 static inline spinlock_t *ila_get_lock(struct ila_net *ilan, __be64 identifier)
  77 {
  78         return &ilan->locks[ila_identifier_hash(identifier) & ilan->locks_mask];
  79 }
  80
  81 static inline int ila_cmp_wildcards(struct ila_map *ila, __be64 loc,
  82                                     int ifindex, unsigned int dir)
  83 {
  84         return (ila->p.ip.locator_match && ila->p.ip.locator_match != loc) ||
  85                (ila->p.ifindex && ila->p.ifindex != ifindex) ||
  86                !(ila->p.dir & dir);
  87 }
  88
  89 static inline int ila_cmp_params(struct ila_map *ila, struct ila_xlat_params *p)
  90 {
  91         return (ila->p.ip.locator_match != p->ip.locator_match) ||
  92                (ila->p.ifindex != p->ifindex) ||
  93                (ila->p.dir != p->dir);
  94 }
  95
  96 static int ila_cmpfn(struct rhashtable_compare_arg *arg,
  97                      const void *obj)
  98 {
  99         const struct ila_map *ila = obj;
 100
 101         return (ila->p.identifier != *(__be64 *)arg->key);
 102 }
 103
 104 static inline int ila_order(struct ila_map *ila)
 105 {
 106         int score = 0;
 107
 108         if (ila->p.ip.locator_match)
 109                 score += 1 << 0;
 110
 111         if (ila->p.ifindex)
 112                 score += 1 << 1;
 113
 114         return score;
 115 }
 116
 117 static const struct rhashtable_params rht_params = {
 118         .nelem_hint = 1024,
 119         .head_offset = offsetof(struct ila_map, node),
 120         .key_offset = offsetof(struct ila_map, p.identifier),
 121         .key_len = sizeof(u64), /* identifier */
 122         .max_size = 1048576,
 123         .min_size = 256,
 124         .automatic_shrinking = true,
 125         .obj_cmpfn = ila_cmpfn,
 126 };
 127
 128 static struct genl_family ila_nl_family = {
 129         .id             = GENL_ID_GENERATE,
 130         .hdrsize        = 0,
 131         .name           = ILA_GENL_NAME,
 132         .version        = ILA_GENL_VERSION,
 133         .maxattr        = ILA_ATTR_MAX,
 134         .netnsok        = true,
 135         .parallel_ops   = true,
 136 };
 137
 138 static struct nla_policy ila_nl_policy[ILA_ATTR_MAX + 1] = {
 139         [ILA_ATTR_IDENTIFIER] = { .type = NLA_U64, },
 140         [ILA_ATTR_LOCATOR] = { .type = NLA_U64, },
 141         [ILA_ATTR_LOCATOR_MATCH] = { .type = NLA_U64, },
 142         [ILA_ATTR_IFINDEX] = { .type = NLA_U32, },
 143         [ILA_ATTR_DIR] = { .type = NLA_U32, },
 144 };
 145
 146 static int parse_nl_config(struct genl_info *info,
 147                            struct ila_xlat_params *p)
 148 {
 149         memset(p, 0, sizeof(*p));
 150
 151         if (info->attrs[ILA_ATTR_IDENTIFIER])
 152                 p->identifier = (__force __be64)nla_get_u64(
 153                         info->attrs[ILA_ATTR_IDENTIFIER]);
 154
 155         if (info->attrs[ILA_ATTR_LOCATOR])
 156                 p->ip.locator = (__force __be64)nla_get_u64(
 157                         info->attrs[ILA_ATTR_LOCATOR]);
 158
 159         if (info->attrs[ILA_ATTR_LOCATOR_MATCH])
 160                 p->ip.locator_match = (__force __be64)nla_get_u64(
 161                         info->attrs[ILA_ATTR_LOCATOR_MATCH]);
 162
 163         if (info->attrs[ILA_ATTR_IFINDEX])
 164                 p->ifindex = nla_get_s32(info->attrs[ILA_ATTR_IFINDEX]);
 165
 166         if (info->attrs[ILA_ATTR_DIR])
 167                 p->dir = nla_get_u32(info->attrs[ILA_ATTR_DIR]);
 168
 169         return 0;
 170 }
 171
 172 /* Must be called with rcu readlock */
 173 static inline struct ila_map *ila_lookup_wildcards(__be64 id, __be64 loc,
 174                                                    int ifindex,
 175                                                    unsigned int dir,
 176                                                    struct ila_net *ilan)
 177 {
 178         struct ila_map *ila;
 179
 180         ila = rhashtable_lookup_fast(&ilan->rhash_table, &id, rht_params);
 181         while (ila) {
 182                 if (!ila_cmp_wildcards(ila, loc, ifindex, dir))
 183                         return ila;
 184                 ila = rcu_access_pointer(ila->next);
 185         }
 186
 187         return NULL;
 188 }
 189
 190 /* Must be called with rcu readlock */
 191 static inline struct ila_map *ila_lookup_by_params(struct ila_xlat_params *p,
 192                                                    struct ila_net *ilan)
 193 {
 194         struct ila_map *ila;
 195
 196         ila = rhashtable_lookup_fast(&ilan->rhash_table, &p->identifier,
 197                                      rht_params);
 198         while (ila) {
 199                 if (!ila_cmp_params(ila, p))
 200                         return ila;
 201                 ila = rcu_access_pointer(ila->next);
 202         }
 203
 204         return NULL;
 205 }
 206
 207 static inline void ila_release(struct ila_map *ila)
 208 {
 209         kfree_rcu(ila, rcu);
 210 }
 211
 212 static void ila_free_cb(void *ptr, void *arg)
 213 {
 214         struct ila_map *ila = (struct ila_map *)ptr, *next;
 215
 216         /* Assume rcu_readlock held */
 217         while (ila) {
 218                 next = rcu_access_pointer(ila->next);
 219                 ila_release(ila);
 220                 ila = next;
 221         }
 222 }
 223
 224 static int ila_xlat_addr(struct sk_buff *skb, int dir);
 225
 226 static unsigned int
 227 ila_nf_input(void *priv,
 228              struct sk_buff *skb,
 229              const struct nf_hook_state *state)
 230 {
 231         ila_xlat_addr(skb, ILA_DIR_IN);
 232         return NF_ACCEPT;
 233 }
 234
 235 static struct nf_hook_ops ila_nf_hook_ops[] __read_mostly = {
 236         {
 237                 .hook = ila_nf_input,
 238                 .pf = NFPROTO_IPV6,
 239                 .hooknum = NF_INET_PRE_ROUTING,
 240                 .priority = -1,
 241         },
 242 };
 243
 244 static int ila_add_mapping(struct net *net, struct ila_xlat_params *p)
 245 {
 246         struct ila_net *ilan = net_generic(net, ila_net_id);
 247         struct ila_map *ila, *head;
 248         spinlock_t *lock = ila_get_lock(ilan, p->identifier);
 249         int err = 0, order;
 250
 251         if (!ilan->hooks_registered) {
 252                 /* We defer registering net hooks in the namespace until the
 253                  * first mapping is added.
 254                  */
 255                 err = nf_register_net_hooks(net, ila_nf_hook_ops,
 256                                             ARRAY_SIZE(ila_nf_hook_ops));
 257                 if (err)
 258                         return err;
 259
 260                 ilan->hooks_registered = true;
 261         }
 262
 263         ila = kzalloc(sizeof(*ila), GFP_KERNEL);
 264         if (!ila)
 265                 return -ENOMEM;
 266
 267         ila->p = *p;
 268
 269         if (p->ip.locator_match) {
 270                 /* Precompute checksum difference for translation since we
 271                  * know both the old identifier and the new one.
 272                  */
 273                 ila->p.ip.csum_diff = compute_csum_diff8(
 274                         (__be32 *)&p->ip.locator_match,
 275                         (__be32 *)&p->ip.locator);
 276         }
 277
 278         order = ila_order(ila);
 279
 280         spin_lock(lock);
 281
 282         head = rhashtable_lookup_fast(&ilan->rhash_table, &p->identifier,
 283                                       rht_params);
 284         if (!head) {
 285                 /* New entry for the rhash_table */
 286                 err = rhashtable_lookup_insert_fast(&ilan->rhash_table,
 287                                                     &ila->node, rht_params);
 288         } else {
 289                 struct ila_map *tila = head, *prev = NULL;
 290
 291                 do {
 292                         if (!ila_cmp_params(tila, p)) {
 293                                 err = -EEXIST;
 294                                 goto out;
 295                         }
 296
 297                         if (order > ila_order(tila))
 298                                 break;
 299
 300                         prev = tila;
 301                         tila = rcu_dereference_protected(tila->next,
 302                                 lockdep_is_held(lock));
 303                 } while (tila);
 304
 305                 if (prev) {
 306                         /* Insert in sub list of head */
 307                         RCU_INIT_POINTER(ila->next, tila);
 308                         rcu_assign_pointer(prev->next, ila);
 309                 } else {
 310                         /* Make this ila new head */
 311                         RCU_INIT_POINTER(ila->next, head);
 312                         err = rhashtable_replace_fast(&ilan->rhash_table,
 313                                                       &head->node,
 314                                                       &ila->node, rht_params);
 315                         if (err)
 316                                 goto out;
 317                 }
 318         }
 319
 320 out:
 321         spin_unlock(lock);
 322
 323         if (err)
 324                 kfree(ila);
 325
 326         return err;
 327 }
 328
 329 static int ila_del_mapping(struct net *net, struct ila_xlat_params *p)
 330 {
 331         struct ila_net *ilan = net_generic(net, ila_net_id);
 332         struct ila_map *ila, *head, *prev;
 333         spinlock_t *lock = ila_get_lock(ilan, p->identifier);
 334         int err = -ENOENT;
 335
 336         spin_lock(lock);
 337
 338         head = rhashtable_lookup_fast(&ilan->rhash_table,
 339                                       &p->identifier, rht_params);
 340         ila = head;
 341
 342         prev = NULL;
 343
 344         while (ila) {
 345                 if (ila_cmp_params(ila, p)) {
 346                         prev = ila;
 347                         ila = rcu_dereference_protected(ila->next,
 348                                                         lockdep_is_held(lock));
 349                         continue;
 350                 }
 351
 352                 err = 0;
 353
 354                 if (prev) {
 355                         /* Not head, just delete from list */
 356                         rcu_assign_pointer(prev->next, ila->next);
 357                 } else {
 358                         /* It is the head. If there is something in the
 359                          * sublist we need to make a new head.
 360                          */
 361                         head = rcu_dereference_protected(ila->next,
 362                                                          lockdep_is_held(lock));
 363                         if (head) {
 364                                 /* Put first entry in the sublist into the
 365                                  * table
 366                                  */
 367                                 err = rhashtable_replace_fast(
 368                                         &ilan->rhash_table, &ila->node,
 369                                         &head->node, rht_params);
 370                                 if (err)
 371                                         goto out;
 372                         } else {
 373                                 /* Entry no longer used */
 374                                 err = rhashtable_remove_fast(&ilan->rhash_table,
 375                                                              &ila->node,
 376                                                              rht_params);
 377                         }
 378                 }
 379
 380                 ila_release(ila);
 381
 382                 break;
 383         }
 384
 385 out:
 386         spin_unlock(lock);
 387
 388         return err;
 389 }
 390
 391 static int ila_nl_cmd_add_mapping(struct sk_buff *skb, struct genl_info *info)
 392 {
 393         struct net *net = genl_info_net(info);
 394         struct ila_xlat_params p;
 395         int err;
 396
 397         err = parse_nl_config(info, &p);
 398         if (err)
 399                 return err;
 400
 401         return ila_add_mapping(net, &p);
 402 }
 403
 404 static int ila_nl_cmd_del_mapping(struct sk_buff *skb, struct genl_info *info)
 405 {
 406         struct net *net = genl_info_net(info);
 407         struct ila_xlat_params p;
 408         int err;
 409
 410         err = parse_nl_config(info, &p);
 411         if (err)
 412                 return err;
 413
 414         ila_del_mapping(net, &p);
 415
 416         return 0;
 417 }
 418
 419 static int ila_fill_info(struct ila_map *ila, struct sk_buff *msg)
 420 {
 421         if (nla_put_u64(msg, ILA_ATTR_IDENTIFIER,
 422                         (__force u64)ila->p.identifier) ||
 423             nla_put_u64(msg, ILA_ATTR_LOCATOR,
 424                         (__force u64)ila->p.ip.locator) ||
 425             nla_put_u64(msg, ILA_ATTR_LOCATOR_MATCH,
 426                         (__force u64)ila->p.ip.locator_match) ||
 427             nla_put_s32(msg, ILA_ATTR_IFINDEX, ila->p.ifindex) ||
 428             nla_put_u32(msg, ILA_ATTR_DIR, ila->p.dir))
 429                 return -1;
 430
 431         return 0;
 432 }
 433
 434 static int ila_dump_info(struct ila_map *ila,
 435                          u32 portid, u32 seq, u32 flags,
 436                          struct sk_buff *skb, u8 cmd)
 437 {
 438         void *hdr;
 439
 440         hdr = genlmsg_put(skb, portid, seq, &ila_nl_family, flags, cmd);
 441         if (!hdr)
 442                 return -ENOMEM;
 443
 444         if (ila_fill_info(ila, skb) < 0)
 445                 goto nla_put_failure;
 446
 447         genlmsg_end(skb, hdr);
 448         return 0;
 449
 450 nla_put_failure:
 451         genlmsg_cancel(skb, hdr);
 452         return -EMSGSIZE;
 453 }
 454
 455 static int ila_nl_cmd_get_mapping(struct sk_buff *skb, struct genl_info *info)
 456 {
 457         struct net *net = genl_info_net(info);
 458         struct ila_net *ilan = net_generic(net, ila_net_id);
 459         struct sk_buff *msg;
 460         struct ila_xlat_params p;
 461         struct ila_map *ila;
 462         int ret;
 463
 464         ret = parse_nl_config(info, &p);
 465         if (ret)
 466                 return ret;
 467
 468         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
 469         if (!msg)
 470                 return -ENOMEM;
 471
 472         rcu_read_lock();
 473
 474         ila = ila_lookup_by_params(&p, ilan);
 475         if (ila) {
 476                 ret = ila_dump_info(ila,
 477                                     info->snd_portid,
 478                                     info->snd_seq, 0, msg,
 479                                     info->genlhdr->cmd);
 480         }
 481
 482         rcu_read_unlock();
 483
 484         if (ret < 0)
 485                 goto out_free;
 486
 487         return genlmsg_reply(msg, info);
 488
 489 out_free:
 490         nlmsg_free(msg);
 491         return ret;
 492 }
 493
 494 struct ila_dump_iter {
 495         struct rhashtable_iter rhiter;
 496 };
 497
 498 static int ila_nl_dump_start(struct netlink_callback *cb)
 499 {
 500         struct net *net = sock_net(cb->skb->sk);
 501         struct ila_net *ilan = net_generic(net, ila_net_id);
 502         struct ila_dump_iter *iter = (struct ila_dump_iter *)cb->args;
 503
 504         return rhashtable_walk_init(&ilan->rhash_table, &iter->rhiter);
 505 }
 506
 507 static int ila_nl_dump_done(struct netlink_callback *cb)
 508 {
 509         struct ila_dump_iter *iter = (struct ila_dump_iter *)cb->args;
 510
 511         rhashtable_walk_exit(&iter->rhiter);
 512
 513         return 0;
 514 }
 515
 516 static int ila_nl_dump(struct sk_buff *skb, struct netlink_callback *cb)
 517 {
 518         struct ila_dump_iter *iter = (struct ila_dump_iter *)cb->args;
 519         struct rhashtable_iter *rhiter = &iter->rhiter;
 520         struct ila_map *ila;
 521         int ret;
 522
 523         ret = rhashtable_walk_start(rhiter);
 524         if (ret && ret != -EAGAIN)
 525                 goto done;
 526
 527         for (;;) {
 528                 ila = rhashtable_walk_next(rhiter);
 529
 530                 if (IS_ERR(ila)) {
 531                         if (PTR_ERR(ila) == -EAGAIN)
 532                                 continue;
 533                         ret = PTR_ERR(ila);
 534                         goto done;
 535                 } else if (!ila) {
 536                         break;
 537                 }
 538
 539                 while (ila) {
 540                         ret =  ila_dump_info(ila, NETLINK_CB(cb->skb).portid,
 541                                              cb->nlh->nlmsg_seq, NLM_F_MULTI,
 542                                              skb, ILA_CMD_GET);
 543                         if (ret)
 544                                 goto done;
 545
 546                         ila = rcu_access_pointer(ila->next);
 547                 }
 548         }
 549
 550         ret = skb->len;
 551
 552 done:
 553         rhashtable_walk_stop(rhiter);
 554         return ret;
 555 }
 556
 557 static const struct genl_ops ila_nl_ops[] = {
 558         {
 559                 .cmd = ILA_CMD_ADD,
 560                 .doit = ila_nl_cmd_add_mapping,
 561                 .policy = ila_nl_policy,
 562                 .flags = GENL_ADMIN_PERM,
 563         },
 564         {
 565                 .cmd = ILA_CMD_DEL,
 566                 .doit = ila_nl_cmd_del_mapping,
 567                 .policy = ila_nl_policy,
 568                 .flags = GENL_ADMIN_PERM,
 569         },
 570         {
 571                 .cmd = ILA_CMD_GET,
 572                 .doit = ila_nl_cmd_get_mapping,
 573                 .start = ila_nl_dump_start,
 574                 .dumpit = ila_nl_dump,
 575                 .done = ila_nl_dump_done,
 576                 .policy = ila_nl_policy,
 577         },
 578 };
 579
 580 #define ILA_HASH_TABLE_SIZE 1024
 581
 582 static __net_init int ila_init_net(struct net *net)
 583 {
 584         int err;
 585         struct ila_net *ilan = net_generic(net, ila_net_id);
 586
 587         err = alloc_ila_locks(ilan);
 588         if (err)
 589                 return err;
 590
 591         rhashtable_init(&ilan->rhash_table, &rht_params);
 592
 593         return 0;
 594 }
 595
 596 static __net_exit void ila_exit_net(struct net *net)
 597 {
 598         struct ila_net *ilan = net_generic(net, ila_net_id);
 599
 600         rhashtable_free_and_destroy(&ilan->rhash_table, ila_free_cb, NULL);
 601
 602         kvfree(ilan->locks);
 603
 604         if (ilan->hooks_registered)
 605                 nf_unregister_net_hooks(net, ila_nf_hook_ops,
 606                                         ARRAY_SIZE(ila_nf_hook_ops));
 607 }
 608
 609 static struct pernet_operations ila_net_ops = {
 610         .init = ila_init_net,
 611         .exit = ila_exit_net,
 612         .id   = &ila_net_id,
 613         .size = sizeof(struct ila_net),
 614 };
 615
 616 static int ila_xlat_addr(struct sk_buff *skb, int dir)
 617 {
 618         struct ila_map *ila;
 619         struct ipv6hdr *ip6h = ipv6_hdr(skb);
 620         struct net *net = dev_net(skb->dev);
 621         struct ila_net *ilan = net_generic(net, ila_net_id);
 622         __be64 identifier, locator_match;
 623         size_t nhoff;
 624
 625         /* Assumes skb contains a valid IPv6 header that is pulled */
 626
 627         identifier = *(__be64 *)&ip6h->daddr.in6_u.u6_addr8[8];
 628         locator_match = *(__be64 *)&ip6h->daddr.in6_u.u6_addr8[0];
 629         nhoff = sizeof(struct ipv6hdr);
 630
 631         rcu_read_lock();
 632
 633         ila = ila_lookup_wildcards(identifier, locator_match,
 634                                    skb->dev->ifindex, dir, ilan);
 635         if (ila)
 636                 update_ipv6_locator(skb, &ila->p.ip);
 637
 638         rcu_read_unlock();
 639
 640         return 0;
 641 }
 642
 643 int ila_xlat_incoming(struct sk_buff *skb)
 644 {
 645         return ila_xlat_addr(skb, ILA_DIR_IN);
 646 }
 647 EXPORT_SYMBOL(ila_xlat_incoming);
 648
 649 int ila_xlat_outgoing(struct sk_buff *skb)
 650 {
 651         return ila_xlat_addr(skb, ILA_DIR_OUT);
 652 }
 653 EXPORT_SYMBOL(ila_xlat_outgoing);
 654
 655 int ila_xlat_init(void)
 656 {
 657         int ret;
 658
 659         ret = register_pernet_device(&ila_net_ops);
 660         if (ret)
 661                 goto exit;
 662
 663         ret = genl_register_family_with_ops(&ila_nl_family,
 664                                             ila_nl_ops);
 665         if (ret < 0)
 666                 goto unregister;
 667
 668         return 0;
 669
 670 unregister:
 671         unregister_pernet_device(&ila_net_ops);
 672 exit:
 673         return ret;
 674 }
 675
 676 void ila_xlat_fini(void)
 677 {
 678         genl_unregister_family(&ila_nl_family);
 679         unregister_pernet_device(&ila_net_ops);
 680 }