net/netfilter/nf_conntrack_core.c

   1 /* Connection state tracking for netfilter.  This is separated from,
   2    but required by, the NAT layer; it can also be used by an iptables
   3    extension. */
   4
   5 /* (C) 1999-2001 Paul `Rusty' Russell
   6  * (C) 2002-2006 Netfilter Core Team <coreteam@netfilter.org>
   7  * (C) 2003,2004 USAGI/WIDE Project <http://www.linux-ipv6.org>
   8  * (C) 2005-2012 Patrick McHardy <kaber@trash.net>
   9  *
  10  * This program is free software; you can redistribute it and/or modify
  11  * it under the terms of the GNU General Public License version 2 as
  12  * published by the Free Software Foundation.
  13  */
  14
  15 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  16
  17 #include <linux/types.h>
  18 #include <linux/netfilter.h>
  19 #include <linux/module.h>
  20 #include <linux/sched.h>
  21 #include <linux/skbuff.h>
  22 #include <linux/proc_fs.h>
  23 #include <linux/vmalloc.h>
  24 #include <linux/stddef.h>
  25 #include <linux/slab.h>
  26 #include <linux/random.h>
  27 #include <linux/jhash.h>
  28 #include <linux/err.h>
  29 #include <linux/percpu.h>
  30 #include <linux/moduleparam.h>
  31 #include <linux/notifier.h>
  32 #include <linux/kernel.h>
  33 #include <linux/netdevice.h>
  34 #include <linux/socket.h>
  35 #include <linux/mm.h>
  36 #include <linux/nsproxy.h>
  37 #include <linux/rculist_nulls.h>
  38
  39 #include <net/netfilter/nf_conntrack.h>
  40 #include <net/netfilter/nf_conntrack_l3proto.h>
  41 #include <net/netfilter/nf_conntrack_l4proto.h>
  42 #include <net/netfilter/nf_conntrack_expect.h>
  43 #include <net/netfilter/nf_conntrack_helper.h>
  44 #include <net/netfilter/nf_conntrack_seqadj.h>
  45 #include <net/netfilter/nf_conntrack_core.h>
  46 #include <net/netfilter/nf_conntrack_extend.h>
  47 #include <net/netfilter/nf_conntrack_acct.h>
  48 #include <net/netfilter/nf_conntrack_ecache.h>
  49 #include <net/netfilter/nf_conntrack_zones.h>
  50 #include <net/netfilter/nf_conntrack_timestamp.h>
  51 #include <net/netfilter/nf_conntrack_timeout.h>
  52 #include <net/netfilter/nf_conntrack_labels.h>
  53 #include <net/netfilter/nf_conntrack_synproxy.h>
  54 #include <net/netfilter/nf_nat.h>
  55 #include <net/netfilter/nf_nat_core.h>
  56 #include <net/netfilter/nf_nat_helper.h>
  57 #include <net/netns/hash.h>
  58
  59 #define NF_CONNTRACK_VERSION    "0.5.0"
  60
  61 int (*nfnetlink_parse_nat_setup_hook)(struct nf_conn *ct,
  62                                       enum nf_nat_manip_type manip,
  63                                       const struct nlattr *attr) __read_mostly;
  64 EXPORT_SYMBOL_GPL(nfnetlink_parse_nat_setup_hook);
  65
  66 __cacheline_aligned_in_smp spinlock_t nf_conntrack_locks[CONNTRACK_LOCKS];
  67 EXPORT_SYMBOL_GPL(nf_conntrack_locks);
  68
  69 __cacheline_aligned_in_smp DEFINE_SPINLOCK(nf_conntrack_expect_lock);
  70 EXPORT_SYMBOL_GPL(nf_conntrack_expect_lock);
  71
  72 struct hlist_nulls_head *nf_conntrack_hash __read_mostly;
  73 EXPORT_SYMBOL_GPL(nf_conntrack_hash);
  74
  75 static __read_mostly struct kmem_cache *nf_conntrack_cachep;
  76 static __read_mostly spinlock_t nf_conntrack_locks_all_lock;
  77 static __read_mostly DEFINE_SPINLOCK(nf_conntrack_locks_all_lock);
  78 static __read_mostly bool nf_conntrack_locks_all;
  79
  80 void nf_conntrack_lock(spinlock_t *lock) __acquires(lock)
  81 {
  82         spin_lock(lock);
  83         while (unlikely(nf_conntrack_locks_all)) {
  84                 spin_unlock(lock);
  85
  86                 /*
  87                  * Order the 'nf_conntrack_locks_all' load vs. the
  88                  * spin_unlock_wait() loads below, to ensure
  89                  * that 'nf_conntrack_locks_all_lock' is indeed held:
  90                  */
  91                 smp_rmb(); /* spin_lock(&nf_conntrack_locks_all_lock) */
  92                 spin_unlock_wait(&nf_conntrack_locks_all_lock);
  93                 spin_lock(lock);
  94         }
  95 }
  96 EXPORT_SYMBOL_GPL(nf_conntrack_lock);
  97
  98 static void nf_conntrack_double_unlock(unsigned int h1, unsigned int h2)
  99 {
 100         h1 %= CONNTRACK_LOCKS;
 101         h2 %= CONNTRACK_LOCKS;
 102         spin_unlock(&nf_conntrack_locks[h1]);
 103         if (h1 != h2)
 104                 spin_unlock(&nf_conntrack_locks[h2]);
 105 }
 106
 107 /* return true if we need to recompute hashes (in case hash table was resized) */
 108 static bool nf_conntrack_double_lock(struct net *net, unsigned int h1,
 109                                      unsigned int h2, unsigned int sequence)
 110 {
 111         h1 %= CONNTRACK_LOCKS;
 112         h2 %= CONNTRACK_LOCKS;
 113         if (h1 <= h2) {
 114                 nf_conntrack_lock(&nf_conntrack_locks[h1]);
 115                 if (h1 != h2)
 116                         spin_lock_nested(&nf_conntrack_locks[h2],
 117                                          SINGLE_DEPTH_NESTING);
 118         } else {
 119                 nf_conntrack_lock(&nf_conntrack_locks[h2]);
 120                 spin_lock_nested(&nf_conntrack_locks[h1],
 121                                  SINGLE_DEPTH_NESTING);
 122         }
 123         if (read_seqcount_retry(&nf_conntrack_generation, sequence)) {
 124                 nf_conntrack_double_unlock(h1, h2);
 125                 return true;
 126         }
 127         return false;
 128 }
 129
 130 static void nf_conntrack_all_lock(void)
 131 {
 132         int i;
 133
 134         spin_lock(&nf_conntrack_locks_all_lock);
 135         nf_conntrack_locks_all = true;
 136
 137         /*
 138          * Order the above store of 'nf_conntrack_locks_all' against
 139          * the spin_unlock_wait() loads below, such that if
 140          * nf_conntrack_lock() observes 'nf_conntrack_locks_all'
 141          * we must observe nf_conntrack_locks[] held:
 142          */
 143         smp_mb(); /* spin_lock(&nf_conntrack_locks_all_lock) */
 144
 145         for (i = 0; i < CONNTRACK_LOCKS; i++) {
 146                 spin_unlock_wait(&nf_conntrack_locks[i]);
 147         }
 148 }
 149
 150 static void nf_conntrack_all_unlock(void)
 151 {
 152         /*
 153          * All prior stores must be complete before we clear
 154          * 'nf_conntrack_locks_all'. Otherwise nf_conntrack_lock()
 155          * might observe the false value but not the entire
 156          * critical section:
 157          */
 158         smp_store_release(&nf_conntrack_locks_all, false);
 159         spin_unlock(&nf_conntrack_locks_all_lock);
 160 }
 161
 162 unsigned int nf_conntrack_htable_size __read_mostly;
 163 EXPORT_SYMBOL_GPL(nf_conntrack_htable_size);
 164
 165 unsigned int nf_conntrack_max __read_mostly;
 166 seqcount_t nf_conntrack_generation __read_mostly;
 167
 168 DEFINE_PER_CPU(struct nf_conn, nf_conntrack_untracked);
 169 EXPORT_PER_CPU_SYMBOL(nf_conntrack_untracked);
 170
 171 static unsigned int nf_conntrack_hash_rnd __read_mostly;
 172
 173 static u32 hash_conntrack_raw(const struct nf_conntrack_tuple *tuple,
 174                               const struct net *net)
 175 {
 176         unsigned int n;
 177         u32 seed;
 178
 179         get_random_once(&nf_conntrack_hash_rnd, sizeof(nf_conntrack_hash_rnd));
 180
 181         /* The direction must be ignored, so we hash everything up to the
 182          * destination ports (which is a multiple of 4) and treat the last
 183          * three bytes manually.
 184          */
 185         seed = nf_conntrack_hash_rnd ^ net_hash_mix(net);
 186         n = (sizeof(tuple->src) + sizeof(tuple->dst.u3)) / sizeof(u32);
 187         return jhash2((u32 *)tuple, n, seed ^
 188                       (((__force __u16)tuple->dst.u.all << 16) |
 189                       tuple->dst.protonum));
 190 }
 191
 192 static u32 scale_hash(u32 hash)
 193 {
 194         return reciprocal_scale(hash, nf_conntrack_htable_size);
 195 }
 196
 197 static u32 __hash_conntrack(const struct net *net,
 198                             const struct nf_conntrack_tuple *tuple,
 199                             unsigned int size)
 200 {
 201         return reciprocal_scale(hash_conntrack_raw(tuple, net), size);
 202 }
 203
 204 static u32 hash_conntrack(const struct net *net,
 205                           const struct nf_conntrack_tuple *tuple)
 206 {
 207         return scale_hash(hash_conntrack_raw(tuple, net));
 208 }
 209
 210 bool
 211 nf_ct_get_tuple(const struct sk_buff *skb,
 212                 unsigned int nhoff,
 213                 unsigned int dataoff,
 214                 u_int16_t l3num,
 215                 u_int8_t protonum,
 216                 struct net *net,
 217                 struct nf_conntrack_tuple *tuple,
 218                 const struct nf_conntrack_l3proto *l3proto,
 219                 const struct nf_conntrack_l4proto *l4proto)
 220 {
 221         memset(tuple, 0, sizeof(*tuple));
 222
 223         tuple->src.l3num = l3num;
 224         if (l3proto->pkt_to_tuple(skb, nhoff, tuple) == 0)
 225                 return false;
 226
 227         tuple->dst.protonum = protonum;
 228         tuple->dst.dir = IP_CT_DIR_ORIGINAL;
 229
 230         return l4proto->pkt_to_tuple(skb, dataoff, net, tuple);
 231 }
 232 EXPORT_SYMBOL_GPL(nf_ct_get_tuple);
 233
 234 bool nf_ct_get_tuplepr(const struct sk_buff *skb, unsigned int nhoff,
 235                        u_int16_t l3num,
 236                        struct net *net, struct nf_conntrack_tuple *tuple)
 237 {
 238         struct nf_conntrack_l3proto *l3proto;
 239         struct nf_conntrack_l4proto *l4proto;
 240         unsigned int protoff;
 241         u_int8_t protonum;
 242         int ret;
 243
 244         rcu_read_lock();
 245
 246         l3proto = __nf_ct_l3proto_find(l3num);
 247         ret = l3proto->get_l4proto(skb, nhoff, &protoff, &protonum);
 248         if (ret != NF_ACCEPT) {
 249                 rcu_read_unlock();
 250                 return false;
 251         }
 252
 253         l4proto = __nf_ct_l4proto_find(l3num, protonum);
 254
 255         ret = nf_ct_get_tuple(skb, nhoff, protoff, l3num, protonum, net, tuple,
 256                               l3proto, l4proto);
 257
 258         rcu_read_unlock();
 259         return ret;
 260 }
 261 EXPORT_SYMBOL_GPL(nf_ct_get_tuplepr);
 262
 263 bool
 264 nf_ct_invert_tuple(struct nf_conntrack_tuple *inverse,
 265                    const struct nf_conntrack_tuple *orig,
 266                    const struct nf_conntrack_l3proto *l3proto,
 267                    const struct nf_conntrack_l4proto *l4proto)
 268 {
 269         memset(inverse, 0, sizeof(*inverse));
 270
 271         inverse->src.l3num = orig->src.l3num;
 272         if (l3proto->invert_tuple(inverse, orig) == 0)
 273                 return false;
 274
 275         inverse->dst.dir = !orig->dst.dir;
 276
 277         inverse->dst.protonum = orig->dst.protonum;
 278         return l4proto->invert_tuple(inverse, orig);
 279 }
 280 EXPORT_SYMBOL_GPL(nf_ct_invert_tuple);
 281
 282 static void
 283 clean_from_lists(struct nf_conn *ct)
 284 {
 285         pr_debug("clean_from_lists(%p)\n", ct);
 286         hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode);
 287         hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode);
 288
 289         /* Destroy all pending expectations */
 290         nf_ct_remove_expectations(ct);
 291 }
 292
 293 /* must be called with local_bh_disable */
 294 static void nf_ct_add_to_dying_list(struct nf_conn *ct)
 295 {
 296         struct ct_pcpu *pcpu;
 297
 298         /* add this conntrack to the (per cpu) dying list */
 299         ct->cpu = smp_processor_id();
 300         pcpu = per_cpu_ptr(nf_ct_net(ct)->ct.pcpu_lists, ct->cpu);
 301
 302         spin_lock(&pcpu->lock);
 303         hlist_nulls_add_head(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode,
 304                              &pcpu->dying);
 305         spin_unlock(&pcpu->lock);
 306 }
 307
 308 /* must be called with local_bh_disable */
 309 static void nf_ct_add_to_unconfirmed_list(struct nf_conn *ct)
 310 {
 311         struct ct_pcpu *pcpu;
 312
 313         /* add this conntrack to the (per cpu) unconfirmed list */
 314         ct->cpu = smp_processor_id();
 315         pcpu = per_cpu_ptr(nf_ct_net(ct)->ct.pcpu_lists, ct->cpu);
 316
 317         spin_lock(&pcpu->lock);
 318         hlist_nulls_add_head(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode,
 319                              &pcpu->unconfirmed);
 320         spin_unlock(&pcpu->lock);
 321 }
 322
 323 /* must be called with local_bh_disable */
 324 static void nf_ct_del_from_dying_or_unconfirmed_list(struct nf_conn *ct)
 325 {
 326         struct ct_pcpu *pcpu;
 327
 328         /* We overload first tuple to link into unconfirmed or dying list.*/
 329         pcpu = per_cpu_ptr(nf_ct_net(ct)->ct.pcpu_lists, ct->cpu);
 330
 331         spin_lock(&pcpu->lock);
 332         BUG_ON(hlist_nulls_unhashed(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode));
 333         hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode);
 334         spin_unlock(&pcpu->lock);
 335 }
 336
 337 /* Released via destroy_conntrack() */
 338 struct nf_conn *nf_ct_tmpl_alloc(struct net *net,
 339                                  const struct nf_conntrack_zone *zone,
 340                                  gfp_t flags)
 341 {
 342         struct nf_conn *tmpl;
 343
 344         tmpl = kzalloc(sizeof(*tmpl), flags);
 345         if (tmpl == NULL)
 346                 return NULL;
 347
 348         tmpl->status = IPS_TEMPLATE;
 349         write_pnet(&tmpl->ct_net, net);
 350         nf_ct_zone_add(tmpl, zone);
 351         atomic_set(&tmpl->ct_general.use, 0);
 352
 353         return tmpl;
 354 }
 355 EXPORT_SYMBOL_GPL(nf_ct_tmpl_alloc);
 356
 357 void nf_ct_tmpl_free(struct nf_conn *tmpl)
 358 {
 359         nf_ct_ext_destroy(tmpl);
 360         nf_ct_ext_free(tmpl);
 361         kfree(tmpl);
 362 }
 363 EXPORT_SYMBOL_GPL(nf_ct_tmpl_free);
 364
 365 static void
 366 destroy_conntrack(struct nf_conntrack *nfct)
 367 {
 368         struct nf_conn *ct = (struct nf_conn *)nfct;
 369         struct net *net = nf_ct_net(ct);
 370         struct nf_conntrack_l4proto *l4proto;
 371
 372         pr_debug("destroy_conntrack(%p)\n", ct);
 373         NF_CT_ASSERT(atomic_read(&nfct->use) == 0);
 374         NF_CT_ASSERT(!timer_pending(&ct->timeout));
 375
 376         if (unlikely(nf_ct_is_template(ct))) {
 377                 nf_ct_tmpl_free(ct);
 378                 return;
 379         }
 380         rcu_read_lock();
 381         l4proto = __nf_ct_l4proto_find(nf_ct_l3num(ct), nf_ct_protonum(ct));
 382         if (l4proto->destroy)
 383                 l4proto->destroy(ct);
 384
 385         rcu_read_unlock();
 386
 387         local_bh_disable();
 388         /* Expectations will have been removed in clean_from_lists,
 389          * except TFTP can create an expectation on the first packet,
 390          * before connection is in the list, so we need to clean here,
 391          * too.
 392          */
 393         nf_ct_remove_expectations(ct);
 394
 395         nf_ct_del_from_dying_or_unconfirmed_list(ct);
 396
 397         NF_CT_STAT_INC(net, delete);
 398         local_bh_enable();
 399
 400         if (ct->master)
 401                 nf_ct_put(ct->master);
 402
 403         pr_debug("destroy_conntrack: returning ct=%p to slab\n", ct);
 404         nf_conntrack_free(ct);
 405 }
 406
 407 static void nf_ct_delete_from_lists(struct nf_conn *ct)
 408 {
 409         struct net *net = nf_ct_net(ct);
 410         unsigned int hash, reply_hash;
 411         unsigned int sequence;
 412
 413         nf_ct_helper_destroy(ct);
 414
 415         local_bh_disable();
 416         do {
 417                 sequence = read_seqcount_begin(&nf_conntrack_generation);
 418                 hash = hash_conntrack(net,
 419                                       &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple);
 420                 reply_hash = hash_conntrack(net,
 421                                            &ct->tuplehash[IP_CT_DIR_REPLY].tuple);
 422         } while (nf_conntrack_double_lock(net, hash, reply_hash, sequence));
 423
 424         clean_from_lists(ct);
 425         nf_conntrack_double_unlock(hash, reply_hash);
 426
 427         nf_ct_add_to_dying_list(ct);
 428
 429         NF_CT_STAT_INC(net, delete_list);
 430         local_bh_enable();
 431 }
 432
 433 bool nf_ct_delete(struct nf_conn *ct, u32 portid, int report)
 434 {
 435         struct nf_conn_tstamp *tstamp;
 436
 437         tstamp = nf_conn_tstamp_find(ct);
 438         if (tstamp && tstamp->stop == 0)
 439                 tstamp->stop = ktime_get_real_ns();
 440
 441         if (nf_ct_is_dying(ct))
 442                 goto delete;
 443
 444         if (nf_conntrack_event_report(IPCT_DESTROY, ct,
 445                                     portid, report) < 0) {
 446                 /* destroy event was not delivered */
 447                 nf_ct_delete_from_lists(ct);
 448                 nf_conntrack_ecache_delayed_work(nf_ct_net(ct));
 449                 return false;
 450         }
 451
 452         nf_conntrack_ecache_work(nf_ct_net(ct));
 453         set_bit(IPS_DYING_BIT, &ct->status);
 454  delete:
 455         nf_ct_delete_from_lists(ct);
 456         nf_ct_put(ct);
 457         return true;
 458 }
 459 EXPORT_SYMBOL_GPL(nf_ct_delete);
 460
 461 static void death_by_timeout(unsigned long ul_conntrack)
 462 {
 463         nf_ct_delete((struct nf_conn *)ul_conntrack, 0, 0);
 464 }
 465
 466 static inline bool
 467 nf_ct_key_equal(struct nf_conntrack_tuple_hash *h,
 468                 const struct nf_conntrack_tuple *tuple,
 469                 const struct nf_conntrack_zone *zone,
 470                 const struct net *net)
 471 {
 472         struct nf_conn *ct = nf_ct_tuplehash_to_ctrack(h);
 473
 474         /* A conntrack can be recreated with the equal tuple,
 475          * so we need to check that the conntrack is confirmed
 476          */
 477         return nf_ct_tuple_equal(tuple, &h->tuple) &&
 478                nf_ct_zone_equal(ct, zone, NF_CT_DIRECTION(h)) &&
 479                nf_ct_is_confirmed(ct) &&
 480                net_eq(net, nf_ct_net(ct));
 481 }
 482
 483 /*
 484  * Warning :
 485  * - Caller must take a reference on returned object
 486  *   and recheck nf_ct_tuple_equal(tuple, &h->tuple)
 487  */
 488 static struct nf_conntrack_tuple_hash *
 489 ____nf_conntrack_find(struct net *net, const struct nf_conntrack_zone *zone,
 490                       const struct nf_conntrack_tuple *tuple, u32 hash)
 491 {
 492         struct nf_conntrack_tuple_hash *h;
 493         struct hlist_nulls_head *ct_hash;
 494         struct hlist_nulls_node *n;
 495         unsigned int bucket, hsize;
 496
 497 begin:
 498         nf_conntrack_get_ht(&ct_hash, &hsize);
 499         bucket = reciprocal_scale(hash, hsize);
 500
 501         hlist_nulls_for_each_entry_rcu(h, n, &ct_hash[bucket], hnnode) {
 502                 if (nf_ct_key_equal(h, tuple, zone, net)) {
 503                         NF_CT_STAT_INC_ATOMIC(net, found);
 504                         return h;
 505                 }
 506                 NF_CT_STAT_INC_ATOMIC(net, searched);
 507         }
 508         /*
 509          * if the nulls value we got at the end of this lookup is
 510          * not the expected one, we must restart lookup.
 511          * We probably met an item that was moved to another chain.
 512          */
 513         if (get_nulls_value(n) != bucket) {
 514                 NF_CT_STAT_INC_ATOMIC(net, search_restart);
 515                 goto begin;
 516         }
 517
 518         return NULL;
 519 }
 520
 521 /* Find a connection corresponding to a tuple. */
 522 static struct nf_conntrack_tuple_hash *
 523 __nf_conntrack_find_get(struct net *net, const struct nf_conntrack_zone *zone,
 524                         const struct nf_conntrack_tuple *tuple, u32 hash)
 525 {
 526         struct nf_conntrack_tuple_hash *h;
 527         struct nf_conn *ct;
 528
 529         rcu_read_lock();
 530 begin:
 531         h = ____nf_conntrack_find(net, zone, tuple, hash);
 532         if (h) {
 533                 ct = nf_ct_tuplehash_to_ctrack(h);
 534                 if (unlikely(nf_ct_is_dying(ct) ||
 535                              !atomic_inc_not_zero(&ct->ct_general.use)))
 536                         h = NULL;
 537                 else {
 538                         if (unlikely(!nf_ct_key_equal(h, tuple, zone, net))) {
 539                                 nf_ct_put(ct);
 540                                 goto begin;
 541                         }
 542                 }
 543         }
 544         rcu_read_unlock();
 545
 546         return h;
 547 }
 548
 549 struct nf_conntrack_tuple_hash *
 550 nf_conntrack_find_get(struct net *net, const struct nf_conntrack_zone *zone,
 551                       const struct nf_conntrack_tuple *tuple)
 552 {
 553         return __nf_conntrack_find_get(net, zone, tuple,
 554                                        hash_conntrack_raw(tuple, net));
 555 }
 556 EXPORT_SYMBOL_GPL(nf_conntrack_find_get);
 557
 558 static void __nf_conntrack_hash_insert(struct nf_conn *ct,
 559                                        unsigned int hash,
 560                                        unsigned int reply_hash)
 561 {
 562         hlist_nulls_add_head_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode,
 563                            &nf_conntrack_hash[hash]);
 564         hlist_nulls_add_head_rcu(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode,
 565                            &nf_conntrack_hash[reply_hash]);
 566 }
 567
 568 int
 569 nf_conntrack_hash_check_insert(struct nf_conn *ct)
 570 {
 571         const struct nf_conntrack_zone *zone;
 572         struct net *net = nf_ct_net(ct);
 573         unsigned int hash, reply_hash;
 574         struct nf_conntrack_tuple_hash *h;
 575         struct hlist_nulls_node *n;
 576         unsigned int sequence;
 577
 578         zone = nf_ct_zone(ct);
 579
 580         local_bh_disable();
 581         do {
 582                 sequence = read_seqcount_begin(&nf_conntrack_generation);
 583                 hash = hash_conntrack(net,
 584                                       &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple);
 585                 reply_hash = hash_conntrack(net,
 586                                            &ct->tuplehash[IP_CT_DIR_REPLY].tuple);
 587         } while (nf_conntrack_double_lock(net, hash, reply_hash, sequence));
 588
 589         /* See if there's one in the list already, including reverse */
 590         hlist_nulls_for_each_entry(h, n, &nf_conntrack_hash[hash], hnnode)
 591                 if (nf_ct_key_equal(h, &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple,
 592                                     zone, net))
 593                         goto out;
 594
 595         hlist_nulls_for_each_entry(h, n, &nf_conntrack_hash[reply_hash], hnnode)
 596                 if (nf_ct_key_equal(h, &ct->tuplehash[IP_CT_DIR_REPLY].tuple,
 597                                     zone, net))
 598                         goto out;
 599
 600         add_timer(&ct->timeout);
 601         smp_wmb();
 602         /* The caller holds a reference to this object */
 603         atomic_set(&ct->ct_general.use, 2);
 604         __nf_conntrack_hash_insert(ct, hash, reply_hash);
 605         nf_conntrack_double_unlock(hash, reply_hash);
 606         NF_CT_STAT_INC(net, insert);
 607         local_bh_enable();
 608         return 0;
 609
 610 out:
 611         nf_conntrack_double_unlock(hash, reply_hash);
 612         NF_CT_STAT_INC(net, insert_failed);
 613         local_bh_enable();
 614         return -EEXIST;
 615 }
 616 EXPORT_SYMBOL_GPL(nf_conntrack_hash_check_insert);
 617
 618 static inline void nf_ct_acct_update(struct nf_conn *ct,
 619                                      enum ip_conntrack_info ctinfo,
 620                                      unsigned int len)
 621 {
 622         struct nf_conn_acct *acct;
 623
 624         acct = nf_conn_acct_find(ct);
 625         if (acct) {
 626                 struct nf_conn_counter *counter = acct->counter;
 627
 628                 atomic64_inc(&counter[CTINFO2DIR(ctinfo)].packets);
 629                 atomic64_add(len, &counter[CTINFO2DIR(ctinfo)].bytes);
 630         }
 631 }
 632
 633 static void nf_ct_acct_merge(struct nf_conn *ct, enum ip_conntrack_info ctinfo,
 634                              const struct nf_conn *loser_ct)
 635 {
 636         struct nf_conn_acct *acct;
 637
 638         acct = nf_conn_acct_find(loser_ct);
 639         if (acct) {
 640                 struct nf_conn_counter *counter = acct->counter;
 641                 unsigned int bytes;
 642
 643                 /* u32 should be fine since we must have seen one packet. */
 644                 bytes = atomic64_read(&counter[CTINFO2DIR(ctinfo)].bytes);
 645                 nf_ct_acct_update(ct, ctinfo, bytes);
 646         }
 647 }
 648
 649 /* Resolve race on insertion if this protocol allows this. */
 650 static int nf_ct_resolve_clash(struct net *net, struct sk_buff *skb,
 651                                enum ip_conntrack_info ctinfo,
 652                                struct nf_conntrack_tuple_hash *h)
 653 {
 654         /* This is the conntrack entry already in hashes that won race. */
 655         struct nf_conn *ct = nf_ct_tuplehash_to_ctrack(h);
 656         struct nf_conntrack_l4proto *l4proto;
 657
 658         l4proto = __nf_ct_l4proto_find(nf_ct_l3num(ct), nf_ct_protonum(ct));
 659         if (l4proto->allow_clash &&
 660             !nfct_nat(ct) &&
 661             !nf_ct_is_dying(ct) &&
 662             atomic_inc_not_zero(&ct->ct_general.use)) {
 663                 nf_ct_acct_merge(ct, ctinfo, (struct nf_conn *)skb->nfct);
 664                 nf_conntrack_put(skb->nfct);
 665                 /* Assign conntrack already in hashes to this skbuff. Don't
 666                  * modify skb->nfctinfo to ensure consistent stateful filtering.
 667                  */
 668                 skb->nfct = &ct->ct_general;
 669                 return NF_ACCEPT;
 670         }
 671         NF_CT_STAT_INC(net, drop);
 672         return NF_DROP;
 673 }
 674
 675 /* Confirm a connection given skb; places it in hash table */
 676 int
 677 __nf_conntrack_confirm(struct sk_buff *skb)
 678 {
 679         const struct nf_conntrack_zone *zone;
 680         unsigned int hash, reply_hash;
 681         struct nf_conntrack_tuple_hash *h;
 682         struct nf_conn *ct;
 683         struct nf_conn_help *help;
 684         struct nf_conn_tstamp *tstamp;
 685         struct hlist_nulls_node *n;
 686         enum ip_conntrack_info ctinfo;
 687         struct net *net;
 688         unsigned int sequence;
 689         int ret = NF_DROP;
 690
 691         ct = nf_ct_get(skb, &ctinfo);
 692         net = nf_ct_net(ct);
 693
 694         /* ipt_REJECT uses nf_conntrack_attach to attach related
 695            ICMP/TCP RST packets in other direction.  Actual packet
 696            which created connection will be IP_CT_NEW or for an
 697            expected connection, IP_CT_RELATED. */
 698         if (CTINFO2DIR(ctinfo) != IP_CT_DIR_ORIGINAL)
 699                 return NF_ACCEPT;
 700
 701         zone = nf_ct_zone(ct);
 702         local_bh_disable();
 703
 704         do {
 705                 sequence = read_seqcount_begin(&nf_conntrack_generation);
 706                 /* reuse the hash saved before */
 707                 hash = *(unsigned long *)&ct->tuplehash[IP_CT_DIR_REPLY].hnnode.pprev;
 708                 hash = scale_hash(hash);
 709                 reply_hash = hash_conntrack(net,
 710                                            &ct->tuplehash[IP_CT_DIR_REPLY].tuple);
 711
 712         } while (nf_conntrack_double_lock(net, hash, reply_hash, sequence));
 713
 714         /* We're not in hash table, and we refuse to set up related
 715          * connections for unconfirmed conns.  But packet copies and
 716          * REJECT will give spurious warnings here.
 717          */
 718         /* NF_CT_ASSERT(atomic_read(&ct->ct_general.use) == 1); */
 719
 720         /* No external references means no one else could have
 721          * confirmed us.
 722          */
 723         NF_CT_ASSERT(!nf_ct_is_confirmed(ct));
 724         pr_debug("Confirming conntrack %p\n", ct);
 725         /* We have to check the DYING flag after unlink to prevent
 726          * a race against nf_ct_get_next_corpse() possibly called from
 727          * user context, else we insert an already 'dead' hash, blocking
 728          * further use of that particular connection -JM.
 729          */
 730         nf_ct_del_from_dying_or_unconfirmed_list(ct);
 731
 732         if (unlikely(nf_ct_is_dying(ct))) {
 733                 nf_ct_add_to_dying_list(ct);
 734                 goto dying;
 735         }
 736
 737         /* See if there's one in the list already, including reverse:
 738            NAT could have grabbed it without realizing, since we're
 739            not in the hash.  If there is, we lost race. */
 740         hlist_nulls_for_each_entry(h, n, &nf_conntrack_hash[hash], hnnode)
 741                 if (nf_ct_key_equal(h, &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple,
 742                                     zone, net))
 743                         goto out;
 744
 745         hlist_nulls_for_each_entry(h, n, &nf_conntrack_hash[reply_hash], hnnode)
 746                 if (nf_ct_key_equal(h, &ct->tuplehash[IP_CT_DIR_REPLY].tuple,
 747                                     zone, net))
 748                         goto out;
 749
 750         /* Timer relative to confirmation time, not original
 751            setting time, otherwise we'd get timer wrap in
 752            weird delay cases. */
 753         ct->timeout.expires += jiffies;
 754         add_timer(&ct->timeout);
 755         atomic_inc(&ct->ct_general.use);
 756         ct->status |= IPS_CONFIRMED;
 757
 758         /* set conntrack timestamp, if enabled. */
 759         tstamp = nf_conn_tstamp_find(ct);
 760         if (tstamp) {
 761                 if (skb->tstamp.tv64 == 0)
 762                         __net_timestamp(skb);
 763
 764                 tstamp->start = ktime_to_ns(skb->tstamp);
 765         }
 766         /* Since the lookup is lockless, hash insertion must be done after
 767          * starting the timer and setting the CONFIRMED bit. The RCU barriers
 768          * guarantee that no other CPU can find the conntrack before the above
 769          * stores are visible.
 770          */
 771         __nf_conntrack_hash_insert(ct, hash, reply_hash);
 772         nf_conntrack_double_unlock(hash, reply_hash);
 773         NF_CT_STAT_INC(net, insert);
 774         local_bh_enable();
 775
 776         help = nfct_help(ct);
 777         if (help && help->helper)
 778                 nf_conntrack_event_cache(IPCT_HELPER, ct);
 779
 780         nf_conntrack_event_cache(master_ct(ct) ?
 781                                  IPCT_RELATED : IPCT_NEW, ct);
 782         return NF_ACCEPT;
 783
 784 out:
 785         nf_ct_add_to_dying_list(ct);
 786         ret = nf_ct_resolve_clash(net, skb, ctinfo, h);
 787 dying:
 788         nf_conntrack_double_unlock(hash, reply_hash);
 789         NF_CT_STAT_INC(net, insert_failed);
 790         local_bh_enable();
 791         return ret;
 792 }
 793 EXPORT_SYMBOL_GPL(__nf_conntrack_confirm);
 794
 795 /* Returns true if a connection correspondings to the tuple (required
 796    for NAT). */
 797 int
 798 nf_conntrack_tuple_taken(const struct nf_conntrack_tuple *tuple,
 799                          const struct nf_conn *ignored_conntrack)
 800 {
 801         struct net *net = nf_ct_net(ignored_conntrack);
 802         const struct nf_conntrack_zone *zone;
 803         struct nf_conntrack_tuple_hash *h;
 804         struct hlist_nulls_head *ct_hash;
 805         unsigned int hash, hsize;
 806         struct hlist_nulls_node *n;
 807         struct nf_conn *ct;
 808
 809         zone = nf_ct_zone(ignored_conntrack);
 810
 811         rcu_read_lock();
 812         nf_conntrack_get_ht(&ct_hash, &hsize);
 813         hash = __hash_conntrack(net, tuple, hsize);
 814
 815         hlist_nulls_for_each_entry_rcu(h, n, &ct_hash[hash], hnnode) {
 816                 ct = nf_ct_tuplehash_to_ctrack(h);
 817                 if (ct != ignored_conntrack &&
 818                     nf_ct_key_equal(h, tuple, zone, net)) {
 819                         NF_CT_STAT_INC_ATOMIC(net, found);
 820                         rcu_read_unlock();
 821                         return 1;
 822                 }
 823                 NF_CT_STAT_INC_ATOMIC(net, searched);
 824         }
 825         rcu_read_unlock();
 826
 827         return 0;
 828 }
 829 EXPORT_SYMBOL_GPL(nf_conntrack_tuple_taken);
 830
 831 #define NF_CT_EVICTION_RANGE    8
 832
 833 /* There's a small race here where we may free a just-assured
 834    connection.  Too bad: we're in trouble anyway. */
 835 static unsigned int early_drop_list(struct net *net,
 836                                     struct hlist_nulls_head *head)
 837 {
 838         struct nf_conntrack_tuple_hash *h;
 839         struct hlist_nulls_node *n;
 840         unsigned int drops = 0;
 841         struct nf_conn *tmp;
 842
 843         hlist_nulls_for_each_entry_rcu(h, n, head, hnnode) {
 844                 tmp = nf_ct_tuplehash_to_ctrack(h);
 845
 846                 if (test_bit(IPS_ASSURED_BIT, &tmp->status) ||
 847                     !net_eq(nf_ct_net(tmp), net) ||
 848                     nf_ct_is_dying(tmp))
 849                         continue;
 850
 851                 if (!atomic_inc_not_zero(&tmp->ct_general.use))
 852                         continue;
 853
 854                 /* kill only if still in same netns -- might have moved due to
 855                  * SLAB_DESTROY_BY_RCU rules.
 856                  *
 857                  * We steal the timer reference.  If that fails timer has
 858                  * already fired or someone else deleted it. Just drop ref
 859                  * and move to next entry.
 860                  */
 861                 if (net_eq(nf_ct_net(tmp), net) &&
 862                     nf_ct_is_confirmed(tmp) &&
 863                     del_timer(&tmp->timeout) &&
 864                     nf_ct_delete(tmp, 0, 0))
 865                         drops++;
 866
 867                 nf_ct_put(tmp);
 868         }
 869
 870         return drops;
 871 }
 872
 873 static noinline int early_drop(struct net *net, unsigned int _hash)
 874 {
 875         unsigned int i;
 876
 877         for (i = 0; i < NF_CT_EVICTION_RANGE; i++) {
 878                 struct hlist_nulls_head *ct_hash;
 879                 unsigned int hash, hsize, drops;
 880
 881                 rcu_read_lock();
 882                 nf_conntrack_get_ht(&ct_hash, &hsize);
 883                 hash = reciprocal_scale(_hash++, hsize);
 884
 885                 drops = early_drop_list(net, &ct_hash[hash]);
 886                 rcu_read_unlock();
 887
 888                 if (drops) {
 889                         NF_CT_STAT_ADD_ATOMIC(net, early_drop, drops);
 890                         return true;
 891                 }
 892         }
 893
 894         return false;
 895 }
 896
 897 static struct nf_conn *
 898 __nf_conntrack_alloc(struct net *net,
 899                      const struct nf_conntrack_zone *zone,
 900                      const struct nf_conntrack_tuple *orig,
 901                      const struct nf_conntrack_tuple *repl,
 902                      gfp_t gfp, u32 hash)
 903 {
 904         struct nf_conn *ct;
 905
 906         /* We don't want any race condition at early drop stage */
 907         atomic_inc(&net->ct.count);
 908
 909         if (nf_conntrack_max &&
 910             unlikely(atomic_read(&net->ct.count) > nf_conntrack_max)) {
 911                 if (!early_drop(net, hash)) {
 912                         atomic_dec(&net->ct.count);
 913                         net_warn_ratelimited("nf_conntrack: table full, dropping packet\n");
 914                         return ERR_PTR(-ENOMEM);
 915                 }
 916         }
 917
 918         /*
 919          * Do not use kmem_cache_zalloc(), as this cache uses
 920          * SLAB_DESTROY_BY_RCU.
 921          */
 922         ct = kmem_cache_alloc(nf_conntrack_cachep, gfp);
 923         if (ct == NULL)
 924                 goto out;
 925
 926         spin_lock_init(&ct->lock);
 927         ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple = *orig;
 928         ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode.pprev = NULL;
 929         ct->tuplehash[IP_CT_DIR_REPLY].tuple = *repl;
 930         /* save hash for reusing when confirming */
 931         *(unsigned long *)(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode.pprev) = hash;
 932         ct->status = 0;
 933         /* Don't set timer yet: wait for confirmation */
 934         setup_timer(&ct->timeout, death_by_timeout, (unsigned long)ct);
 935         write_pnet(&ct->ct_net, net);
 936         memset(&ct->__nfct_init_offset[0], 0,
 937                offsetof(struct nf_conn, proto) -
 938                offsetof(struct nf_conn, __nfct_init_offset[0]));
 939
 940         nf_ct_zone_add(ct, zone);
 941
 942         /* Because we use RCU lookups, we set ct_general.use to zero before
 943          * this is inserted in any list.
 944          */
 945         atomic_set(&ct->ct_general.use, 0);
 946         return ct;
 947 out:
 948         atomic_dec(&net->ct.count);
 949         return ERR_PTR(-ENOMEM);
 950 }
 951
 952 struct nf_conn *nf_conntrack_alloc(struct net *net,
 953                                    const struct nf_conntrack_zone *zone,
 954                                    const struct nf_conntrack_tuple *orig,
 955                                    const struct nf_conntrack_tuple *repl,
 956                                    gfp_t gfp)
 957 {
 958         return __nf_conntrack_alloc(net, zone, orig, repl, gfp, 0);
 959 }
 960 EXPORT_SYMBOL_GPL(nf_conntrack_alloc);
 961
 962 void nf_conntrack_free(struct nf_conn *ct)
 963 {
 964         struct net *net = nf_ct_net(ct);
 965
 966         /* A freed object has refcnt == 0, that's
 967          * the golden rule for SLAB_DESTROY_BY_RCU
 968          */
 969         NF_CT_ASSERT(atomic_read(&ct->ct_general.use) == 0);
 970
 971         nf_ct_ext_destroy(ct);
 972         nf_ct_ext_free(ct);
 973         kmem_cache_free(nf_conntrack_cachep, ct);
 974         smp_mb__before_atomic();
 975         atomic_dec(&net->ct.count);
 976 }
 977 EXPORT_SYMBOL_GPL(nf_conntrack_free);
 978
 979
 980 /* Allocate a new conntrack: we return -ENOMEM if classification
 981    failed due to stress.  Otherwise it really is unclassifiable. */
 982 static struct nf_conntrack_tuple_hash *
 983 init_conntrack(struct net *net, struct nf_conn *tmpl,
 984                const struct nf_conntrack_tuple *tuple,
 985                struct nf_conntrack_l3proto *l3proto,
 986                struct nf_conntrack_l4proto *l4proto,
 987                struct sk_buff *skb,
 988                unsigned int dataoff, u32 hash)
 989 {
 990         struct nf_conn *ct;
 991         struct nf_conn_help *help;
 992         struct nf_conntrack_tuple repl_tuple;
 993         struct nf_conntrack_ecache *ecache;
 994         struct nf_conntrack_expect *exp = NULL;
 995         const struct nf_conntrack_zone *zone;
 996         struct nf_conn_timeout *timeout_ext;
 997         struct nf_conntrack_zone tmp;
 998         unsigned int *timeouts;
 999
1000         if (!nf_ct_invert_tuple(&repl_tuple, tuple, l3proto, l4proto)) {
1001                 pr_debug("Can't invert tuple.\n");
1002                 return NULL;
1003         }
1004
1005         zone = nf_ct_zone_tmpl(tmpl, skb, &tmp);
1006         ct = __nf_conntrack_alloc(net, zone, tuple, &repl_tuple, GFP_ATOMIC,
1007                                   hash);
1008         if (IS_ERR(ct))
1009                 return (struct nf_conntrack_tuple_hash *)ct;
1010
1011         if (tmpl && nfct_synproxy(tmpl)) {
1012                 nfct_seqadj_ext_add(ct);
1013                 nfct_synproxy_ext_add(ct);
1014         }
1015
1016         timeout_ext = tmpl ? nf_ct_timeout_find(tmpl) : NULL;
1017         if (timeout_ext) {
1018                 timeouts = nf_ct_timeout_data(timeout_ext);
1019                 if (unlikely(!timeouts))
1020                         timeouts = l4proto->get_timeouts(net);
1021         } else {
1022                 timeouts = l4proto->get_timeouts(net);
1023         }
1024
1025         if (!l4proto->new(ct, skb, dataoff, timeouts)) {
1026                 nf_conntrack_free(ct);
1027                 pr_debug("can't track with proto module\n");
1028                 return NULL;
1029         }
1030
1031         if (timeout_ext)
1032                 nf_ct_timeout_ext_add(ct, rcu_dereference(timeout_ext->timeout),
1033                                       GFP_ATOMIC);
1034
1035         nf_ct_acct_ext_add(ct, GFP_ATOMIC);
1036         nf_ct_tstamp_ext_add(ct, GFP_ATOMIC);
1037         nf_ct_labels_ext_add(ct);
1038
1039         ecache = tmpl ? nf_ct_ecache_find(tmpl) : NULL;
1040         nf_ct_ecache_ext_add(ct, ecache ? ecache->ctmask : 0,
1041                                  ecache ? ecache->expmask : 0,
1042                              GFP_ATOMIC);
1043
1044         local_bh_disable();
1045         if (net->ct.expect_count) {
1046                 spin_lock(&nf_conntrack_expect_lock);
1047                 exp = nf_ct_find_expectation(net, zone, tuple);
1048                 if (exp) {
1049                         pr_debug("expectation arrives ct=%p exp=%p\n",
1050                                  ct, exp);
1051                         /* Welcome, Mr. Bond.  We've been expecting you... */
1052                         __set_bit(IPS_EXPECTED_BIT, &ct->status);
1053                         /* exp->master safe, refcnt bumped in nf_ct_find_expectation */
1054                         ct->master = exp->master;
1055                         if (exp->helper) {
1056                                 help = nf_ct_helper_ext_add(ct, exp->helper,
1057                                                             GFP_ATOMIC);
1058                                 if (help)
1059                                         rcu_assign_pointer(help->helper, exp->helper);
1060                         }
1061
1062 #ifdef CONFIG_NF_CONNTRACK_MARK
1063                         ct->mark = exp->master->mark;
1064 #endif
1065 #ifdef CONFIG_NF_CONNTRACK_SECMARK
1066                         ct->secmark = exp->master->secmark;
1067 #endif
1068                         NF_CT_STAT_INC(net, expect_new);
1069                 }
1070                 spin_unlock(&nf_conntrack_expect_lock);
1071         }
1072         if (!exp) {
1073                 __nf_ct_try_assign_helper(ct, tmpl, GFP_ATOMIC);
1074                 NF_CT_STAT_INC(net, new);
1075         }
1076
1077         /* Now it is inserted into the unconfirmed list, bump refcount */
1078         nf_conntrack_get(&ct->ct_general);
1079         nf_ct_add_to_unconfirmed_list(ct);
1080
1081         local_bh_enable();
1082
1083         if (exp) {
1084                 if (exp->expectfn)
1085                         exp->expectfn(ct, exp);
1086                 nf_ct_expect_put(exp);
1087         }
1088
1089         return &ct->tuplehash[IP_CT_DIR_ORIGINAL];
1090 }
1091
1092 /* On success, returns conntrack ptr, sets skb->nfct and ctinfo */
1093 static inline struct nf_conn *
1094 resolve_normal_ct(struct net *net, struct nf_conn *tmpl,
1095                   struct sk_buff *skb,
1096                   unsigned int dataoff,
1097                   u_int16_t l3num,
1098                   u_int8_t protonum,
1099                   struct nf_conntrack_l3proto *l3proto,
1100                   struct nf_conntrack_l4proto *l4proto,
1101                   int *set_reply,
1102                   enum ip_conntrack_info *ctinfo)
1103 {
1104         const struct nf_conntrack_zone *zone;
1105         struct nf_conntrack_tuple tuple;
1106         struct nf_conntrack_tuple_hash *h;
1107         struct nf_conntrack_zone tmp;
1108         struct nf_conn *ct;
1109         u32 hash;
1110
1111         if (!nf_ct_get_tuple(skb, skb_network_offset(skb),
1112                              dataoff, l3num, protonum, net, &tuple, l3proto,
1113                              l4proto)) {
1114                 pr_debug("Can't get tuple\n");
1115                 return NULL;
1116         }
1117
1118         /* look for tuple match */
1119         zone = nf_ct_zone_tmpl(tmpl, skb, &tmp);
1120         hash = hash_conntrack_raw(&tuple, net);
1121         h = __nf_conntrack_find_get(net, zone, &tuple, hash);
1122         if (!h) {
1123                 h = init_conntrack(net, tmpl, &tuple, l3proto, l4proto,
1124                                    skb, dataoff, hash);
1125                 if (!h)
1126                         return NULL;
1127                 if (IS_ERR(h))
1128                         return (void *)h;
1129         }
1130         ct = nf_ct_tuplehash_to_ctrack(h);
1131
1132         /* It exists; we have (non-exclusive) reference. */
1133         if (NF_CT_DIRECTION(h) == IP_CT_DIR_REPLY) {
1134                 *ctinfo = IP_CT_ESTABLISHED_REPLY;
1135                 /* Please set reply bit if this packet OK */
1136                 *set_reply = 1;
1137         } else {
1138                 /* Once we've had two way comms, always ESTABLISHED. */
1139                 if (test_bit(IPS_SEEN_REPLY_BIT, &ct->status)) {
1140                         pr_debug("normal packet for %p\n", ct);
1141                         *ctinfo = IP_CT_ESTABLISHED;
1142                 } else if (test_bit(IPS_EXPECTED_BIT, &ct->status)) {
1143                         pr_debug("related packet for %p\n", ct);
1144                         *ctinfo = IP_CT_RELATED;
1145                 } else {
1146                         pr_debug("new packet for %p\n", ct);
1147                         *ctinfo = IP_CT_NEW;
1148                 }
1149                 *set_reply = 0;
1150         }
1151         skb->nfct = &ct->ct_general;
1152         skb->nfctinfo = *ctinfo;
1153         return ct;
1154 }
1155
1156 unsigned int
1157 nf_conntrack_in(struct net *net, u_int8_t pf, unsigned int hooknum,
1158                 struct sk_buff *skb)
1159 {
1160         struct nf_conn *ct, *tmpl = NULL;
1161         enum ip_conntrack_info ctinfo;
1162         struct nf_conntrack_l3proto *l3proto;
1163         struct nf_conntrack_l4proto *l4proto;
1164         unsigned int *timeouts;
1165         unsigned int dataoff;
1166         u_int8_t protonum;
1167         int set_reply = 0;
1168         int ret;
1169
1170         if (skb->nfct) {
1171                 /* Previously seen (loopback or untracked)?  Ignore. */
1172                 tmpl = (struct nf_conn *)skb->nfct;
1173                 if (!nf_ct_is_template(tmpl)) {
1174                         NF_CT_STAT_INC_ATOMIC(net, ignore);
1175                         return NF_ACCEPT;
1176                 }
1177                 skb->nfct = NULL;
1178         }
1179
1180         /* rcu_read_lock()ed by nf_hook_slow */
1181         l3proto = __nf_ct_l3proto_find(pf);
1182         ret = l3proto->get_l4proto(skb, skb_network_offset(skb),
1183                                    &dataoff, &protonum);
1184         if (ret <= 0) {
1185                 pr_debug("not prepared to track yet or error occurred\n");
1186                 NF_CT_STAT_INC_ATOMIC(net, error);
1187                 NF_CT_STAT_INC_ATOMIC(net, invalid);
1188                 ret = -ret;
1189                 goto out;
1190         }
1191
1192         l4proto = __nf_ct_l4proto_find(pf, protonum);
1193
1194         /* It may be an special packet, error, unclean...
1195          * inverse of the return code tells to the netfilter
1196          * core what to do with the packet. */
1197         if (l4proto->error != NULL) {
1198                 ret = l4proto->error(net, tmpl, skb, dataoff, &ctinfo,
1199                                      pf, hooknum);
1200                 if (ret <= 0) {
1201                         NF_CT_STAT_INC_ATOMIC(net, error);
1202                         NF_CT_STAT_INC_ATOMIC(net, invalid);
1203                         ret = -ret;
1204                         goto out;
1205                 }
1206                 /* ICMP[v6] protocol trackers may assign one conntrack. */
1207                 if (skb->nfct)
1208                         goto out;
1209         }
1210
1211         ct = resolve_normal_ct(net, tmpl, skb, dataoff, pf, protonum,
1212                                l3proto, l4proto, &set_reply, &ctinfo);
1213         if (!ct) {
1214                 /* Not valid part of a connection */
1215                 NF_CT_STAT_INC_ATOMIC(net, invalid);
1216                 ret = NF_ACCEPT;
1217                 goto out;
1218         }
1219
1220         if (IS_ERR(ct)) {
1221                 /* Too stressed to deal. */
1222                 NF_CT_STAT_INC_ATOMIC(net, drop);
1223                 ret = NF_DROP;
1224                 goto out;
1225         }
1226
1227         NF_CT_ASSERT(skb->nfct);
1228
1229         /* Decide what timeout policy we want to apply to this flow. */
1230         timeouts = nf_ct_timeout_lookup(net, ct, l4proto);
1231
1232         ret = l4proto->packet(ct, skb, dataoff, ctinfo, pf, hooknum, timeouts);
1233         if (ret <= 0) {
1234                 /* Invalid: inverse of the return code tells
1235                  * the netfilter core what to do */
1236                 pr_debug("nf_conntrack_in: Can't track with proto module\n");
1237                 nf_conntrack_put(skb->nfct);
1238                 skb->nfct = NULL;
1239                 NF_CT_STAT_INC_ATOMIC(net, invalid);
1240                 if (ret == -NF_DROP)
1241                         NF_CT_STAT_INC_ATOMIC(net, drop);
1242                 ret = -ret;
1243                 goto out;
1244         }
1245
1246         if (set_reply && !test_and_set_bit(IPS_SEEN_REPLY_BIT, &ct->status))
1247                 nf_conntrack_event_cache(IPCT_REPLY, ct);
1248 out:
1249         if (tmpl) {
1250                 /* Special case: we have to repeat this hook, assign the
1251                  * template again to this packet. We assume that this packet
1252                  * has no conntrack assigned. This is used by nf_ct_tcp. */
1253                 if (ret == NF_REPEAT)
1254                         skb->nfct = (struct nf_conntrack *)tmpl;
1255                 else
1256                         nf_ct_put(tmpl);
1257         }
1258
1259         return ret;
1260 }
1261 EXPORT_SYMBOL_GPL(nf_conntrack_in);
1262
1263 bool nf_ct_invert_tuplepr(struct nf_conntrack_tuple *inverse,
1264                           const struct nf_conntrack_tuple *orig)
1265 {
1266         bool ret;
1267
1268         rcu_read_lock();
1269         ret = nf_ct_invert_tuple(inverse, orig,
1270                                  __nf_ct_l3proto_find(orig->src.l3num),
1271                                  __nf_ct_l4proto_find(orig->src.l3num,
1272                                                       orig->dst.protonum));
1273         rcu_read_unlock();
1274         return ret;
1275 }
1276 EXPORT_SYMBOL_GPL(nf_ct_invert_tuplepr);
1277
1278 /* Alter reply tuple (maybe alter helper).  This is for NAT, and is
1279    implicitly racy: see __nf_conntrack_confirm */
1280 void nf_conntrack_alter_reply(struct nf_conn *ct,
1281                               const struct nf_conntrack_tuple *newreply)
1282 {
1283         struct nf_conn_help *help = nfct_help(ct);
1284
1285         /* Should be unconfirmed, so not in hash table yet */
1286         NF_CT_ASSERT(!nf_ct_is_confirmed(ct));
1287
1288         pr_debug("Altering reply tuple of %p to ", ct);
1289         nf_ct_dump_tuple(newreply);
1290
1291         ct->tuplehash[IP_CT_DIR_REPLY].tuple = *newreply;
1292         if (ct->master || (help && !hlist_empty(&help->expectations)))
1293                 return;
1294
1295         rcu_read_lock();
1296         __nf_ct_try_assign_helper(ct, NULL, GFP_ATOMIC);
1297         rcu_read_unlock();
1298 }
1299 EXPORT_SYMBOL_GPL(nf_conntrack_alter_reply);
1300
1301 /* Refresh conntrack for this many jiffies and do accounting if do_acct is 1 */
1302 void __nf_ct_refresh_acct(struct nf_conn *ct,
1303                           enum ip_conntrack_info ctinfo,
1304                           const struct sk_buff *skb,
1305                           unsigned long extra_jiffies,
1306                           int do_acct)
1307 {
1308         NF_CT_ASSERT(ct->timeout.data == (unsigned long)ct);
1309         NF_CT_ASSERT(skb);
1310
1311         /* Only update if this is not a fixed timeout */
1312         if (test_bit(IPS_FIXED_TIMEOUT_BIT, &ct->status))
1313                 goto acct;
1314
1315         /* If not in hash table, timer will not be active yet */
1316         if (!nf_ct_is_confirmed(ct)) {
1317                 ct->timeout.expires = extra_jiffies;
1318         } else {
1319                 unsigned long newtime = jiffies + extra_jiffies;
1320
1321                 /* Only update the timeout if the new timeout is at least
1322                    HZ jiffies from the old timeout. Need del_timer for race
1323                    avoidance (may already be dying). */
1324                 if (newtime - ct->timeout.expires >= HZ)
1325                         mod_timer_pending(&ct->timeout, newtime);
1326         }
1327
1328 acct:
1329         if (do_acct)
1330                 nf_ct_acct_update(ct, ctinfo, skb->len);
1331 }
1332 EXPORT_SYMBOL_GPL(__nf_ct_refresh_acct);
1333
1334 bool __nf_ct_kill_acct(struct nf_conn *ct,
1335                        enum ip_conntrack_info ctinfo,
1336                        const struct sk_buff *skb,
1337                        int do_acct)
1338 {
1339         if (do_acct)
1340                 nf_ct_acct_update(ct, ctinfo, skb->len);
1341
1342         if (del_timer(&ct->timeout)) {
1343                 ct->timeout.function((unsigned long)ct);
1344                 return true;
1345         }
1346         return false;
1347 }
1348 EXPORT_SYMBOL_GPL(__nf_ct_kill_acct);
1349
1350 #if IS_ENABLED(CONFIG_NF_CT_NETLINK)
1351
1352 #include <linux/netfilter/nfnetlink.h>
1353 #include <linux/netfilter/nfnetlink_conntrack.h>
1354 #include <linux/mutex.h>
1355
1356 /* Generic function for tcp/udp/sctp/dccp and alike. This needs to be
1357  * in ip_conntrack_core, since we don't want the protocols to autoload
1358  * or depend on ctnetlink */
1359 int nf_ct_port_tuple_to_nlattr(struct sk_buff *skb,
1360                                const struct nf_conntrack_tuple *tuple)
1361 {
1362         if (nla_put_be16(skb, CTA_PROTO_SRC_PORT, tuple->src.u.tcp.port) ||
1363             nla_put_be16(skb, CTA_PROTO_DST_PORT, tuple->dst.u.tcp.port))
1364                 goto nla_put_failure;
1365         return 0;
1366
1367 nla_put_failure:
1368         return -1;
1369 }
1370 EXPORT_SYMBOL_GPL(nf_ct_port_tuple_to_nlattr);
1371
1372 const struct nla_policy nf_ct_port_nla_policy[CTA_PROTO_MAX+1] = {
1373         [CTA_PROTO_SRC_PORT]  = { .type = NLA_U16 },
1374         [CTA_PROTO_DST_PORT]  = { .type = NLA_U16 },
1375 };
1376 EXPORT_SYMBOL_GPL(nf_ct_port_nla_policy);
1377
1378 int nf_ct_port_nlattr_to_tuple(struct nlattr *tb[],
1379                                struct nf_conntrack_tuple *t)
1380 {
1381         if (!tb[CTA_PROTO_SRC_PORT] || !tb[CTA_PROTO_DST_PORT])
1382                 return -EINVAL;
1383
1384         t->src.u.tcp.port = nla_get_be16(tb[CTA_PROTO_SRC_PORT]);
1385         t->dst.u.tcp.port = nla_get_be16(tb[CTA_PROTO_DST_PORT]);
1386
1387         return 0;
1388 }
1389 EXPORT_SYMBOL_GPL(nf_ct_port_nlattr_to_tuple);
1390
1391 int nf_ct_port_nlattr_tuple_size(void)
1392 {
1393         return nla_policy_len(nf_ct_port_nla_policy, CTA_PROTO_MAX + 1);
1394 }
1395 EXPORT_SYMBOL_GPL(nf_ct_port_nlattr_tuple_size);
1396 #endif
1397
1398 /* Used by ipt_REJECT and ip6t_REJECT. */
1399 static void nf_conntrack_attach(struct sk_buff *nskb, const struct sk_buff *skb)
1400 {
1401         struct nf_conn *ct;
1402         enum ip_conntrack_info ctinfo;
1403
1404         /* This ICMP is in reverse direction to the packet which caused it */
1405         ct = nf_ct_get(skb, &ctinfo);
1406         if (CTINFO2DIR(ctinfo) == IP_CT_DIR_ORIGINAL)
1407                 ctinfo = IP_CT_RELATED_REPLY;
1408         else
1409                 ctinfo = IP_CT_RELATED;
1410
1411         /* Attach to new skbuff, and increment count */
1412         nskb->nfct = &ct->ct_general;
1413         nskb->nfctinfo = ctinfo;
1414         nf_conntrack_get(nskb->nfct);
1415 }
1416
1417 /* Bring out ya dead! */
1418 static struct nf_conn *
1419 get_next_corpse(struct net *net, int (*iter)(struct nf_conn *i, void *data),
1420                 void *data, unsigned int *bucket)
1421 {
1422         struct nf_conntrack_tuple_hash *h;
1423         struct nf_conn *ct;
1424         struct hlist_nulls_node *n;
1425         int cpu;
1426         spinlock_t *lockp;
1427
1428         for (; *bucket < nf_conntrack_htable_size; (*bucket)++) {
1429                 lockp = &nf_conntrack_locks[*bucket % CONNTRACK_LOCKS];
1430                 local_bh_disable();
1431                 nf_conntrack_lock(lockp);
1432                 if (*bucket < nf_conntrack_htable_size) {
1433                         hlist_nulls_for_each_entry(h, n, &nf_conntrack_hash[*bucket], hnnode) {
1434                                 if (NF_CT_DIRECTION(h) != IP_CT_DIR_ORIGINAL)
1435                                         continue;
1436                                 ct = nf_ct_tuplehash_to_ctrack(h);
1437                                 if (net_eq(nf_ct_net(ct), net) &&
1438                                     iter(ct, data))
1439                                         goto found;
1440                         }
1441                 }
1442                 spin_unlock(lockp);
1443                 local_bh_enable();
1444                 cond_resched();
1445         }
1446
1447         for_each_possible_cpu(cpu) {
1448                 struct ct_pcpu *pcpu = per_cpu_ptr(net->ct.pcpu_lists, cpu);
1449
1450                 spin_lock_bh(&pcpu->lock);
1451                 hlist_nulls_for_each_entry(h, n, &pcpu->unconfirmed, hnnode) {
1452                         ct = nf_ct_tuplehash_to_ctrack(h);
1453                         if (iter(ct, data))
1454                                 set_bit(IPS_DYING_BIT, &ct->status);
1455                 }
1456                 spin_unlock_bh(&pcpu->lock);
1457                 cond_resched();
1458         }
1459         return NULL;
1460 found:
1461         atomic_inc(&ct->ct_general.use);
1462         spin_unlock(lockp);
1463         local_bh_enable();
1464         return ct;
1465 }
1466
1467 void nf_ct_iterate_cleanup(struct net *net,
1468                            int (*iter)(struct nf_conn *i, void *data),
1469                            void *data, u32 portid, int report)
1470 {
1471         struct nf_conn *ct;
1472         unsigned int bucket = 0;
1473
1474         might_sleep();
1475
1476         if (atomic_read(&net->ct.count) == 0)
1477                 return;
1478
1479         while ((ct = get_next_corpse(net, iter, data, &bucket)) != NULL) {
1480                 /* Time to push up daises... */
1481                 if (del_timer(&ct->timeout))
1482                         nf_ct_delete(ct, portid, report);
1483
1484                 /* ... else the timer will get him soon. */
1485
1486                 nf_ct_put(ct);
1487                 cond_resched();
1488         }
1489 }
1490 EXPORT_SYMBOL_GPL(nf_ct_iterate_cleanup);
1491
1492 static int kill_all(struct nf_conn *i, void *data)
1493 {
1494         return 1;
1495 }
1496
1497 void nf_ct_free_hashtable(void *hash, unsigned int size)
1498 {
1499         if (is_vmalloc_addr(hash))
1500                 vfree(hash);
1501         else
1502                 free_pages((unsigned long)hash,
1503                            get_order(sizeof(struct hlist_head) * size));
1504 }
1505 EXPORT_SYMBOL_GPL(nf_ct_free_hashtable);
1506
1507 static int untrack_refs(void)
1508 {
1509         int cnt = 0, cpu;
1510
1511         for_each_possible_cpu(cpu) {
1512                 struct nf_conn *ct = &per_cpu(nf_conntrack_untracked, cpu);
1513
1514                 cnt += atomic_read(&ct->ct_general.use) - 1;
1515         }
1516         return cnt;
1517 }
1518
1519 void nf_conntrack_cleanup_start(void)
1520 {
1521         RCU_INIT_POINTER(ip_ct_attach, NULL);
1522 }
1523
1524 void nf_conntrack_cleanup_end(void)
1525 {
1526         RCU_INIT_POINTER(nf_ct_destroy, NULL);
1527         while (untrack_refs() > 0)
1528                 schedule();
1529
1530         nf_ct_free_hashtable(nf_conntrack_hash, nf_conntrack_htable_size);
1531
1532         nf_conntrack_proto_fini();
1533         nf_conntrack_seqadj_fini();
1534         nf_conntrack_labels_fini();
1535         nf_conntrack_helper_fini();
1536         nf_conntrack_timeout_fini();
1537         nf_conntrack_ecache_fini();
1538         nf_conntrack_tstamp_fini();
1539         nf_conntrack_acct_fini();
1540         nf_conntrack_expect_fini();
1541
1542         kmem_cache_destroy(nf_conntrack_cachep);
1543 }
1544
1545 /*
1546  * Mishearing the voices in his head, our hero wonders how he's
1547  * supposed to kill the mall.
1548  */
1549 void nf_conntrack_cleanup_net(struct net *net)
1550 {
1551         LIST_HEAD(single);
1552
1553         list_add(&net->exit_list, &single);
1554         nf_conntrack_cleanup_net_list(&single);
1555 }
1556
1557 void nf_conntrack_cleanup_net_list(struct list_head *net_exit_list)
1558 {
1559         int busy;
1560         struct net *net;
1561
1562         /*
1563          * This makes sure all current packets have passed through
1564          *  netfilter framework.  Roll on, two-stage module
1565          *  delete...
1566          */
1567         synchronize_net();
1568 i_see_dead_people:
1569         busy = 0;
1570         list_for_each_entry(net, net_exit_list, exit_list) {
1571                 nf_ct_iterate_cleanup(net, kill_all, NULL, 0, 0);
1572                 if (atomic_read(&net->ct.count) != 0)
1573                         busy = 1;
1574         }
1575         if (busy) {
1576                 schedule();
1577                 goto i_see_dead_people;
1578         }
1579
1580         list_for_each_entry(net, net_exit_list, exit_list) {
1581                 nf_conntrack_proto_pernet_fini(net);
1582                 nf_conntrack_helper_pernet_fini(net);
1583                 nf_conntrack_ecache_pernet_fini(net);
1584                 nf_conntrack_tstamp_pernet_fini(net);
1585                 nf_conntrack_acct_pernet_fini(net);
1586                 nf_conntrack_expect_pernet_fini(net);
1587                 free_percpu(net->ct.stat);
1588                 free_percpu(net->ct.pcpu_lists);
1589         }
1590 }
1591
1592 void *nf_ct_alloc_hashtable(unsigned int *sizep, int nulls)
1593 {
1594         struct hlist_nulls_head *hash;
1595         unsigned int nr_slots, i;
1596         size_t sz;
1597
1598         if (*sizep > (UINT_MAX / sizeof(struct hlist_nulls_head)))
1599                 return NULL;
1600
1601         BUILD_BUG_ON(sizeof(struct hlist_nulls_head) != sizeof(struct hlist_head));
1602         nr_slots = *sizep = roundup(*sizep, PAGE_SIZE / sizeof(struct hlist_nulls_head));
1603
1604         if (nr_slots > (UINT_MAX / sizeof(struct hlist_nulls_head)))
1605                 return NULL;
1606
1607         sz = nr_slots * sizeof(struct hlist_nulls_head);
1608         hash = (void *)__get_free_pages(GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO,
1609                                         get_order(sz));
1610         if (!hash)
1611                 hash = vzalloc(sz);
1612
1613         if (hash && nulls)
1614                 for (i = 0; i < nr_slots; i++)
1615                         INIT_HLIST_NULLS_HEAD(&hash[i], i);
1616
1617         return hash;
1618 }
1619 EXPORT_SYMBOL_GPL(nf_ct_alloc_hashtable);
1620
1621 int nf_conntrack_hash_resize(unsigned int hashsize)
1622 {
1623         int i, bucket;
1624         unsigned int old_size;
1625         struct hlist_nulls_head *hash, *old_hash;
1626         struct nf_conntrack_tuple_hash *h;
1627         struct nf_conn *ct;
1628
1629         if (!hashsize)
1630                 return -EINVAL;
1631
1632         hash = nf_ct_alloc_hashtable(&hashsize, 1);
1633         if (!hash)
1634                 return -ENOMEM;
1635
1636         old_size = nf_conntrack_htable_size;
1637         if (old_size == hashsize) {
1638                 nf_ct_free_hashtable(hash, hashsize);
1639                 return 0;
1640         }
1641
1642         local_bh_disable();
1643         nf_conntrack_all_lock();
1644         write_seqcount_begin(&nf_conntrack_generation);
1645
1646         /* Lookups in the old hash might happen in parallel, which means we
1647          * might get false negatives during connection lookup. New connections
1648          * created because of a false negative won't make it into the hash
1649          * though since that required taking the locks.
1650          */
1651
1652         for (i = 0; i < nf_conntrack_htable_size; i++) {
1653                 while (!hlist_nulls_empty(&nf_conntrack_hash[i])) {
1654                         h = hlist_nulls_entry(nf_conntrack_hash[i].first,
1655                                               struct nf_conntrack_tuple_hash, hnnode);
1656                         ct = nf_ct_tuplehash_to_ctrack(h);
1657                         hlist_nulls_del_rcu(&h->hnnode);
1658                         bucket = __hash_conntrack(nf_ct_net(ct),
1659                                                   &h->tuple, hashsize);
1660                         hlist_nulls_add_head_rcu(&h->hnnode, &hash[bucket]);
1661                 }
1662         }
1663         old_size = nf_conntrack_htable_size;
1664         old_hash = nf_conntrack_hash;
1665
1666         nf_conntrack_hash = hash;
1667         nf_conntrack_htable_size = hashsize;
1668
1669         write_seqcount_end(&nf_conntrack_generation);
1670         nf_conntrack_all_unlock();
1671         local_bh_enable();
1672
1673         synchronize_net();
1674         nf_ct_free_hashtable(old_hash, old_size);
1675         return 0;
1676 }
1677
1678 int nf_conntrack_set_hashsize(const char *val, struct kernel_param *kp)
1679 {
1680         unsigned int hashsize;
1681         int rc;
1682
1683         if (current->nsproxy->net_ns != &init_net)
1684                 return -EOPNOTSUPP;
1685
1686         /* On boot, we can set this without any fancy locking. */
1687         if (!nf_conntrack_htable_size)
1688                 return param_set_uint(val, kp);
1689
1690         rc = kstrtouint(val, 0, &hashsize);
1691         if (rc)
1692                 return rc;
1693
1694         return nf_conntrack_hash_resize(hashsize);
1695 }
1696 EXPORT_SYMBOL_GPL(nf_conntrack_set_hashsize);
1697
1698 module_param_call(hashsize, nf_conntrack_set_hashsize, param_get_uint,
1699                   &nf_conntrack_htable_size, 0600);
1700
1701 void nf_ct_untracked_status_or(unsigned long bits)
1702 {
1703         int cpu;
1704
1705         for_each_possible_cpu(cpu)
1706                 per_cpu(nf_conntrack_untracked, cpu).status |= bits;
1707 }
1708 EXPORT_SYMBOL_GPL(nf_ct_untracked_status_or);
1709
1710 int nf_conntrack_init_start(void)
1711 {
1712         int max_factor = 8;
1713         int ret = -ENOMEM;
1714         int i, cpu;
1715
1716         seqcount_init(&nf_conntrack_generation);
1717
1718         for (i = 0; i < CONNTRACK_LOCKS; i++)
1719                 spin_lock_init(&nf_conntrack_locks[i]);
1720
1721         if (!nf_conntrack_htable_size) {
1722                 /* Idea from tcp.c: use 1/16384 of memory.
1723                  * On i386: 32MB machine has 512 buckets.
1724                  * >= 1GB machines have 16384 buckets.
1725                  * >= 4GB machines have 65536 buckets.
1726                  */
1727                 nf_conntrack_htable_size
1728                         = (((totalram_pages << PAGE_SHIFT) / 16384)
1729                            / sizeof(struct hlist_head));
1730                 if (totalram_pages > (4 * (1024 * 1024 * 1024 / PAGE_SIZE)))
1731                         nf_conntrack_htable_size = 65536;
1732                 else if (totalram_pages > (1024 * 1024 * 1024 / PAGE_SIZE))
1733                         nf_conntrack_htable_size = 16384;
1734                 if (nf_conntrack_htable_size < 32)
1735                         nf_conntrack_htable_size = 32;
1736
1737                 /* Use a max. factor of four by default to get the same max as
1738                  * with the old struct list_heads. When a table size is given
1739                  * we use the old value of 8 to avoid reducing the max.
1740                  * entries. */
1741                 max_factor = 4;
1742         }
1743
1744         nf_conntrack_hash = nf_ct_alloc_hashtable(&nf_conntrack_htable_size, 1);
1745         if (!nf_conntrack_hash)
1746                 return -ENOMEM;
1747
1748         nf_conntrack_max = max_factor * nf_conntrack_htable_size;
1749
1750         nf_conntrack_cachep = kmem_cache_create("nf_conntrack",
1751                                                 sizeof(struct nf_conn), 0,
1752                                                 SLAB_DESTROY_BY_RCU | SLAB_HWCACHE_ALIGN, NULL);
1753         if (!nf_conntrack_cachep)
1754                 goto err_cachep;
1755
1756         printk(KERN_INFO "nf_conntrack version %s (%u buckets, %d max)\n",
1757                NF_CONNTRACK_VERSION, nf_conntrack_htable_size,
1758                nf_conntrack_max);
1759
1760         ret = nf_conntrack_expect_init();
1761         if (ret < 0)
1762                 goto err_expect;
1763
1764         ret = nf_conntrack_acct_init();
1765         if (ret < 0)
1766                 goto err_acct;
1767
1768         ret = nf_conntrack_tstamp_init();
1769         if (ret < 0)
1770                 goto err_tstamp;
1771
1772         ret = nf_conntrack_ecache_init();
1773         if (ret < 0)
1774                 goto err_ecache;
1775
1776         ret = nf_conntrack_timeout_init();
1777         if (ret < 0)
1778                 goto err_timeout;
1779
1780         ret = nf_conntrack_helper_init();
1781         if (ret < 0)
1782                 goto err_helper;
1783
1784         ret = nf_conntrack_labels_init();
1785         if (ret < 0)
1786                 goto err_labels;
1787
1788         ret = nf_conntrack_seqadj_init();
1789         if (ret < 0)
1790                 goto err_seqadj;
1791
1792         ret = nf_conntrack_proto_init();
1793         if (ret < 0)
1794                 goto err_proto;
1795
1796         /* Set up fake conntrack: to never be deleted, not in any hashes */
1797         for_each_possible_cpu(cpu) {
1798                 struct nf_conn *ct = &per_cpu(nf_conntrack_untracked, cpu);
1799                 write_pnet(&ct->ct_net, &init_net);
1800                 atomic_set(&ct->ct_general.use, 1);
1801         }
1802         /*  - and look it like as a confirmed connection */
1803         nf_ct_untracked_status_or(IPS_CONFIRMED | IPS_UNTRACKED);
1804         return 0;
1805
1806 err_proto:
1807         nf_conntrack_seqadj_fini();
1808 err_seqadj:
1809         nf_conntrack_labels_fini();
1810 err_labels:
1811         nf_conntrack_helper_fini();
1812 err_helper:
1813         nf_conntrack_timeout_fini();
1814 err_timeout:
1815         nf_conntrack_ecache_fini();
1816 err_ecache:
1817         nf_conntrack_tstamp_fini();
1818 err_tstamp:
1819         nf_conntrack_acct_fini();
1820 err_acct:
1821         nf_conntrack_expect_fini();
1822 err_expect:
1823         kmem_cache_destroy(nf_conntrack_cachep);
1824 err_cachep:
1825         nf_ct_free_hashtable(nf_conntrack_hash, nf_conntrack_htable_size);
1826         return ret;
1827 }
1828
1829 void nf_conntrack_init_end(void)
1830 {
1831         /* For use by REJECT target */
1832         RCU_INIT_POINTER(ip_ct_attach, nf_conntrack_attach);
1833         RCU_INIT_POINTER(nf_ct_destroy, destroy_conntrack);
1834 }
1835
1836 /*
1837  * We need to use special "null" values, not used in hash table
1838  */
1839 #define UNCONFIRMED_NULLS_VAL   ((1<<30)+0)
1840 #define DYING_NULLS_VAL         ((1<<30)+1)
1841 #define TEMPLATE_NULLS_VAL      ((1<<30)+2)
1842
1843 int nf_conntrack_init_net(struct net *net)
1844 {
1845         int ret = -ENOMEM;
1846         int cpu;
1847
1848         atomic_set(&net->ct.count, 0);
1849
1850         net->ct.pcpu_lists = alloc_percpu(struct ct_pcpu);
1851         if (!net->ct.pcpu_lists)
1852                 goto err_stat;
1853
1854         for_each_possible_cpu(cpu) {
1855                 struct ct_pcpu *pcpu = per_cpu_ptr(net->ct.pcpu_lists, cpu);
1856
1857                 spin_lock_init(&pcpu->lock);
1858                 INIT_HLIST_NULLS_HEAD(&pcpu->unconfirmed, UNCONFIRMED_NULLS_VAL);
1859                 INIT_HLIST_NULLS_HEAD(&pcpu->dying, DYING_NULLS_VAL);
1860         }
1861
1862         net->ct.stat = alloc_percpu(struct ip_conntrack_stat);
1863         if (!net->ct.stat)
1864                 goto err_pcpu_lists;
1865
1866         ret = nf_conntrack_expect_pernet_init(net);
1867         if (ret < 0)
1868                 goto err_expect;
1869         ret = nf_conntrack_acct_pernet_init(net);
1870         if (ret < 0)
1871                 goto err_acct;
1872         ret = nf_conntrack_tstamp_pernet_init(net);
1873         if (ret < 0)
1874                 goto err_tstamp;
1875         ret = nf_conntrack_ecache_pernet_init(net);
1876         if (ret < 0)
1877                 goto err_ecache;
1878         ret = nf_conntrack_helper_pernet_init(net);
1879         if (ret < 0)
1880                 goto err_helper;
1881         ret = nf_conntrack_proto_pernet_init(net);
1882         if (ret < 0)
1883                 goto err_proto;
1884         return 0;
1885
1886 err_proto:
1887         nf_conntrack_helper_pernet_fini(net);
1888 err_helper:
1889         nf_conntrack_ecache_pernet_fini(net);
1890 err_ecache:
1891         nf_conntrack_tstamp_pernet_fini(net);
1892 err_tstamp:
1893         nf_conntrack_acct_pernet_fini(net);
1894 err_acct:
1895         nf_conntrack_expect_pernet_fini(net);
1896 err_expect:
1897         free_percpu(net->ct.stat);
1898 err_pcpu_lists:
1899         free_percpu(net->ct.pcpu_lists);
1900 err_stat:
1901         return ret;
1902 }