net/netfilter/ipvs/ip_vs_xmit.c

   1 /*
   2  * ip_vs_xmit.c: various packet transmitters for IPVS
   3  *
   4  * Authors:     Wensong Zhang <wensong@linuxvirtualserver.org>
   5  *              Julian Anastasov <ja@ssi.bg>
   6  *
   7  *              This program is free software; you can redistribute it and/or
   8  *              modify it under the terms of the GNU General Public License
   9  *              as published by the Free Software Foundation; either version
  10  *              2 of the License, or (at your option) any later version.
  11  *
  12  * Changes:
  13  *
  14  * Description of forwarding methods:
  15  * - all transmitters are called from LOCAL_IN (remote clients) and
  16  * LOCAL_OUT (local clients) but for ICMP can be called from FORWARD
  17  * - not all connections have destination server, for example,
  18  * connections in backup server when fwmark is used
  19  * - bypass connections use daddr from packet
  20  * LOCAL_OUT rules:
  21  * - skb->dev is NULL, skb->protocol is not set (both are set in POST_ROUTING)
  22  * - skb->pkt_type is not set yet
  23  * - the only place where we can see skb->sk != NULL
  24  */
  25
  26 #define KMSG_COMPONENT "IPVS"
  27 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  28
  29 #include <linux/kernel.h>
  30 #include <linux/slab.h>
  31 #include <linux/tcp.h>                  /* for tcphdr */
  32 #include <net/ip.h>
  33 #include <net/tcp.h>                    /* for csum_tcpudp_magic */
  34 #include <net/udp.h>
  35 #include <net/icmp.h>                   /* for icmp_send */
  36 #include <net/route.h>                  /* for ip_route_output */
  37 #include <net/ipv6.h>
  38 #include <net/ip6_route.h>
  39 #include <net/addrconf.h>
  40 #include <linux/icmpv6.h>
  41 #include <linux/netfilter.h>
  42 #include <linux/netfilter_ipv4.h>
  43
  44 #include <net/ip_vs.h>
  45
  46 enum {
  47         IP_VS_RT_MODE_LOCAL     = 1, /* Allow local dest */
  48         IP_VS_RT_MODE_NON_LOCAL = 2, /* Allow non-local dest */
  49         IP_VS_RT_MODE_RDR       = 4, /* Allow redirect from remote daddr to
  50                                       * local
  51                                       */
  52 };
  53
  54 /*
  55  *      Destination cache to speed up outgoing route lookup
  56  */
  57 static inline void
  58 __ip_vs_dst_set(struct ip_vs_dest *dest, u32 rtos, struct dst_entry *dst,
  59                 u32 dst_cookie)
  60 {
  61         struct dst_entry *old_dst;
  62
  63         old_dst = dest->dst_cache;
  64         dest->dst_cache = dst;
  65         dest->dst_rtos = rtos;
  66         dest->dst_cookie = dst_cookie;
  67         dst_release(old_dst);
  68 }
  69
  70 static inline struct dst_entry *
  71 __ip_vs_dst_check(struct ip_vs_dest *dest, u32 rtos)
  72 {
  73         struct dst_entry *dst = dest->dst_cache;
  74
  75         if (!dst)
  76                 return NULL;
  77         if ((dst->obsolete || rtos != dest->dst_rtos) &&
  78             dst->ops->check(dst, dest->dst_cookie) == NULL) {
  79                 dest->dst_cache = NULL;
  80                 dst_release(dst);
  81                 return NULL;
  82         }
  83         dst_hold(dst);
  84         return dst;
  85 }
  86
  87 /* Get route to destination or remote server */
  88 static struct rtable *
  89 __ip_vs_get_out_rt(struct sk_buff *skb, struct ip_vs_dest *dest,
  90                    __be32 daddr, u32 rtos, int rt_mode, __be32 *ret_saddr)
  91 {
  92         struct net *net = dev_net(skb_dst(skb)->dev);
  93         struct rtable *rt;                      /* Route to the other host */
  94         struct rtable *ort;                     /* Original route */
  95         int local;
  96
  97         if (dest) {
  98                 spin_lock(&dest->dst_lock);
  99                 if (!(rt = (struct rtable *)
 100                       __ip_vs_dst_check(dest, rtos))) {
 101                         struct flowi4 fl4;
 102
 103                         memset(&fl4, 0, sizeof(fl4));
 104                         fl4.daddr = dest->addr.ip;
 105                         fl4.flowi4_tos = rtos;
 106                         rt = ip_route_output_key(net, &fl4);
 107                         if (IS_ERR(rt)) {
 108                                 spin_unlock(&dest->dst_lock);
 109                                 IP_VS_DBG_RL("ip_route_output error, dest: %pI4\n",
 110                                              &dest->addr.ip);
 111                                 return NULL;
 112                         }
 113                         __ip_vs_dst_set(dest, rtos, dst_clone(&rt->dst), 0);
 114                         dest->dst_saddr.ip = fl4.saddr;
 115                         IP_VS_DBG(10, "new dst %pI4, src %pI4, refcnt=%d, "
 116                                   "rtos=%X\n",
 117                                   &dest->addr.ip, &dest->dst_saddr.ip,
 118                                   atomic_read(&rt->dst.__refcnt), rtos);
 119                 }
 120                 daddr = dest->addr.ip;
 121                 if (ret_saddr)
 122                         *ret_saddr = dest->dst_saddr.ip;
 123                 spin_unlock(&dest->dst_lock);
 124         } else {
 125                 struct flowi4 fl4;
 126
 127                 memset(&fl4, 0, sizeof(fl4));
 128                 fl4.daddr = daddr;
 129                 fl4.flowi4_tos = rtos;
 130                 rt = ip_route_output_key(net, &fl4);
 131                 if (IS_ERR(rt)) {
 132                         IP_VS_DBG_RL("ip_route_output error, dest: %pI4\n",
 133                                      &daddr);
 134                         return NULL;
 135                 }
 136                 if (ret_saddr)
 137                         *ret_saddr = fl4.saddr;
 138         }
 139
 140         local = rt->rt_flags & RTCF_LOCAL;
 141         if (!((local ? IP_VS_RT_MODE_LOCAL : IP_VS_RT_MODE_NON_LOCAL) &
 142               rt_mode)) {
 143                 IP_VS_DBG_RL("Stopping traffic to %s address, dest: %pI4\n",
 144                              (rt->rt_flags & RTCF_LOCAL) ?
 145                              "local":"non-local", &daddr);
 146                 ip_rt_put(rt);
 147                 return NULL;
 148         }
 149         if (local && !(rt_mode & IP_VS_RT_MODE_RDR) &&
 150             !((ort = skb_rtable(skb)) && ort->rt_flags & RTCF_LOCAL)) {
 151                 IP_VS_DBG_RL("Redirect from non-local address %pI4 to local "
 152                              "requires NAT method, dest: %pI4\n",
 153                              &ip_hdr(skb)->daddr, &daddr);
 154                 ip_rt_put(rt);
 155                 return NULL;
 156         }
 157         if (unlikely(!local && ipv4_is_loopback(ip_hdr(skb)->saddr))) {
 158                 IP_VS_DBG_RL("Stopping traffic from loopback address %pI4 "
 159                              "to non-local address, dest: %pI4\n",
 160                              &ip_hdr(skb)->saddr, &daddr);
 161                 ip_rt_put(rt);
 162                 return NULL;
 163         }
 164
 165         return rt;
 166 }
 167
 168 /* Reroute packet to local IPv4 stack after DNAT */
 169 static int
 170 __ip_vs_reroute_locally(struct sk_buff *skb)
 171 {
 172         struct rtable *rt = skb_rtable(skb);
 173         struct net_device *dev = rt->dst.dev;
 174         struct net *net = dev_net(dev);
 175         struct iphdr *iph = ip_hdr(skb);
 176
 177         if (rt_is_input_route(rt)) {
 178                 unsigned long orefdst = skb->_skb_refdst;
 179
 180                 if (ip_route_input(skb, iph->daddr, iph->saddr,
 181                                    iph->tos, skb->dev))
 182                         return 0;
 183                 refdst_drop(orefdst);
 184         } else {
 185                 struct flowi4 fl4 = {
 186                         .daddr = iph->daddr,
 187                         .saddr = iph->saddr,
 188                         .flowi4_tos = RT_TOS(iph->tos),
 189                         .flowi4_mark = skb->mark,
 190                 };
 191
 192                 rt = ip_route_output_key(net, &fl4);
 193                 if (IS_ERR(rt))
 194                         return 0;
 195                 if (!(rt->rt_flags & RTCF_LOCAL)) {
 196                         ip_rt_put(rt);
 197                         return 0;
 198                 }
 199                 /* Drop old route. */
 200                 skb_dst_drop(skb);
 201                 skb_dst_set(skb, &rt->dst);
 202         }
 203         return 1;
 204 }
 205
 206 #ifdef CONFIG_IP_VS_IPV6
 207
 208 static inline int __ip_vs_is_local_route6(struct rt6_info *rt)
 209 {
 210         return rt->rt6i_dev && rt->rt6i_dev->flags & IFF_LOOPBACK;
 211 }
 212
 213 static struct dst_entry *
 214 __ip_vs_route_output_v6(struct net *net, struct in6_addr *daddr,
 215                         struct in6_addr *ret_saddr, int do_xfrm)
 216 {
 217         struct dst_entry *dst;
 218         struct flowi6 fl6 = {
 219                 .daddr = *daddr,
 220         };
 221
 222         dst = ip6_route_output(net, NULL, &fl6);
 223         if (dst->error)
 224                 goto out_err;
 225         if (!ret_saddr)
 226                 return dst;
 227         if (ipv6_addr_any(&fl6.saddr) &&
 228             ipv6_dev_get_saddr(net, ip6_dst_idev(dst)->dev,
 229                                &fl6.daddr, 0, &fl6.saddr) < 0)
 230                 goto out_err;
 231         if (do_xfrm) {
 232                 dst = xfrm_lookup(net, dst, flowi6_to_flowi(&fl6), NULL, 0);
 233                 if (IS_ERR(dst)) {
 234                         dst = NULL;
 235                         goto out_err;
 236                 }
 237         }
 238         *ret_saddr = fl6.saddr;
 239         return dst;
 240
 241 out_err:
 242         dst_release(dst);
 243         IP_VS_DBG_RL("ip6_route_output error, dest: %pI6\n", daddr);
 244         return NULL;
 245 }
 246
 247 /*
 248  * Get route to destination or remote server
 249  */
 250 static struct rt6_info *
 251 __ip_vs_get_out_rt_v6(struct sk_buff *skb, struct ip_vs_dest *dest,
 252                       struct in6_addr *daddr, struct in6_addr *ret_saddr,
 253                       int do_xfrm, int rt_mode)
 254 {
 255         struct net *net = dev_net(skb_dst(skb)->dev);
 256         struct rt6_info *rt;                    /* Route to the other host */
 257         struct rt6_info *ort;                   /* Original route */
 258         struct dst_entry *dst;
 259         int local;
 260
 261         if (dest) {
 262                 spin_lock(&dest->dst_lock);
 263                 rt = (struct rt6_info *)__ip_vs_dst_check(dest, 0);
 264                 if (!rt) {
 265                         u32 cookie;
 266
 267                         dst = __ip_vs_route_output_v6(net, &dest->addr.in6,
 268                                                       &dest->dst_saddr.in6,
 269                                                       do_xfrm);
 270                         if (!dst) {
 271                                 spin_unlock(&dest->dst_lock);
 272                                 return NULL;
 273                         }
 274                         rt = (struct rt6_info *) dst;
 275                         cookie = rt->rt6i_node ? rt->rt6i_node->fn_sernum : 0;
 276                         __ip_vs_dst_set(dest, 0, dst_clone(&rt->dst), cookie);
 277                         IP_VS_DBG(10, "new dst %pI6, src %pI6, refcnt=%d\n",
 278                                   &dest->addr.in6, &dest->dst_saddr.in6,
 279                                   atomic_read(&rt->dst.__refcnt));
 280                 }
 281                 if (ret_saddr)
 282                         *ret_saddr = dest->dst_saddr.in6;
 283                 spin_unlock(&dest->dst_lock);
 284         } else {
 285                 dst = __ip_vs_route_output_v6(net, daddr, ret_saddr, do_xfrm);
 286                 if (!dst)
 287                         return NULL;
 288                 rt = (struct rt6_info *) dst;
 289         }
 290
 291         local = __ip_vs_is_local_route6(rt);
 292         if (!((local ? IP_VS_RT_MODE_LOCAL : IP_VS_RT_MODE_NON_LOCAL) &
 293               rt_mode)) {
 294                 IP_VS_DBG_RL("Stopping traffic to %s address, dest: %pI6\n",
 295                              local ? "local":"non-local", daddr);
 296                 dst_release(&rt->dst);
 297                 return NULL;
 298         }
 299         if (local && !(rt_mode & IP_VS_RT_MODE_RDR) &&
 300             !((ort = (struct rt6_info *) skb_dst(skb)) &&
 301               __ip_vs_is_local_route6(ort))) {
 302                 IP_VS_DBG_RL("Redirect from non-local address %pI6 to local "
 303                              "requires NAT method, dest: %pI6\n",
 304                              &ipv6_hdr(skb)->daddr, daddr);
 305                 dst_release(&rt->dst);
 306                 return NULL;
 307         }
 308         if (unlikely(!local && (!skb->dev || skb->dev->flags & IFF_LOOPBACK) &&
 309                      ipv6_addr_type(&ipv6_hdr(skb)->saddr) &
 310                                     IPV6_ADDR_LOOPBACK)) {
 311                 IP_VS_DBG_RL("Stopping traffic from loopback address %pI6 "
 312                              "to non-local address, dest: %pI6\n",
 313                              &ipv6_hdr(skb)->saddr, daddr);
 314                 dst_release(&rt->dst);
 315                 return NULL;
 316         }
 317
 318         return rt;
 319 }
 320 #endif
 321
 322
 323 /*
 324  *      Release dest->dst_cache before a dest is removed
 325  */
 326 void
 327 ip_vs_dst_reset(struct ip_vs_dest *dest)
 328 {
 329         struct dst_entry *old_dst;
 330
 331         old_dst = dest->dst_cache;
 332         dest->dst_cache = NULL;
 333         dst_release(old_dst);
 334 }
 335
 336 #define IP_VS_XMIT_TUNNEL(skb, cp)                              \
 337 ({                                                              \
 338         int __ret = NF_ACCEPT;                                  \
 339                                                                 \
 340         (skb)->ipvs_property = 1;                               \
 341         if (unlikely((cp)->flags & IP_VS_CONN_F_NFCT))          \
 342                 __ret = ip_vs_confirm_conntrack(skb);           \
 343         if (__ret == NF_ACCEPT) {                               \
 344                 nf_reset(skb);                                  \
 345                 skb_forward_csum(skb);                          \
 346         }                                                       \
 347         __ret;                                                  \
 348 })
 349
 350 #define IP_VS_XMIT_NAT(pf, skb, cp, local)              \
 351 do {                                                    \
 352         (skb)->ipvs_property = 1;                       \
 353         if (likely(!((cp)->flags & IP_VS_CONN_F_NFCT))) \
 354                 ip_vs_notrack(skb);                     \
 355         else                                            \
 356                 ip_vs_update_conntrack(skb, cp, 1);     \
 357         if (local)                                      \
 358                 return NF_ACCEPT;                       \
 359         skb_forward_csum(skb);                          \
 360         NF_HOOK(pf, NF_INET_LOCAL_OUT, (skb), NULL,     \
 361                 skb_dst(skb)->dev, dst_output);         \
 362 } while (0)
 363
 364 #define IP_VS_XMIT(pf, skb, cp, local)                  \
 365 do {                                                    \
 366         (skb)->ipvs_property = 1;                       \
 367         if (likely(!((cp)->flags & IP_VS_CONN_F_NFCT))) \
 368                 ip_vs_notrack(skb);                     \
 369         if (local)                                      \
 370                 return NF_ACCEPT;                       \
 371         skb_forward_csum(skb);                          \
 372         NF_HOOK(pf, NF_INET_LOCAL_OUT, (skb), NULL,     \
 373                 skb_dst(skb)->dev, dst_output);         \
 374 } while (0)
 375
 376
 377 /*
 378  *      NULL transmitter (do nothing except return NF_ACCEPT)
 379  */
 380 int
 381 ip_vs_null_xmit(struct sk_buff *skb, struct ip_vs_conn *cp,
 382                 struct ip_vs_protocol *pp)
 383 {
 384         /* we do not touch skb and do not need pskb ptr */
 385         IP_VS_XMIT(NFPROTO_IPV4, skb, cp, 1);
 386 }
 387
 388
 389 /*
 390  *      Bypass transmitter
 391  *      Let packets bypass the destination when the destination is not
 392  *      available, it may be only used in transparent cache cluster.
 393  */
 394 int
 395 ip_vs_bypass_xmit(struct sk_buff *skb, struct ip_vs_conn *cp,
 396                   struct ip_vs_protocol *pp)
 397 {
 398         struct rtable *rt;                      /* Route to the other host */
 399         struct iphdr  *iph = ip_hdr(skb);
 400         int    mtu;
 401
 402         EnterFunction(10);
 403
 404         if (!(rt = __ip_vs_get_out_rt(skb, NULL, iph->daddr, RT_TOS(iph->tos),
 405                                       IP_VS_RT_MODE_NON_LOCAL, NULL)))
 406                 goto tx_error_icmp;
 407
 408         /* MTU checking */
 409         mtu = dst_mtu(&rt->dst);
 410         if ((skb->len > mtu) && (iph->frag_off & htons(IP_DF)) &&
 411             !skb_is_gso(skb)) {
 412                 ip_rt_put(rt);
 413                 icmp_send(skb, ICMP_DEST_UNREACH,ICMP_FRAG_NEEDED, htonl(mtu));
 414                 IP_VS_DBG_RL("%s(): frag needed\n", __func__);
 415                 goto tx_error;
 416         }
 417
 418         /*
 419          * Call ip_send_check because we are not sure it is called
 420          * after ip_defrag. Is copy-on-write needed?
 421          */
 422         if (unlikely((skb = skb_share_check(skb, GFP_ATOMIC)) == NULL)) {
 423                 ip_rt_put(rt);
 424                 return NF_STOLEN;
 425         }
 426         ip_send_check(ip_hdr(skb));
 427
 428         /* drop old route */
 429         skb_dst_drop(skb);
 430         skb_dst_set(skb, &rt->dst);
 431
 432         /* Another hack: avoid icmp_send in ip_fragment */
 433         skb->local_df = 1;
 434
 435         IP_VS_XMIT(NFPROTO_IPV4, skb, cp, 0);
 436
 437         LeaveFunction(10);
 438         return NF_STOLEN;
 439
 440  tx_error_icmp:
 441         dst_link_failure(skb);
 442  tx_error:
 443         kfree_skb(skb);
 444         LeaveFunction(10);
 445         return NF_STOLEN;
 446 }
 447
 448 #ifdef CONFIG_IP_VS_IPV6
 449 int
 450 ip_vs_bypass_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp,
 451                      struct ip_vs_protocol *pp)
 452 {
 453         struct rt6_info *rt;                    /* Route to the other host */
 454         struct ipv6hdr  *iph = ipv6_hdr(skb);
 455         int    mtu;
 456
 457         EnterFunction(10);
 458
 459         if (!(rt = __ip_vs_get_out_rt_v6(skb, NULL, &iph->daddr, NULL, 0,
 460                                          IP_VS_RT_MODE_NON_LOCAL)))
 461                 goto tx_error_icmp;
 462
 463         /* MTU checking */
 464         mtu = dst_mtu(&rt->dst);
 465         if (skb->len > mtu && !skb_is_gso(skb)) {
 466                 if (!skb->dev) {
 467                         struct net *net = dev_net(skb_dst(skb)->dev);
 468
 469                         skb->dev = net->loopback_dev;
 470                 }
 471                 icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu);
 472                 dst_release(&rt->dst);
 473                 IP_VS_DBG_RL("%s(): frag needed\n", __func__);
 474                 goto tx_error;
 475         }
 476
 477         /*
 478          * Call ip_send_check because we are not sure it is called
 479          * after ip_defrag. Is copy-on-write needed?
 480          */
 481         skb = skb_share_check(skb, GFP_ATOMIC);
 482         if (unlikely(skb == NULL)) {
 483                 dst_release(&rt->dst);
 484                 return NF_STOLEN;
 485         }
 486
 487         /* drop old route */
 488         skb_dst_drop(skb);
 489         skb_dst_set(skb, &rt->dst);
 490
 491         /* Another hack: avoid icmp_send in ip_fragment */
 492         skb->local_df = 1;
 493
 494         IP_VS_XMIT(NFPROTO_IPV6, skb, cp, 0);
 495
 496         LeaveFunction(10);
 497         return NF_STOLEN;
 498
 499  tx_error_icmp:
 500         dst_link_failure(skb);
 501  tx_error:
 502         kfree_skb(skb);
 503         LeaveFunction(10);
 504         return NF_STOLEN;
 505 }
 506 #endif
 507
 508 /*
 509  *      NAT transmitter (only for outside-to-inside nat forwarding)
 510  *      Not used for related ICMP
 511  */
 512 int
 513 ip_vs_nat_xmit(struct sk_buff *skb, struct ip_vs_conn *cp,
 514                struct ip_vs_protocol *pp)
 515 {
 516         struct rtable *rt;              /* Route to the other host */
 517         int mtu;
 518         struct iphdr *iph = ip_hdr(skb);
 519         int local;
 520
 521         EnterFunction(10);
 522
 523         /* check if it is a connection of no-client-port */
 524         if (unlikely(cp->flags & IP_VS_CONN_F_NO_CPORT)) {
 525                 __be16 _pt, *p;
 526                 p = skb_header_pointer(skb, iph->ihl*4, sizeof(_pt), &_pt);
 527                 if (p == NULL)
 528                         goto tx_error;
 529                 ip_vs_conn_fill_cport(cp, *p);
 530                 IP_VS_DBG(10, "filled cport=%d\n", ntohs(*p));
 531         }
 532
 533         if (!(rt = __ip_vs_get_out_rt(skb, cp->dest, cp->daddr.ip,
 534                                       RT_TOS(iph->tos),
 535                                       IP_VS_RT_MODE_LOCAL |
 536                                         IP_VS_RT_MODE_NON_LOCAL |
 537                                         IP_VS_RT_MODE_RDR, NULL)))
 538                 goto tx_error_icmp;
 539         local = rt->rt_flags & RTCF_LOCAL;
 540         /*
 541          * Avoid duplicate tuple in reply direction for NAT traffic
 542          * to local address when connection is sync-ed
 543          */
 544 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
 545         if (cp->flags & IP_VS_CONN_F_SYNC && local) {
 546                 enum ip_conntrack_info ctinfo;
 547                 struct nf_conn *ct = ct = nf_ct_get(skb, &ctinfo);
 548
 549                 if (ct && !nf_ct_is_untracked(ct)) {
 550                         IP_VS_DBG_RL_PKT(10, AF_INET, pp, skb, 0,
 551                                          "ip_vs_nat_xmit(): "
 552                                          "stopping DNAT to local address");
 553                         goto tx_error_put;
 554                 }
 555         }
 556 #endif
 557
 558         /* From world but DNAT to loopback address? */
 559         if (local && ipv4_is_loopback(cp->daddr.ip) &&
 560             rt_is_input_route(skb_rtable(skb))) {
 561                 IP_VS_DBG_RL_PKT(1, AF_INET, pp, skb, 0, "ip_vs_nat_xmit(): "
 562                                  "stopping DNAT to loopback address");
 563                 goto tx_error_put;
 564         }
 565
 566         /* MTU checking */
 567         mtu = dst_mtu(&rt->dst);
 568         if ((skb->len > mtu) && (iph->frag_off & htons(IP_DF)) &&
 569             !skb_is_gso(skb)) {
 570                 icmp_send(skb, ICMP_DEST_UNREACH,ICMP_FRAG_NEEDED, htonl(mtu));
 571                 IP_VS_DBG_RL_PKT(0, AF_INET, pp, skb, 0,
 572                                  "ip_vs_nat_xmit(): frag needed for");
 573                 goto tx_error_put;
 574         }
 575
 576         /* copy-on-write the packet before mangling it */
 577         if (!skb_make_writable(skb, sizeof(struct iphdr)))
 578                 goto tx_error_put;
 579
 580         if (skb_cow(skb, rt->dst.dev->hard_header_len))
 581                 goto tx_error_put;
 582
 583         /* mangle the packet */
 584         if (pp->dnat_handler && !pp->dnat_handler(skb, pp, cp))
 585                 goto tx_error_put;
 586         ip_hdr(skb)->daddr = cp->daddr.ip;
 587         ip_send_check(ip_hdr(skb));
 588
 589         if (!local) {
 590                 /* drop old route */
 591                 skb_dst_drop(skb);
 592                 skb_dst_set(skb, &rt->dst);
 593         } else {
 594                 ip_rt_put(rt);
 595                 /*
 596                  * Some IPv4 replies get local address from routes,
 597                  * not from iph, so while we DNAT after routing
 598                  * we need this second input/output route.
 599                  */
 600                 if (!__ip_vs_reroute_locally(skb))
 601                         goto tx_error;
 602         }
 603
 604         IP_VS_DBG_PKT(10, AF_INET, pp, skb, 0, "After DNAT");
 605
 606         /* FIXME: when application helper enlarges the packet and the length
 607            is larger than the MTU of outgoing device, there will be still
 608            MTU problem. */
 609
 610         /* Another hack: avoid icmp_send in ip_fragment */
 611         skb->local_df = 1;
 612
 613         IP_VS_XMIT_NAT(NFPROTO_IPV4, skb, cp, local);
 614
 615         LeaveFunction(10);
 616         return NF_STOLEN;
 617
 618   tx_error_icmp:
 619         dst_link_failure(skb);
 620   tx_error:
 621         kfree_skb(skb);
 622         LeaveFunction(10);
 623         return NF_STOLEN;
 624   tx_error_put:
 625         ip_rt_put(rt);
 626         goto tx_error;
 627 }
 628
 629 #ifdef CONFIG_IP_VS_IPV6
 630 int
 631 ip_vs_nat_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp,
 632                   struct ip_vs_protocol *pp)
 633 {
 634         struct rt6_info *rt;            /* Route to the other host */
 635         int mtu;
 636         int local;
 637
 638         EnterFunction(10);
 639
 640         /* check if it is a connection of no-client-port */
 641         if (unlikely(cp->flags & IP_VS_CONN_F_NO_CPORT)) {
 642                 __be16 _pt, *p;
 643                 p = skb_header_pointer(skb, sizeof(struct ipv6hdr),
 644                                        sizeof(_pt), &_pt);
 645                 if (p == NULL)
 646                         goto tx_error;
 647                 ip_vs_conn_fill_cport(cp, *p);
 648                 IP_VS_DBG(10, "filled cport=%d\n", ntohs(*p));
 649         }
 650
 651         if (!(rt = __ip_vs_get_out_rt_v6(skb, cp->dest, &cp->daddr.in6, NULL,
 652                                          0, (IP_VS_RT_MODE_LOCAL |
 653                                              IP_VS_RT_MODE_NON_LOCAL |
 654                                              IP_VS_RT_MODE_RDR))))
 655                 goto tx_error_icmp;
 656         local = __ip_vs_is_local_route6(rt);
 657         /*
 658          * Avoid duplicate tuple in reply direction for NAT traffic
 659          * to local address when connection is sync-ed
 660          */
 661 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
 662         if (cp->flags & IP_VS_CONN_F_SYNC && local) {
 663                 enum ip_conntrack_info ctinfo;
 664                 struct nf_conn *ct = ct = nf_ct_get(skb, &ctinfo);
 665
 666                 if (ct && !nf_ct_is_untracked(ct)) {
 667                         IP_VS_DBG_RL_PKT(10, AF_INET6, pp, skb, 0,
 668                                          "ip_vs_nat_xmit_v6(): "
 669                                          "stopping DNAT to local address");
 670                         goto tx_error_put;
 671                 }
 672         }
 673 #endif
 674
 675         /* From world but DNAT to loopback address? */
 676         if (local && skb->dev && !(skb->dev->flags & IFF_LOOPBACK) &&
 677             ipv6_addr_type(&rt->rt6i_dst.addr) & IPV6_ADDR_LOOPBACK) {
 678                 IP_VS_DBG_RL_PKT(1, AF_INET6, pp, skb, 0,
 679                                  "ip_vs_nat_xmit_v6(): "
 680                                  "stopping DNAT to loopback address");
 681                 goto tx_error_put;
 682         }
 683
 684         /* MTU checking */
 685         mtu = dst_mtu(&rt->dst);
 686         if (skb->len > mtu && !skb_is_gso(skb)) {
 687                 if (!skb->dev) {
 688                         struct net *net = dev_net(skb_dst(skb)->dev);
 689
 690                         skb->dev = net->loopback_dev;
 691                 }
 692                 icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu);
 693                 IP_VS_DBG_RL_PKT(0, AF_INET6, pp, skb, 0,
 694                                  "ip_vs_nat_xmit_v6(): frag needed for");
 695                 goto tx_error_put;
 696         }
 697
 698         /* copy-on-write the packet before mangling it */
 699         if (!skb_make_writable(skb, sizeof(struct ipv6hdr)))
 700                 goto tx_error_put;
 701
 702         if (skb_cow(skb, rt->dst.dev->hard_header_len))
 703                 goto tx_error_put;
 704
 705         /* mangle the packet */
 706         if (pp->dnat_handler && !pp->dnat_handler(skb, pp, cp))
 707                 goto tx_error;
 708         ipv6_hdr(skb)->daddr = cp->daddr.in6;
 709
 710         if (!local || !skb->dev) {
 711                 /* drop the old route when skb is not shared */
 712                 skb_dst_drop(skb);
 713                 skb_dst_set(skb, &rt->dst);
 714         } else {
 715                 /* destined to loopback, do we need to change route? */
 716                 dst_release(&rt->dst);
 717         }
 718
 719         IP_VS_DBG_PKT(10, AF_INET6, pp, skb, 0, "After DNAT");
 720
 721         /* FIXME: when application helper enlarges the packet and the length
 722            is larger than the MTU of outgoing device, there will be still
 723            MTU problem. */
 724
 725         /* Another hack: avoid icmp_send in ip_fragment */
 726         skb->local_df = 1;
 727
 728         IP_VS_XMIT_NAT(NFPROTO_IPV6, skb, cp, local);
 729
 730         LeaveFunction(10);
 731         return NF_STOLEN;
 732
 733 tx_error_icmp:
 734         dst_link_failure(skb);
 735 tx_error:
 736         LeaveFunction(10);
 737         kfree_skb(skb);
 738         return NF_STOLEN;
 739 tx_error_put:
 740         dst_release(&rt->dst);
 741         goto tx_error;
 742 }
 743 #endif
 744
 745
 746 /*
 747  *   IP Tunneling transmitter
 748  *
 749  *   This function encapsulates the packet in a new IP packet, its
 750  *   destination will be set to cp->daddr. Most code of this function
 751  *   is taken from ipip.c.
 752  *
 753  *   It is used in VS/TUN cluster. The load balancer selects a real
 754  *   server from a cluster based on a scheduling algorithm,
 755  *   encapsulates the request packet and forwards it to the selected
 756  *   server. For example, all real servers are configured with
 757  *   "ifconfig tunl0 <Virtual IP Address> up". When the server receives
 758  *   the encapsulated packet, it will decapsulate the packet, processe
 759  *   the request and return the response packets directly to the client
 760  *   without passing the load balancer. This can greatly increase the
 761  *   scalability of virtual server.
 762  *
 763  *   Used for ANY protocol
 764  */
 765 int
 766 ip_vs_tunnel_xmit(struct sk_buff *skb, struct ip_vs_conn *cp,
 767                   struct ip_vs_protocol *pp)
 768 {
 769         struct rtable *rt;                      /* Route to the other host */
 770         __be32 saddr;                           /* Source for tunnel */
 771         struct net_device *tdev;                /* Device to other host */
 772         struct iphdr  *old_iph = ip_hdr(skb);
 773         u8     tos = old_iph->tos;
 774         __be16 df = old_iph->frag_off;
 775         struct iphdr  *iph;                     /* Our new IP header */
 776         unsigned int max_headroom;              /* The extra header space needed */
 777         int    mtu;
 778         int ret;
 779
 780         EnterFunction(10);
 781
 782         if (!(rt = __ip_vs_get_out_rt(skb, cp->dest, cp->daddr.ip,
 783                                       RT_TOS(tos), IP_VS_RT_MODE_LOCAL |
 784                                                    IP_VS_RT_MODE_NON_LOCAL,
 785                                                    &saddr)))
 786                 goto tx_error_icmp;
 787         if (rt->rt_flags & RTCF_LOCAL) {
 788                 ip_rt_put(rt);
 789                 IP_VS_XMIT(NFPROTO_IPV4, skb, cp, 1);
 790         }
 791
 792         tdev = rt->dst.dev;
 793
 794         mtu = dst_mtu(&rt->dst) - sizeof(struct iphdr);
 795         if (mtu < 68) {
 796                 IP_VS_DBG_RL("%s(): mtu less than 68\n", __func__);
 797                 goto tx_error_put;
 798         }
 799         if (skb_dst(skb))
 800                 skb_dst(skb)->ops->update_pmtu(skb_dst(skb), mtu);
 801
 802         df |= (old_iph->frag_off & htons(IP_DF));
 803
 804         if ((old_iph->frag_off & htons(IP_DF) &&
 805             mtu < ntohs(old_iph->tot_len) && !skb_is_gso(skb))) {
 806                 icmp_send(skb, ICMP_DEST_UNREACH,ICMP_FRAG_NEEDED, htonl(mtu));
 807                 IP_VS_DBG_RL("%s(): frag needed\n", __func__);
 808                 goto tx_error_put;
 809         }
 810
 811         /*
 812          * Okay, now see if we can stuff it in the buffer as-is.
 813          */
 814         max_headroom = LL_RESERVED_SPACE(tdev) + sizeof(struct iphdr);
 815
 816         if (skb_headroom(skb) < max_headroom
 817             || skb_cloned(skb) || skb_shared(skb)) {
 818                 struct sk_buff *new_skb =
 819                         skb_realloc_headroom(skb, max_headroom);
 820                 if (!new_skb) {
 821                         ip_rt_put(rt);
 822                         kfree_skb(skb);
 823                         IP_VS_ERR_RL("%s(): no memory\n", __func__);
 824                         return NF_STOLEN;
 825                 }
 826                 kfree_skb(skb);
 827                 skb = new_skb;
 828                 old_iph = ip_hdr(skb);
 829         }
 830
 831         skb->transport_header = skb->network_header;
 832
 833         /* fix old IP header checksum */
 834         ip_send_check(old_iph);
 835
 836         skb_push(skb, sizeof(struct iphdr));
 837         skb_reset_network_header(skb);
 838         memset(&(IPCB(skb)->opt), 0, sizeof(IPCB(skb)->opt));
 839
 840         /* drop old route */
 841         skb_dst_drop(skb);
 842         skb_dst_set(skb, &rt->dst);
 843
 844         /*
 845          *      Push down and install the IPIP header.
 846          */
 847         iph                     =       ip_hdr(skb);
 848         iph->version            =       4;
 849         iph->ihl                =       sizeof(struct iphdr)>>2;
 850         iph->frag_off           =       df;
 851         iph->protocol           =       IPPROTO_IPIP;
 852         iph->tos                =       tos;
 853         iph->daddr              =       cp->daddr.ip;
 854         iph->saddr              =       saddr;
 855         iph->ttl                =       old_iph->ttl;
 856         ip_select_ident(iph, &rt->dst, NULL);
 857
 858         /* Another hack: avoid icmp_send in ip_fragment */
 859         skb->local_df = 1;
 860
 861         ret = IP_VS_XMIT_TUNNEL(skb, cp);
 862         if (ret == NF_ACCEPT)
 863                 ip_local_out(skb);
 864         else if (ret == NF_DROP)
 865                 kfree_skb(skb);
 866
 867         LeaveFunction(10);
 868
 869         return NF_STOLEN;
 870
 871   tx_error_icmp:
 872         dst_link_failure(skb);
 873   tx_error:
 874         kfree_skb(skb);
 875         LeaveFunction(10);
 876         return NF_STOLEN;
 877 tx_error_put:
 878         ip_rt_put(rt);
 879         goto tx_error;
 880 }
 881
 882 #ifdef CONFIG_IP_VS_IPV6
 883 int
 884 ip_vs_tunnel_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp,
 885                      struct ip_vs_protocol *pp)
 886 {
 887         struct rt6_info *rt;            /* Route to the other host */
 888         struct in6_addr saddr;          /* Source for tunnel */
 889         struct net_device *tdev;        /* Device to other host */
 890         struct ipv6hdr  *old_iph = ipv6_hdr(skb);
 891         struct ipv6hdr  *iph;           /* Our new IP header */
 892         unsigned int max_headroom;      /* The extra header space needed */
 893         int    mtu;
 894         int ret;
 895
 896         EnterFunction(10);
 897
 898         if (!(rt = __ip_vs_get_out_rt_v6(skb, cp->dest, &cp->daddr.in6,
 899                                          &saddr, 1, (IP_VS_RT_MODE_LOCAL |
 900                                                      IP_VS_RT_MODE_NON_LOCAL))))
 901                 goto tx_error_icmp;
 902         if (__ip_vs_is_local_route6(rt)) {
 903                 dst_release(&rt->dst);
 904                 IP_VS_XMIT(NFPROTO_IPV6, skb, cp, 1);
 905         }
 906
 907         tdev = rt->dst.dev;
 908
 909         mtu = dst_mtu(&rt->dst) - sizeof(struct ipv6hdr);
 910         if (mtu < IPV6_MIN_MTU) {
 911                 IP_VS_DBG_RL("%s(): mtu less than %d\n", __func__,
 912                              IPV6_MIN_MTU);
 913                 goto tx_error_put;
 914         }
 915         if (skb_dst(skb))
 916                 skb_dst(skb)->ops->update_pmtu(skb_dst(skb), mtu);
 917
 918         if (mtu < ntohs(old_iph->payload_len) + sizeof(struct ipv6hdr) &&
 919             !skb_is_gso(skb)) {
 920                 if (!skb->dev) {
 921                         struct net *net = dev_net(skb_dst(skb)->dev);
 922
 923                         skb->dev = net->loopback_dev;
 924                 }
 925                 icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu);
 926                 IP_VS_DBG_RL("%s(): frag needed\n", __func__);
 927                 goto tx_error_put;
 928         }
 929
 930         /*
 931          * Okay, now see if we can stuff it in the buffer as-is.
 932          */
 933         max_headroom = LL_RESERVED_SPACE(tdev) + sizeof(struct ipv6hdr);
 934
 935         if (skb_headroom(skb) < max_headroom
 936             || skb_cloned(skb) || skb_shared(skb)) {
 937                 struct sk_buff *new_skb =
 938                         skb_realloc_headroom(skb, max_headroom);
 939                 if (!new_skb) {
 940                         dst_release(&rt->dst);
 941                         kfree_skb(skb);
 942                         IP_VS_ERR_RL("%s(): no memory\n", __func__);
 943                         return NF_STOLEN;
 944                 }
 945                 kfree_skb(skb);
 946                 skb = new_skb;
 947                 old_iph = ipv6_hdr(skb);
 948         }
 949
 950         skb->transport_header = skb->network_header;
 951
 952         skb_push(skb, sizeof(struct ipv6hdr));
 953         skb_reset_network_header(skb);
 954         memset(&(IPCB(skb)->opt), 0, sizeof(IPCB(skb)->opt));
 955
 956         /* drop old route */
 957         skb_dst_drop(skb);
 958         skb_dst_set(skb, &rt->dst);
 959
 960         /*
 961          *      Push down and install the IPIP header.
 962          */
 963         iph                     =       ipv6_hdr(skb);
 964         iph->version            =       6;
 965         iph->nexthdr            =       IPPROTO_IPV6;
 966         iph->payload_len        =       old_iph->payload_len;
 967         be16_add_cpu(&iph->payload_len, sizeof(*old_iph));
 968         iph->priority           =       old_iph->priority;
 969         memset(&iph->flow_lbl, 0, sizeof(iph->flow_lbl));
 970         iph->daddr = cp->daddr.in6;
 971         iph->saddr = saddr;
 972         iph->hop_limit          =       old_iph->hop_limit;
 973
 974         /* Another hack: avoid icmp_send in ip_fragment */
 975         skb->local_df = 1;
 976
 977         ret = IP_VS_XMIT_TUNNEL(skb, cp);
 978         if (ret == NF_ACCEPT)
 979                 ip6_local_out(skb);
 980         else if (ret == NF_DROP)
 981                 kfree_skb(skb);
 982
 983         LeaveFunction(10);
 984
 985         return NF_STOLEN;
 986
 987 tx_error_icmp:
 988         dst_link_failure(skb);
 989 tx_error:
 990         kfree_skb(skb);
 991         LeaveFunction(10);
 992         return NF_STOLEN;
 993 tx_error_put:
 994         dst_release(&rt->dst);
 995         goto tx_error;
 996 }
 997 #endif
 998
 999
1000 /*
1001  *      Direct Routing transmitter
1002  *      Used for ANY protocol
1003  */
1004 int
1005 ip_vs_dr_xmit(struct sk_buff *skb, struct ip_vs_conn *cp,
1006               struct ip_vs_protocol *pp)
1007 {
1008         struct rtable *rt;                      /* Route to the other host */
1009         struct iphdr  *iph = ip_hdr(skb);
1010         int    mtu;
1011
1012         EnterFunction(10);
1013
1014         if (!(rt = __ip_vs_get_out_rt(skb, cp->dest, cp->daddr.ip,
1015                                       RT_TOS(iph->tos),
1016                                       IP_VS_RT_MODE_LOCAL |
1017                                         IP_VS_RT_MODE_NON_LOCAL, NULL)))
1018                 goto tx_error_icmp;
1019         if (rt->rt_flags & RTCF_LOCAL) {
1020                 ip_rt_put(rt);
1021                 IP_VS_XMIT(NFPROTO_IPV4, skb, cp, 1);
1022         }
1023
1024         /* MTU checking */
1025         mtu = dst_mtu(&rt->dst);
1026         if ((iph->frag_off & htons(IP_DF)) && skb->len > mtu &&
1027             !skb_is_gso(skb)) {
1028                 icmp_send(skb, ICMP_DEST_UNREACH,ICMP_FRAG_NEEDED, htonl(mtu));
1029                 ip_rt_put(rt);
1030                 IP_VS_DBG_RL("%s(): frag needed\n", __func__);
1031                 goto tx_error;
1032         }
1033
1034         /*
1035          * Call ip_send_check because we are not sure it is called
1036          * after ip_defrag. Is copy-on-write needed?
1037          */
1038         if (unlikely((skb = skb_share_check(skb, GFP_ATOMIC)) == NULL)) {
1039                 ip_rt_put(rt);
1040                 return NF_STOLEN;
1041         }
1042         ip_send_check(ip_hdr(skb));
1043
1044         /* drop old route */
1045         skb_dst_drop(skb);
1046         skb_dst_set(skb, &rt->dst);
1047
1048         /* Another hack: avoid icmp_send in ip_fragment */
1049         skb->local_df = 1;
1050
1051         IP_VS_XMIT(NFPROTO_IPV4, skb, cp, 0);
1052
1053         LeaveFunction(10);
1054         return NF_STOLEN;
1055
1056   tx_error_icmp:
1057         dst_link_failure(skb);
1058   tx_error:
1059         kfree_skb(skb);
1060         LeaveFunction(10);
1061         return NF_STOLEN;
1062 }
1063
1064 #ifdef CONFIG_IP_VS_IPV6
1065 int
1066 ip_vs_dr_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp,
1067                  struct ip_vs_protocol *pp)
1068 {
1069         struct rt6_info *rt;                    /* Route to the other host */
1070         int    mtu;
1071
1072         EnterFunction(10);
1073
1074         if (!(rt = __ip_vs_get_out_rt_v6(skb, cp->dest, &cp->daddr.in6, NULL,
1075                                          0, (IP_VS_RT_MODE_LOCAL |
1076                                              IP_VS_RT_MODE_NON_LOCAL))))
1077                 goto tx_error_icmp;
1078         if (__ip_vs_is_local_route6(rt)) {
1079                 dst_release(&rt->dst);
1080                 IP_VS_XMIT(NFPROTO_IPV6, skb, cp, 1);
1081         }
1082
1083         /* MTU checking */
1084         mtu = dst_mtu(&rt->dst);
1085         if (skb->len > mtu) {
1086                 if (!skb->dev) {
1087                         struct net *net = dev_net(skb_dst(skb)->dev);
1088
1089                         skb->dev = net->loopback_dev;
1090                 }
1091                 icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu);
1092                 dst_release(&rt->dst);
1093                 IP_VS_DBG_RL("%s(): frag needed\n", __func__);
1094                 goto tx_error;
1095         }
1096
1097         /*
1098          * Call ip_send_check because we are not sure it is called
1099          * after ip_defrag. Is copy-on-write needed?
1100          */
1101         skb = skb_share_check(skb, GFP_ATOMIC);
1102         if (unlikely(skb == NULL)) {
1103                 dst_release(&rt->dst);
1104                 return NF_STOLEN;
1105         }
1106
1107         /* drop old route */
1108         skb_dst_drop(skb);
1109         skb_dst_set(skb, &rt->dst);
1110
1111         /* Another hack: avoid icmp_send in ip_fragment */
1112         skb->local_df = 1;
1113
1114         IP_VS_XMIT(NFPROTO_IPV6, skb, cp, 0);
1115
1116         LeaveFunction(10);
1117         return NF_STOLEN;
1118
1119 tx_error_icmp:
1120         dst_link_failure(skb);
1121 tx_error:
1122         kfree_skb(skb);
1123         LeaveFunction(10);
1124         return NF_STOLEN;
1125 }
1126 #endif
1127
1128
1129 /*
1130  *      ICMP packet transmitter
1131  *      called by the ip_vs_in_icmp
1132  */
1133 int
1134 ip_vs_icmp_xmit(struct sk_buff *skb, struct ip_vs_conn *cp,
1135                 struct ip_vs_protocol *pp, int offset, unsigned int hooknum)
1136 {
1137         struct rtable   *rt;    /* Route to the other host */
1138         int mtu;
1139         int rc;
1140         int local;
1141         int rt_mode;
1142
1143         EnterFunction(10);
1144
1145         /* The ICMP packet for VS/TUN, VS/DR and LOCALNODE will be
1146            forwarded directly here, because there is no need to
1147            translate address/port back */
1148         if (IP_VS_FWD_METHOD(cp) != IP_VS_CONN_F_MASQ) {
1149                 if (cp->packet_xmit)
1150                         rc = cp->packet_xmit(skb, cp, pp);
1151                 else
1152                         rc = NF_ACCEPT;
1153                 /* do not touch skb anymore */
1154                 atomic_inc(&cp->in_pkts);
1155                 goto out;
1156         }
1157
1158         /*
1159          * mangle and send the packet here (only for VS/NAT)
1160          */
1161
1162         /* LOCALNODE from FORWARD hook is not supported */
1163         rt_mode = (hooknum != NF_INET_FORWARD) ?
1164                   IP_VS_RT_MODE_LOCAL | IP_VS_RT_MODE_NON_LOCAL |
1165                   IP_VS_RT_MODE_RDR : IP_VS_RT_MODE_NON_LOCAL;
1166         if (!(rt = __ip_vs_get_out_rt(skb, cp->dest, cp->daddr.ip,
1167                                       RT_TOS(ip_hdr(skb)->tos),
1168                                       rt_mode, NULL)))
1169                 goto tx_error_icmp;
1170         local = rt->rt_flags & RTCF_LOCAL;
1171
1172         /*
1173          * Avoid duplicate tuple in reply direction for NAT traffic
1174          * to local address when connection is sync-ed
1175          */
1176 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
1177         if (cp->flags & IP_VS_CONN_F_SYNC && local) {
1178                 enum ip_conntrack_info ctinfo;
1179                 struct nf_conn *ct = ct = nf_ct_get(skb, &ctinfo);
1180
1181                 if (ct && !nf_ct_is_untracked(ct)) {
1182                         IP_VS_DBG(10, "%s(): "
1183                                   "stopping DNAT to local address %pI4\n",
1184                                   __func__, &cp->daddr.ip);
1185                         goto tx_error_put;
1186                 }
1187         }
1188 #endif
1189
1190         /* From world but DNAT to loopback address? */
1191         if (local && ipv4_is_loopback(cp->daddr.ip) &&
1192             rt_is_input_route(skb_rtable(skb))) {
1193                 IP_VS_DBG(1, "%s(): "
1194                           "stopping DNAT to loopback %pI4\n",
1195                           __func__, &cp->daddr.ip);
1196                 goto tx_error_put;
1197         }
1198
1199         /* MTU checking */
1200         mtu = dst_mtu(&rt->dst);
1201         if ((skb->len > mtu) && (ip_hdr(skb)->frag_off & htons(IP_DF)) &&
1202             !skb_is_gso(skb)) {
1203                 icmp_send(skb, ICMP_DEST_UNREACH, ICMP_FRAG_NEEDED, htonl(mtu));
1204                 IP_VS_DBG_RL("%s(): frag needed\n", __func__);
1205                 goto tx_error_put;
1206         }
1207
1208         /* copy-on-write the packet before mangling it */
1209         if (!skb_make_writable(skb, offset))
1210                 goto tx_error_put;
1211
1212         if (skb_cow(skb, rt->dst.dev->hard_header_len))
1213                 goto tx_error_put;
1214
1215         ip_vs_nat_icmp(skb, pp, cp, 0);
1216
1217         if (!local) {
1218                 /* drop the old route when skb is not shared */
1219                 skb_dst_drop(skb);
1220                 skb_dst_set(skb, &rt->dst);
1221         } else {
1222                 ip_rt_put(rt);
1223                 /*
1224                  * Some IPv4 replies get local address from routes,
1225                  * not from iph, so while we DNAT after routing
1226                  * we need this second input/output route.
1227                  */
1228                 if (!__ip_vs_reroute_locally(skb))
1229                         goto tx_error;
1230         }
1231
1232         /* Another hack: avoid icmp_send in ip_fragment */
1233         skb->local_df = 1;
1234
1235         IP_VS_XMIT_NAT(NFPROTO_IPV4, skb, cp, local);
1236
1237         rc = NF_STOLEN;
1238         goto out;
1239
1240   tx_error_icmp:
1241         dst_link_failure(skb);
1242   tx_error:
1243         dev_kfree_skb(skb);
1244         rc = NF_STOLEN;
1245   out:
1246         LeaveFunction(10);
1247         return rc;
1248   tx_error_put:
1249         ip_rt_put(rt);
1250         goto tx_error;
1251 }
1252
1253 #ifdef CONFIG_IP_VS_IPV6
1254 int
1255 ip_vs_icmp_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp,
1256                 struct ip_vs_protocol *pp, int offset, unsigned int hooknum)
1257 {
1258         struct rt6_info *rt;    /* Route to the other host */
1259         int mtu;
1260         int rc;
1261         int local;
1262         int rt_mode;
1263
1264         EnterFunction(10);
1265
1266         /* The ICMP packet for VS/TUN, VS/DR and LOCALNODE will be
1267            forwarded directly here, because there is no need to
1268            translate address/port back */
1269         if (IP_VS_FWD_METHOD(cp) != IP_VS_CONN_F_MASQ) {
1270                 if (cp->packet_xmit)
1271                         rc = cp->packet_xmit(skb, cp, pp);
1272                 else
1273                         rc = NF_ACCEPT;
1274                 /* do not touch skb anymore */
1275                 atomic_inc(&cp->in_pkts);
1276                 goto out;
1277         }
1278
1279         /*
1280          * mangle and send the packet here (only for VS/NAT)
1281          */
1282
1283         /* LOCALNODE from FORWARD hook is not supported */
1284         rt_mode = (hooknum != NF_INET_FORWARD) ?
1285                   IP_VS_RT_MODE_LOCAL | IP_VS_RT_MODE_NON_LOCAL |
1286                   IP_VS_RT_MODE_RDR : IP_VS_RT_MODE_NON_LOCAL;
1287         if (!(rt = __ip_vs_get_out_rt_v6(skb, cp->dest, &cp->daddr.in6, NULL,
1288                                          0, rt_mode)))
1289                 goto tx_error_icmp;
1290
1291         local = __ip_vs_is_local_route6(rt);
1292         /*
1293          * Avoid duplicate tuple in reply direction for NAT traffic
1294          * to local address when connection is sync-ed
1295          */
1296 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
1297         if (cp->flags & IP_VS_CONN_F_SYNC && local) {
1298                 enum ip_conntrack_info ctinfo;
1299                 struct nf_conn *ct = ct = nf_ct_get(skb, &ctinfo);
1300
1301                 if (ct && !nf_ct_is_untracked(ct)) {
1302                         IP_VS_DBG(10, "%s(): "
1303                                   "stopping DNAT to local address %pI6\n",
1304                                   __func__, &cp->daddr.in6);
1305                         goto tx_error_put;
1306                 }
1307         }
1308 #endif
1309
1310         /* From world but DNAT to loopback address? */
1311         if (local && skb->dev && !(skb->dev->flags & IFF_LOOPBACK) &&
1312             ipv6_addr_type(&rt->rt6i_dst.addr) & IPV6_ADDR_LOOPBACK) {
1313                 IP_VS_DBG(1, "%s(): "
1314                           "stopping DNAT to loopback %pI6\n",
1315                           __func__, &cp->daddr.in6);
1316                 goto tx_error_put;
1317         }
1318
1319         /* MTU checking */
1320         mtu = dst_mtu(&rt->dst);
1321         if (skb->len > mtu && !skb_is_gso(skb)) {
1322                 if (!skb->dev) {
1323                         struct net *net = dev_net(skb_dst(skb)->dev);
1324
1325                         skb->dev = net->loopback_dev;
1326                 }
1327                 icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu);
1328                 IP_VS_DBG_RL("%s(): frag needed\n", __func__);
1329                 goto tx_error_put;
1330         }
1331
1332         /* copy-on-write the packet before mangling it */
1333         if (!skb_make_writable(skb, offset))
1334                 goto tx_error_put;
1335
1336         if (skb_cow(skb, rt->dst.dev->hard_header_len))
1337                 goto tx_error_put;
1338
1339         ip_vs_nat_icmp_v6(skb, pp, cp, 0);
1340
1341         if (!local || !skb->dev) {
1342                 /* drop the old route when skb is not shared */
1343                 skb_dst_drop(skb);
1344                 skb_dst_set(skb, &rt->dst);
1345         } else {
1346                 /* destined to loopback, do we need to change route? */
1347                 dst_release(&rt->dst);
1348         }
1349
1350         /* Another hack: avoid icmp_send in ip_fragment */
1351         skb->local_df = 1;
1352
1353         IP_VS_XMIT_NAT(NFPROTO_IPV6, skb, cp, local);
1354
1355         rc = NF_STOLEN;
1356         goto out;
1357
1358 tx_error_icmp:
1359         dst_link_failure(skb);
1360 tx_error:
1361         dev_kfree_skb(skb);
1362         rc = NF_STOLEN;
1363 out:
1364         LeaveFunction(10);
1365         return rc;
1366 tx_error_put:
1367         dst_release(&rt->dst);
1368         goto tx_error;
1369 }
1370 #endif