2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
6 * PACKET - implements raw packet sockets.
9 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10 * Alan Cox, <gw4pts@gw4pts.ampr.org>
13 * Alan Cox : verify_area() now used correctly
14 * Alan Cox : new skbuff lists, look ma no backlogs!
15 * Alan Cox : tidied skbuff lists.
16 * Alan Cox : Now uses generic datagram routines I
17 * added. Also fixed the peek/read crash
18 * from all old Linux datagram code.
19 * Alan Cox : Uses the improved datagram code.
20 * Alan Cox : Added NULL's for socket options.
21 * Alan Cox : Re-commented the code.
22 * Alan Cox : Use new kernel side addressing
23 * Rob Janssen : Correct MTU usage.
24 * Dave Platt : Counter leaks caused by incorrect
25 * interrupt locking and some slightly
26 * dubious gcc output. Can you read
27 * compiler: it said _VOLATILE_
28 * Richard Kooijman : Timestamp fixes.
29 * Alan Cox : New buffers. Use sk->mac.raw.
30 * Alan Cox : sendmsg/recvmsg support.
31 * Alan Cox : Protocol setting support
32 * Alexey Kuznetsov : Untied from IPv4 stack.
33 * Cyrus Durgin : Fixed kerneld for kmod.
34 * Michal Ostrowski : Module initialization cleanup.
35 * Ulises Alonso : Frame number limit removal and
36 * packet_set_ring memory leak.
37 * Eric Biederman : Allow for > 8 byte hardware addresses.
38 * The convention is that longer addresses
39 * will simply extend the hardware address
40 * byte arrays at the end of sockaddr_ll
42 * Johann Baudy : Added TX RING.
43 * Chetan Loke : Implemented TPACKET_V3 block abstraction
45 * Copyright (C) 2011, <lokec@ccs.neu.edu>
48 * This program is free software; you can redistribute it and/or
49 * modify it under the terms of the GNU General Public License
50 * as published by the Free Software Foundation; either version
51 * 2 of the License, or (at your option) any later version.
55 #include <linux/types.h>
57 #include <linux/capability.h>
58 #include <linux/fcntl.h>
59 #include <linux/socket.h>
61 #include <linux/inet.h>
62 #include <linux/netdevice.h>
63 #include <linux/if_packet.h>
64 #include <linux/wireless.h>
65 #include <linux/kernel.h>
66 #include <linux/kmod.h>
67 #include <linux/slab.h>
68 #include <linux/vmalloc.h>
69 #include <net/net_namespace.h>
71 #include <net/protocol.h>
72 #include <linux/skbuff.h>
74 #include <linux/errno.h>
75 #include <linux/timer.h>
76 #include <asm/uaccess.h>
77 #include <asm/ioctls.h>
79 #include <asm/cacheflush.h>
81 #include <linux/proc_fs.h>
82 #include <linux/seq_file.h>
83 #include <linux/poll.h>
84 #include <linux/module.h>
85 #include <linux/init.h>
86 #include <linux/mutex.h>
87 #include <linux/if_vlan.h>
88 #include <linux/virtio_net.h>
89 #include <linux/errqueue.h>
90 #include <linux/net_tstamp.h>
91 #include <linux/reciprocal_div.h>
93 #include <net/inet_common.h>
100 - if device has no dev->hard_header routine, it adds and removes ll header
101 inside itself. In this case ll header is invisible outside of device,
102 but higher levels still should reserve dev->hard_header_len.
103 Some devices are enough clever to reallocate skb, when header
104 will not fit to reserved space (tunnel), another ones are silly
106 - packet socket receives packets with pulled ll header,
107 so that SOCK_RAW should push it back.
112 Incoming, dev->hard_header!=NULL
113 mac_header -> ll header
116 Outgoing, dev->hard_header!=NULL
117 mac_header -> ll header
120 Incoming, dev->hard_header==NULL
121 mac_header -> UNKNOWN position. It is very likely, that it points to ll
122 header. PPP makes it, that is wrong, because introduce
123 assymetry between rx and tx paths.
126 Outgoing, dev->hard_header==NULL
127 mac_header -> data. ll header is still not built!
131 If dev->hard_header==NULL we are unlikely to restore sensible ll header.
137 dev->hard_header != NULL
138 mac_header -> ll header
141 dev->hard_header == NULL (ll header is added by device, we cannot control it)
145 We should set nh.raw on output to correct posistion,
146 packet classifier depends on it.
149 /* Private packet socket structures. */
151 /* identical to struct packet_mreq except it has
152 * a longer address field.
154 struct packet_mreq_max {
156 unsigned short mr_type;
157 unsigned short mr_alen;
158 unsigned char mr_address[MAX_ADDR_LEN];
162 struct tpacket_hdr *h1;
163 struct tpacket2_hdr *h2;
164 struct tpacket3_hdr *h3;
168 static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
169 int closing, int tx_ring);
171 #define V3_ALIGNMENT (8)
173 #define BLK_HDR_LEN (ALIGN(sizeof(struct tpacket_block_desc), V3_ALIGNMENT))
175 #define BLK_PLUS_PRIV(sz_of_priv) \
176 (BLK_HDR_LEN + ALIGN((sz_of_priv), V3_ALIGNMENT))
178 #define PGV_FROM_VMALLOC 1
180 #define BLOCK_STATUS(x) ((x)->hdr.bh1.block_status)
181 #define BLOCK_NUM_PKTS(x) ((x)->hdr.bh1.num_pkts)
182 #define BLOCK_O2FP(x) ((x)->hdr.bh1.offset_to_first_pkt)
183 #define BLOCK_LEN(x) ((x)->hdr.bh1.blk_len)
184 #define BLOCK_SNUM(x) ((x)->hdr.bh1.seq_num)
185 #define BLOCK_O2PRIV(x) ((x)->offset_to_priv)
186 #define BLOCK_PRIV(x) ((void *)((char *)(x) + BLOCK_O2PRIV(x)))
189 static int tpacket_snd(struct packet_sock *po, struct msghdr *msg);
190 static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
191 struct packet_type *pt, struct net_device *orig_dev);
193 static void *packet_previous_frame(struct packet_sock *po,
194 struct packet_ring_buffer *rb,
196 static void packet_increment_head(struct packet_ring_buffer *buff);
197 static int prb_curr_blk_in_use(struct tpacket_kbdq_core *,
198 struct tpacket_block_desc *);
199 static void *prb_dispatch_next_block(struct tpacket_kbdq_core *,
200 struct packet_sock *);
201 static void prb_retire_current_block(struct tpacket_kbdq_core *,
202 struct packet_sock *, unsigned int status);
203 static int prb_queue_frozen(struct tpacket_kbdq_core *);
204 static void prb_open_block(struct tpacket_kbdq_core *,
205 struct tpacket_block_desc *);
206 static void prb_retire_rx_blk_timer_expired(unsigned long);
207 static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *);
208 static void prb_init_blk_timer(struct packet_sock *,
209 struct tpacket_kbdq_core *,
210 void (*func) (unsigned long));
211 static void prb_fill_rxhash(struct tpacket_kbdq_core *, struct tpacket3_hdr *);
212 static void prb_clear_rxhash(struct tpacket_kbdq_core *,
213 struct tpacket3_hdr *);
214 static void prb_fill_vlan_info(struct tpacket_kbdq_core *,
215 struct tpacket3_hdr *);
216 static void packet_flush_mclist(struct sock *sk);
218 struct packet_skb_cb {
219 unsigned int origlen;
221 struct sockaddr_pkt pkt;
222 struct sockaddr_ll ll;
226 #define PACKET_SKB_CB(__skb) ((struct packet_skb_cb *)((__skb)->cb))
228 #define GET_PBDQC_FROM_RB(x) ((struct tpacket_kbdq_core *)(&(x)->prb_bdqc))
229 #define GET_PBLOCK_DESC(x, bid) \
230 ((struct tpacket_block_desc *)((x)->pkbdq[(bid)].buffer))
231 #define GET_CURR_PBLOCK_DESC_FROM_CORE(x) \
232 ((struct tpacket_block_desc *)((x)->pkbdq[(x)->kactive_blk_num].buffer))
233 #define GET_NEXT_PRB_BLK_NUM(x) \
234 (((x)->kactive_blk_num < ((x)->knum_blocks-1)) ? \
235 ((x)->kactive_blk_num+1) : 0)
237 static void __fanout_unlink(struct sock *sk, struct packet_sock *po);
238 static void __fanout_link(struct sock *sk, struct packet_sock *po);
240 static int packet_direct_xmit(struct sk_buff *skb)
242 struct net_device *dev = skb->dev;
243 const struct net_device_ops *ops = dev->netdev_ops;
244 netdev_features_t features;
245 struct netdev_queue *txq;
249 if (unlikely(!netif_running(dev) ||
250 !netif_carrier_ok(dev))) {
252 return NET_XMIT_DROP;
255 features = netif_skb_features(skb);
256 if (skb_needs_linearize(skb, features) &&
257 __skb_linearize(skb)) {
259 return NET_XMIT_DROP;
262 queue_map = skb_get_queue_mapping(skb);
263 txq = netdev_get_tx_queue(dev, queue_map);
265 __netif_tx_lock_bh(txq);
266 if (unlikely(netif_xmit_frozen_or_stopped(txq))) {
267 ret = NETDEV_TX_BUSY;
272 ret = ops->ndo_start_xmit(skb, dev);
273 if (likely(dev_xmit_complete(ret)))
274 txq_trans_update(txq);
278 __netif_tx_unlock_bh(txq);
282 static struct net_device *packet_cached_dev_get(struct packet_sock *po)
284 struct net_device *dev;
287 dev = rcu_dereference(po->cached_dev);
295 static void packet_cached_dev_assign(struct packet_sock *po,
296 struct net_device *dev)
298 rcu_assign_pointer(po->cached_dev, dev);
301 static void packet_cached_dev_reset(struct packet_sock *po)
303 RCU_INIT_POINTER(po->cached_dev, NULL);
306 static bool packet_use_direct_xmit(const struct packet_sock *po)
308 return po->xmit == packet_direct_xmit;
311 static u16 packet_pick_tx_queue(struct net_device *dev)
313 return (u16) raw_smp_processor_id() % dev->real_num_tx_queues;
316 /* register_prot_hook must be invoked with the po->bind_lock held,
317 * or from a context in which asynchronous accesses to the packet
318 * socket is not possible (packet_create()).
320 static void register_prot_hook(struct sock *sk)
322 struct packet_sock *po = pkt_sk(sk);
326 __fanout_link(sk, po);
328 dev_add_pack(&po->prot_hook);
335 /* {,__}unregister_prot_hook() must be invoked with the po->bind_lock
336 * held. If the sync parameter is true, we will temporarily drop
337 * the po->bind_lock and do a synchronize_net to make sure no
338 * asynchronous packet processing paths still refer to the elements
339 * of po->prot_hook. If the sync parameter is false, it is the
340 * callers responsibility to take care of this.
342 static void __unregister_prot_hook(struct sock *sk, bool sync)
344 struct packet_sock *po = pkt_sk(sk);
349 __fanout_unlink(sk, po);
351 __dev_remove_pack(&po->prot_hook);
356 spin_unlock(&po->bind_lock);
358 spin_lock(&po->bind_lock);
362 static void unregister_prot_hook(struct sock *sk, bool sync)
364 struct packet_sock *po = pkt_sk(sk);
367 __unregister_prot_hook(sk, sync);
370 static inline __pure struct page *pgv_to_page(void *addr)
372 if (is_vmalloc_addr(addr))
373 return vmalloc_to_page(addr);
374 return virt_to_page(addr);
377 static void __packet_set_status(struct packet_sock *po, void *frame, int status)
379 union tpacket_uhdr h;
382 switch (po->tp_version) {
384 h.h1->tp_status = status;
385 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
388 h.h2->tp_status = status;
389 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
393 WARN(1, "TPACKET version not supported.\n");
400 static int __packet_get_status(struct packet_sock *po, void *frame)
402 union tpacket_uhdr h;
407 switch (po->tp_version) {
409 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
410 return h.h1->tp_status;
412 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
413 return h.h2->tp_status;
416 WARN(1, "TPACKET version not supported.\n");
422 static __u32 tpacket_get_timestamp(struct sk_buff *skb, struct timespec *ts,
425 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
428 if ((flags & SOF_TIMESTAMPING_SYS_HARDWARE) &&
429 ktime_to_timespec_cond(shhwtstamps->syststamp, ts))
430 return TP_STATUS_TS_SYS_HARDWARE;
431 if ((flags & SOF_TIMESTAMPING_RAW_HARDWARE) &&
432 ktime_to_timespec_cond(shhwtstamps->hwtstamp, ts))
433 return TP_STATUS_TS_RAW_HARDWARE;
436 if (ktime_to_timespec_cond(skb->tstamp, ts))
437 return TP_STATUS_TS_SOFTWARE;
442 static __u32 __packet_set_timestamp(struct packet_sock *po, void *frame,
445 union tpacket_uhdr h;
449 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
453 switch (po->tp_version) {
455 h.h1->tp_sec = ts.tv_sec;
456 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
459 h.h2->tp_sec = ts.tv_sec;
460 h.h2->tp_nsec = ts.tv_nsec;
464 WARN(1, "TPACKET version not supported.\n");
468 /* one flush is safe, as both fields always lie on the same cacheline */
469 flush_dcache_page(pgv_to_page(&h.h1->tp_sec));
475 static void *packet_lookup_frame(struct packet_sock *po,
476 struct packet_ring_buffer *rb,
477 unsigned int position,
480 unsigned int pg_vec_pos, frame_offset;
481 union tpacket_uhdr h;
483 pg_vec_pos = position / rb->frames_per_block;
484 frame_offset = position % rb->frames_per_block;
486 h.raw = rb->pg_vec[pg_vec_pos].buffer +
487 (frame_offset * rb->frame_size);
489 if (status != __packet_get_status(po, h.raw))
495 static void *packet_current_frame(struct packet_sock *po,
496 struct packet_ring_buffer *rb,
499 return packet_lookup_frame(po, rb, rb->head, status);
502 static void prb_del_retire_blk_timer(struct tpacket_kbdq_core *pkc)
504 del_timer_sync(&pkc->retire_blk_timer);
507 static void prb_shutdown_retire_blk_timer(struct packet_sock *po,
509 struct sk_buff_head *rb_queue)
511 struct tpacket_kbdq_core *pkc;
513 pkc = tx_ring ? GET_PBDQC_FROM_RB(&po->tx_ring) :
514 GET_PBDQC_FROM_RB(&po->rx_ring);
516 spin_lock_bh(&rb_queue->lock);
517 pkc->delete_blk_timer = 1;
518 spin_unlock_bh(&rb_queue->lock);
520 prb_del_retire_blk_timer(pkc);
523 static void prb_init_blk_timer(struct packet_sock *po,
524 struct tpacket_kbdq_core *pkc,
525 void (*func) (unsigned long))
527 init_timer(&pkc->retire_blk_timer);
528 pkc->retire_blk_timer.data = (long)po;
529 pkc->retire_blk_timer.function = func;
530 pkc->retire_blk_timer.expires = jiffies;
533 static void prb_setup_retire_blk_timer(struct packet_sock *po, int tx_ring)
535 struct tpacket_kbdq_core *pkc;
540 pkc = tx_ring ? GET_PBDQC_FROM_RB(&po->tx_ring) :
541 GET_PBDQC_FROM_RB(&po->rx_ring);
542 prb_init_blk_timer(po, pkc, prb_retire_rx_blk_timer_expired);
545 static int prb_calc_retire_blk_tmo(struct packet_sock *po,
546 int blk_size_in_bytes)
548 struct net_device *dev;
549 unsigned int mbits = 0, msec = 0, div = 0, tmo = 0;
550 struct ethtool_cmd ecmd;
555 dev = __dev_get_by_index(sock_net(&po->sk), po->ifindex);
556 if (unlikely(!dev)) {
558 return DEFAULT_PRB_RETIRE_TOV;
560 err = __ethtool_get_settings(dev, &ecmd);
561 speed = ethtool_cmd_speed(&ecmd);
565 * If the link speed is so slow you don't really
566 * need to worry about perf anyways
568 if (speed < SPEED_1000 || speed == SPEED_UNKNOWN) {
569 return DEFAULT_PRB_RETIRE_TOV;
576 mbits = (blk_size_in_bytes * 8) / (1024 * 1024);
588 static void prb_init_ft_ops(struct tpacket_kbdq_core *p1,
589 union tpacket_req_u *req_u)
591 p1->feature_req_word = req_u->req3.tp_feature_req_word;
594 static void init_prb_bdqc(struct packet_sock *po,
595 struct packet_ring_buffer *rb,
597 union tpacket_req_u *req_u, int tx_ring)
599 struct tpacket_kbdq_core *p1 = GET_PBDQC_FROM_RB(rb);
600 struct tpacket_block_desc *pbd;
602 memset(p1, 0x0, sizeof(*p1));
604 p1->knxt_seq_num = 1;
606 pbd = (struct tpacket_block_desc *)pg_vec[0].buffer;
607 p1->pkblk_start = pg_vec[0].buffer;
608 p1->kblk_size = req_u->req3.tp_block_size;
609 p1->knum_blocks = req_u->req3.tp_block_nr;
610 p1->hdrlen = po->tp_hdrlen;
611 p1->version = po->tp_version;
612 p1->last_kactive_blk_num = 0;
613 po->stats.stats3.tp_freeze_q_cnt = 0;
614 if (req_u->req3.tp_retire_blk_tov)
615 p1->retire_blk_tov = req_u->req3.tp_retire_blk_tov;
617 p1->retire_blk_tov = prb_calc_retire_blk_tmo(po,
618 req_u->req3.tp_block_size);
619 p1->tov_in_jiffies = msecs_to_jiffies(p1->retire_blk_tov);
620 p1->blk_sizeof_priv = req_u->req3.tp_sizeof_priv;
622 prb_init_ft_ops(p1, req_u);
623 prb_setup_retire_blk_timer(po, tx_ring);
624 prb_open_block(p1, pbd);
627 /* Do NOT update the last_blk_num first.
628 * Assumes sk_buff_head lock is held.
630 static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *pkc)
632 mod_timer(&pkc->retire_blk_timer,
633 jiffies + pkc->tov_in_jiffies);
634 pkc->last_kactive_blk_num = pkc->kactive_blk_num;
639 * 1) We refresh the timer only when we open a block.
640 * By doing this we don't waste cycles refreshing the timer
641 * on packet-by-packet basis.
643 * With a 1MB block-size, on a 1Gbps line, it will take
644 * i) ~8 ms to fill a block + ii) memcpy etc.
645 * In this cut we are not accounting for the memcpy time.
647 * So, if the user sets the 'tmo' to 10ms then the timer
648 * will never fire while the block is still getting filled
649 * (which is what we want). However, the user could choose
650 * to close a block early and that's fine.
652 * But when the timer does fire, we check whether or not to refresh it.
653 * Since the tmo granularity is in msecs, it is not too expensive
654 * to refresh the timer, lets say every '8' msecs.
655 * Either the user can set the 'tmo' or we can derive it based on
656 * a) line-speed and b) block-size.
657 * prb_calc_retire_blk_tmo() calculates the tmo.
660 static void prb_retire_rx_blk_timer_expired(unsigned long data)
662 struct packet_sock *po = (struct packet_sock *)data;
663 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
665 struct tpacket_block_desc *pbd;
667 spin_lock(&po->sk.sk_receive_queue.lock);
669 frozen = prb_queue_frozen(pkc);
670 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
672 if (unlikely(pkc->delete_blk_timer))
675 /* We only need to plug the race when the block is partially filled.
677 * lock(); increment BLOCK_NUM_PKTS; unlock()
678 * copy_bits() is in progress ...
679 * timer fires on other cpu:
680 * we can't retire the current block because copy_bits
684 if (BLOCK_NUM_PKTS(pbd)) {
685 while (atomic_read(&pkc->blk_fill_in_prog)) {
686 /* Waiting for skb_copy_bits to finish... */
691 if (pkc->last_kactive_blk_num == pkc->kactive_blk_num) {
693 prb_retire_current_block(pkc, po, TP_STATUS_BLK_TMO);
694 if (!prb_dispatch_next_block(pkc, po))
699 /* Case 1. Queue was frozen because user-space was
702 if (prb_curr_blk_in_use(pkc, pbd)) {
704 * Ok, user-space is still behind.
705 * So just refresh the timer.
709 /* Case 2. queue was frozen,user-space caught up,
710 * now the link went idle && the timer fired.
711 * We don't have a block to close.So we open this
712 * block and restart the timer.
713 * opening a block thaws the queue,restarts timer
714 * Thawing/timer-refresh is a side effect.
716 prb_open_block(pkc, pbd);
723 _prb_refresh_rx_retire_blk_timer(pkc);
726 spin_unlock(&po->sk.sk_receive_queue.lock);
729 static void prb_flush_block(struct tpacket_kbdq_core *pkc1,
730 struct tpacket_block_desc *pbd1, __u32 status)
732 /* Flush everything minus the block header */
734 #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
739 /* Skip the block header(we know header WILL fit in 4K) */
742 end = (u8 *)PAGE_ALIGN((unsigned long)pkc1->pkblk_end);
743 for (; start < end; start += PAGE_SIZE)
744 flush_dcache_page(pgv_to_page(start));
749 /* Now update the block status. */
751 BLOCK_STATUS(pbd1) = status;
753 /* Flush the block header */
755 #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
757 flush_dcache_page(pgv_to_page(start));
767 * 2) Increment active_blk_num
769 * Note:We DONT refresh the timer on purpose.
770 * Because almost always the next block will be opened.
772 static void prb_close_block(struct tpacket_kbdq_core *pkc1,
773 struct tpacket_block_desc *pbd1,
774 struct packet_sock *po, unsigned int stat)
776 __u32 status = TP_STATUS_USER | stat;
778 struct tpacket3_hdr *last_pkt;
779 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
781 if (po->stats.stats3.tp_drops)
782 status |= TP_STATUS_LOSING;
784 last_pkt = (struct tpacket3_hdr *)pkc1->prev;
785 last_pkt->tp_next_offset = 0;
787 /* Get the ts of the last pkt */
788 if (BLOCK_NUM_PKTS(pbd1)) {
789 h1->ts_last_pkt.ts_sec = last_pkt->tp_sec;
790 h1->ts_last_pkt.ts_nsec = last_pkt->tp_nsec;
792 /* Ok, we tmo'd - so get the current time */
795 h1->ts_last_pkt.ts_sec = ts.tv_sec;
796 h1->ts_last_pkt.ts_nsec = ts.tv_nsec;
801 /* Flush the block */
802 prb_flush_block(pkc1, pbd1, status);
804 pkc1->kactive_blk_num = GET_NEXT_PRB_BLK_NUM(pkc1);
807 static void prb_thaw_queue(struct tpacket_kbdq_core *pkc)
809 pkc->reset_pending_on_curr_blk = 0;
813 * Side effect of opening a block:
815 * 1) prb_queue is thawed.
816 * 2) retire_blk_timer is refreshed.
819 static void prb_open_block(struct tpacket_kbdq_core *pkc1,
820 struct tpacket_block_desc *pbd1)
823 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
827 /* We could have just memset this but we will lose the
828 * flexibility of making the priv area sticky
831 BLOCK_SNUM(pbd1) = pkc1->knxt_seq_num++;
832 BLOCK_NUM_PKTS(pbd1) = 0;
833 BLOCK_LEN(pbd1) = BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
837 h1->ts_first_pkt.ts_sec = ts.tv_sec;
838 h1->ts_first_pkt.ts_nsec = ts.tv_nsec;
840 pkc1->pkblk_start = (char *)pbd1;
841 pkc1->nxt_offset = pkc1->pkblk_start + BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
843 BLOCK_O2FP(pbd1) = (__u32)BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
844 BLOCK_O2PRIV(pbd1) = BLK_HDR_LEN;
846 pbd1->version = pkc1->version;
847 pkc1->prev = pkc1->nxt_offset;
848 pkc1->pkblk_end = pkc1->pkblk_start + pkc1->kblk_size;
850 prb_thaw_queue(pkc1);
851 _prb_refresh_rx_retire_blk_timer(pkc1);
857 * Queue freeze logic:
858 * 1) Assume tp_block_nr = 8 blocks.
859 * 2) At time 't0', user opens Rx ring.
860 * 3) Some time past 't0', kernel starts filling blocks starting from 0 .. 7
861 * 4) user-space is either sleeping or processing block '0'.
862 * 5) tpacket_rcv is currently filling block '7', since there is no space left,
863 * it will close block-7,loop around and try to fill block '0'.
865 * __packet_lookup_frame_in_block
866 * prb_retire_current_block()
867 * prb_dispatch_next_block()
868 * |->(BLOCK_STATUS == USER) evaluates to true
869 * 5.1) Since block-0 is currently in-use, we just freeze the queue.
870 * 6) Now there are two cases:
871 * 6.1) Link goes idle right after the queue is frozen.
872 * But remember, the last open_block() refreshed the timer.
873 * When this timer expires,it will refresh itself so that we can
874 * re-open block-0 in near future.
875 * 6.2) Link is busy and keeps on receiving packets. This is a simple
876 * case and __packet_lookup_frame_in_block will check if block-0
877 * is free and can now be re-used.
879 static void prb_freeze_queue(struct tpacket_kbdq_core *pkc,
880 struct packet_sock *po)
882 pkc->reset_pending_on_curr_blk = 1;
883 po->stats.stats3.tp_freeze_q_cnt++;
886 #define TOTAL_PKT_LEN_INCL_ALIGN(length) (ALIGN((length), V3_ALIGNMENT))
889 * If the next block is free then we will dispatch it
890 * and return a good offset.
891 * Else, we will freeze the queue.
892 * So, caller must check the return value.
894 static void *prb_dispatch_next_block(struct tpacket_kbdq_core *pkc,
895 struct packet_sock *po)
897 struct tpacket_block_desc *pbd;
901 /* 1. Get current block num */
902 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
904 /* 2. If this block is currently in_use then freeze the queue */
905 if (TP_STATUS_USER & BLOCK_STATUS(pbd)) {
906 prb_freeze_queue(pkc, po);
912 * open this block and return the offset where the first packet
913 * needs to get stored.
915 prb_open_block(pkc, pbd);
916 return (void *)pkc->nxt_offset;
919 static void prb_retire_current_block(struct tpacket_kbdq_core *pkc,
920 struct packet_sock *po, unsigned int status)
922 struct tpacket_block_desc *pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
924 /* retire/close the current block */
925 if (likely(TP_STATUS_KERNEL == BLOCK_STATUS(pbd))) {
927 * Plug the case where copy_bits() is in progress on
928 * cpu-0 and tpacket_rcv() got invoked on cpu-1, didn't
929 * have space to copy the pkt in the current block and
930 * called prb_retire_current_block()
932 * We don't need to worry about the TMO case because
933 * the timer-handler already handled this case.
935 if (!(status & TP_STATUS_BLK_TMO)) {
936 while (atomic_read(&pkc->blk_fill_in_prog)) {
937 /* Waiting for skb_copy_bits to finish... */
941 prb_close_block(pkc, pbd, po, status);
946 static int prb_curr_blk_in_use(struct tpacket_kbdq_core *pkc,
947 struct tpacket_block_desc *pbd)
949 return TP_STATUS_USER & BLOCK_STATUS(pbd);
952 static int prb_queue_frozen(struct tpacket_kbdq_core *pkc)
954 return pkc->reset_pending_on_curr_blk;
957 static void prb_clear_blk_fill_status(struct packet_ring_buffer *rb)
959 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
960 atomic_dec(&pkc->blk_fill_in_prog);
963 static void prb_fill_rxhash(struct tpacket_kbdq_core *pkc,
964 struct tpacket3_hdr *ppd)
966 ppd->hv1.tp_rxhash = skb_get_hash(pkc->skb);
969 static void prb_clear_rxhash(struct tpacket_kbdq_core *pkc,
970 struct tpacket3_hdr *ppd)
972 ppd->hv1.tp_rxhash = 0;
975 static void prb_fill_vlan_info(struct tpacket_kbdq_core *pkc,
976 struct tpacket3_hdr *ppd)
978 if (vlan_tx_tag_present(pkc->skb)) {
979 ppd->hv1.tp_vlan_tci = vlan_tx_tag_get(pkc->skb);
980 ppd->hv1.tp_vlan_tpid = ntohs(pkc->skb->vlan_proto);
981 ppd->tp_status = TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
983 ppd->hv1.tp_vlan_tci = 0;
984 ppd->hv1.tp_vlan_tpid = 0;
985 ppd->tp_status = TP_STATUS_AVAILABLE;
989 static void prb_run_all_ft_ops(struct tpacket_kbdq_core *pkc,
990 struct tpacket3_hdr *ppd)
992 ppd->hv1.tp_padding = 0;
993 prb_fill_vlan_info(pkc, ppd);
995 if (pkc->feature_req_word & TP_FT_REQ_FILL_RXHASH)
996 prb_fill_rxhash(pkc, ppd);
998 prb_clear_rxhash(pkc, ppd);
1001 static void prb_fill_curr_block(char *curr,
1002 struct tpacket_kbdq_core *pkc,
1003 struct tpacket_block_desc *pbd,
1006 struct tpacket3_hdr *ppd;
1008 ppd = (struct tpacket3_hdr *)curr;
1009 ppd->tp_next_offset = TOTAL_PKT_LEN_INCL_ALIGN(len);
1011 pkc->nxt_offset += TOTAL_PKT_LEN_INCL_ALIGN(len);
1012 BLOCK_LEN(pbd) += TOTAL_PKT_LEN_INCL_ALIGN(len);
1013 BLOCK_NUM_PKTS(pbd) += 1;
1014 atomic_inc(&pkc->blk_fill_in_prog);
1015 prb_run_all_ft_ops(pkc, ppd);
1018 /* Assumes caller has the sk->rx_queue.lock */
1019 static void *__packet_lookup_frame_in_block(struct packet_sock *po,
1020 struct sk_buff *skb,
1025 struct tpacket_kbdq_core *pkc;
1026 struct tpacket_block_desc *pbd;
1029 pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
1030 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1032 /* Queue is frozen when user space is lagging behind */
1033 if (prb_queue_frozen(pkc)) {
1035 * Check if that last block which caused the queue to freeze,
1036 * is still in_use by user-space.
1038 if (prb_curr_blk_in_use(pkc, pbd)) {
1039 /* Can't record this packet */
1043 * Ok, the block was released by user-space.
1044 * Now let's open that block.
1045 * opening a block also thaws the queue.
1046 * Thawing is a side effect.
1048 prb_open_block(pkc, pbd);
1053 curr = pkc->nxt_offset;
1055 end = (char *)pbd + pkc->kblk_size;
1057 /* first try the current block */
1058 if (curr+TOTAL_PKT_LEN_INCL_ALIGN(len) < end) {
1059 prb_fill_curr_block(curr, pkc, pbd, len);
1060 return (void *)curr;
1063 /* Ok, close the current block */
1064 prb_retire_current_block(pkc, po, 0);
1066 /* Now, try to dispatch the next block */
1067 curr = (char *)prb_dispatch_next_block(pkc, po);
1069 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1070 prb_fill_curr_block(curr, pkc, pbd, len);
1071 return (void *)curr;
1075 * No free blocks are available.user_space hasn't caught up yet.
1076 * Queue was just frozen and now this packet will get dropped.
1081 static void *packet_current_rx_frame(struct packet_sock *po,
1082 struct sk_buff *skb,
1083 int status, unsigned int len)
1086 switch (po->tp_version) {
1089 curr = packet_lookup_frame(po, &po->rx_ring,
1090 po->rx_ring.head, status);
1093 return __packet_lookup_frame_in_block(po, skb, status, len);
1095 WARN(1, "TPACKET version not supported\n");
1101 static void *prb_lookup_block(struct packet_sock *po,
1102 struct packet_ring_buffer *rb,
1106 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
1107 struct tpacket_block_desc *pbd = GET_PBLOCK_DESC(pkc, idx);
1109 if (status != BLOCK_STATUS(pbd))
1114 static int prb_previous_blk_num(struct packet_ring_buffer *rb)
1117 if (rb->prb_bdqc.kactive_blk_num)
1118 prev = rb->prb_bdqc.kactive_blk_num-1;
1120 prev = rb->prb_bdqc.knum_blocks-1;
1124 /* Assumes caller has held the rx_queue.lock */
1125 static void *__prb_previous_block(struct packet_sock *po,
1126 struct packet_ring_buffer *rb,
1129 unsigned int previous = prb_previous_blk_num(rb);
1130 return prb_lookup_block(po, rb, previous, status);
1133 static void *packet_previous_rx_frame(struct packet_sock *po,
1134 struct packet_ring_buffer *rb,
1137 if (po->tp_version <= TPACKET_V2)
1138 return packet_previous_frame(po, rb, status);
1140 return __prb_previous_block(po, rb, status);
1143 static void packet_increment_rx_head(struct packet_sock *po,
1144 struct packet_ring_buffer *rb)
1146 switch (po->tp_version) {
1149 return packet_increment_head(rb);
1152 WARN(1, "TPACKET version not supported.\n");
1158 static void *packet_previous_frame(struct packet_sock *po,
1159 struct packet_ring_buffer *rb,
1162 unsigned int previous = rb->head ? rb->head - 1 : rb->frame_max;
1163 return packet_lookup_frame(po, rb, previous, status);
1166 static void packet_increment_head(struct packet_ring_buffer *buff)
1168 buff->head = buff->head != buff->frame_max ? buff->head+1 : 0;
1171 static bool packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb)
1173 struct sock *sk = &po->sk;
1176 if (po->prot_hook.func != tpacket_rcv)
1177 return (atomic_read(&sk->sk_rmem_alloc) + skb->truesize)
1180 spin_lock(&sk->sk_receive_queue.lock);
1181 if (po->tp_version == TPACKET_V3)
1182 has_room = prb_lookup_block(po, &po->rx_ring,
1183 po->rx_ring.prb_bdqc.kactive_blk_num,
1186 has_room = packet_lookup_frame(po, &po->rx_ring,
1189 spin_unlock(&sk->sk_receive_queue.lock);
1194 static void packet_sock_destruct(struct sock *sk)
1196 skb_queue_purge(&sk->sk_error_queue);
1198 WARN_ON(atomic_read(&sk->sk_rmem_alloc));
1199 WARN_ON(atomic_read(&sk->sk_wmem_alloc));
1201 if (!sock_flag(sk, SOCK_DEAD)) {
1202 pr_err("Attempt to release alive packet socket: %p\n", sk);
1206 sk_refcnt_debug_dec(sk);
1209 static int fanout_rr_next(struct packet_fanout *f, unsigned int num)
1211 int x = atomic_read(&f->rr_cur) + 1;
1219 static unsigned int fanout_demux_hash(struct packet_fanout *f,
1220 struct sk_buff *skb,
1223 return reciprocal_divide(skb->rxhash, num);
1226 static unsigned int fanout_demux_lb(struct packet_fanout *f,
1227 struct sk_buff *skb,
1232 cur = atomic_read(&f->rr_cur);
1233 while ((old = atomic_cmpxchg(&f->rr_cur, cur,
1234 fanout_rr_next(f, num))) != cur)
1239 static unsigned int fanout_demux_cpu(struct packet_fanout *f,
1240 struct sk_buff *skb,
1243 return smp_processor_id() % num;
1246 static unsigned int fanout_demux_rnd(struct packet_fanout *f,
1247 struct sk_buff *skb,
1250 return reciprocal_divide(prandom_u32(), num);
1253 static unsigned int fanout_demux_rollover(struct packet_fanout *f,
1254 struct sk_buff *skb,
1255 unsigned int idx, unsigned int skip,
1260 i = j = min_t(int, f->next[idx], num - 1);
1262 if (i != skip && packet_rcv_has_room(pkt_sk(f->arr[i]), skb)) {
1274 static bool fanout_has_flag(struct packet_fanout *f, u16 flag)
1276 return f->flags & (flag >> 8);
1279 static int packet_rcv_fanout(struct sk_buff *skb, struct net_device *dev,
1280 struct packet_type *pt, struct net_device *orig_dev)
1282 struct packet_fanout *f = pt->af_packet_priv;
1283 unsigned int num = f->num_members;
1284 struct packet_sock *po;
1287 if (!net_eq(dev_net(dev), read_pnet(&f->net)) ||
1294 case PACKET_FANOUT_HASH:
1296 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_DEFRAG)) {
1297 skb = ip_check_defrag(skb, IP_DEFRAG_AF_PACKET);
1302 idx = fanout_demux_hash(f, skb, num);
1304 case PACKET_FANOUT_LB:
1305 idx = fanout_demux_lb(f, skb, num);
1307 case PACKET_FANOUT_CPU:
1308 idx = fanout_demux_cpu(f, skb, num);
1310 case PACKET_FANOUT_RND:
1311 idx = fanout_demux_rnd(f, skb, num);
1313 case PACKET_FANOUT_ROLLOVER:
1314 idx = fanout_demux_rollover(f, skb, 0, (unsigned int) -1, num);
1318 po = pkt_sk(f->arr[idx]);
1319 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_ROLLOVER) &&
1320 unlikely(!packet_rcv_has_room(po, skb))) {
1321 idx = fanout_demux_rollover(f, skb, idx, idx, num);
1322 po = pkt_sk(f->arr[idx]);
1325 return po->prot_hook.func(skb, dev, &po->prot_hook, orig_dev);
1328 DEFINE_MUTEX(fanout_mutex);
1329 EXPORT_SYMBOL_GPL(fanout_mutex);
1330 static LIST_HEAD(fanout_list);
1332 static void __fanout_link(struct sock *sk, struct packet_sock *po)
1334 struct packet_fanout *f = po->fanout;
1336 spin_lock(&f->lock);
1337 f->arr[f->num_members] = sk;
1340 spin_unlock(&f->lock);
1343 static void __fanout_unlink(struct sock *sk, struct packet_sock *po)
1345 struct packet_fanout *f = po->fanout;
1348 spin_lock(&f->lock);
1349 for (i = 0; i < f->num_members; i++) {
1350 if (f->arr[i] == sk)
1353 BUG_ON(i >= f->num_members);
1354 f->arr[i] = f->arr[f->num_members - 1];
1356 spin_unlock(&f->lock);
1359 static bool match_fanout_group(struct packet_type *ptype, struct sock * sk)
1361 if (ptype->af_packet_priv == (void*)((struct packet_sock *)sk)->fanout)
1367 static int fanout_add(struct sock *sk, u16 id, u16 type_flags)
1369 struct packet_sock *po = pkt_sk(sk);
1370 struct packet_fanout *f, *match;
1371 u8 type = type_flags & 0xff;
1372 u8 flags = type_flags >> 8;
1376 case PACKET_FANOUT_ROLLOVER:
1377 if (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)
1379 case PACKET_FANOUT_HASH:
1380 case PACKET_FANOUT_LB:
1381 case PACKET_FANOUT_CPU:
1382 case PACKET_FANOUT_RND:
1394 mutex_lock(&fanout_mutex);
1396 list_for_each_entry(f, &fanout_list, list) {
1398 read_pnet(&f->net) == sock_net(sk)) {
1404 if (match && match->flags != flags)
1408 match = kzalloc(sizeof(*match), GFP_KERNEL);
1411 write_pnet(&match->net, sock_net(sk));
1414 match->flags = flags;
1415 atomic_set(&match->rr_cur, 0);
1416 INIT_LIST_HEAD(&match->list);
1417 spin_lock_init(&match->lock);
1418 atomic_set(&match->sk_ref, 0);
1419 match->prot_hook.type = po->prot_hook.type;
1420 match->prot_hook.dev = po->prot_hook.dev;
1421 match->prot_hook.func = packet_rcv_fanout;
1422 match->prot_hook.af_packet_priv = match;
1423 match->prot_hook.id_match = match_fanout_group;
1424 dev_add_pack(&match->prot_hook);
1425 list_add(&match->list, &fanout_list);
1428 if (match->type == type &&
1429 match->prot_hook.type == po->prot_hook.type &&
1430 match->prot_hook.dev == po->prot_hook.dev) {
1432 if (atomic_read(&match->sk_ref) < PACKET_FANOUT_MAX) {
1433 __dev_remove_pack(&po->prot_hook);
1435 atomic_inc(&match->sk_ref);
1436 __fanout_link(sk, po);
1441 mutex_unlock(&fanout_mutex);
1445 static void fanout_release(struct sock *sk)
1447 struct packet_sock *po = pkt_sk(sk);
1448 struct packet_fanout *f;
1454 mutex_lock(&fanout_mutex);
1457 if (atomic_dec_and_test(&f->sk_ref)) {
1459 dev_remove_pack(&f->prot_hook);
1462 mutex_unlock(&fanout_mutex);
1465 static const struct proto_ops packet_ops;
1467 static const struct proto_ops packet_ops_spkt;
1469 static int packet_rcv_spkt(struct sk_buff *skb, struct net_device *dev,
1470 struct packet_type *pt, struct net_device *orig_dev)
1473 struct sockaddr_pkt *spkt;
1476 * When we registered the protocol we saved the socket in the data
1477 * field for just this event.
1480 sk = pt->af_packet_priv;
1483 * Yank back the headers [hope the device set this
1484 * right or kerboom...]
1486 * Incoming packets have ll header pulled,
1489 * For outgoing ones skb->data == skb_mac_header(skb)
1490 * so that this procedure is noop.
1493 if (skb->pkt_type == PACKET_LOOPBACK)
1496 if (!net_eq(dev_net(dev), sock_net(sk)))
1499 skb = skb_share_check(skb, GFP_ATOMIC);
1503 /* drop any routing info */
1506 /* drop conntrack reference */
1509 spkt = &PACKET_SKB_CB(skb)->sa.pkt;
1511 skb_push(skb, skb->data - skb_mac_header(skb));
1514 * The SOCK_PACKET socket receives _all_ frames.
1517 spkt->spkt_family = dev->type;
1518 strlcpy(spkt->spkt_device, dev->name, sizeof(spkt->spkt_device));
1519 spkt->spkt_protocol = skb->protocol;
1522 * Charge the memory to the socket. This is done specifically
1523 * to prevent sockets using all the memory up.
1526 if (sock_queue_rcv_skb(sk, skb) == 0)
1537 * Output a raw packet to a device layer. This bypasses all the other
1538 * protocol layers and you must therefore supply it with a complete frame
1541 static int packet_sendmsg_spkt(struct kiocb *iocb, struct socket *sock,
1542 struct msghdr *msg, size_t len)
1544 struct sock *sk = sock->sk;
1545 struct sockaddr_pkt *saddr = (struct sockaddr_pkt *)msg->msg_name;
1546 struct sk_buff *skb = NULL;
1547 struct net_device *dev;
1553 * Get and verify the address.
1557 if (msg->msg_namelen < sizeof(struct sockaddr))
1559 if (msg->msg_namelen == sizeof(struct sockaddr_pkt))
1560 proto = saddr->spkt_protocol;
1562 return -ENOTCONN; /* SOCK_PACKET must be sent giving an address */
1565 * Find the device first to size check it
1568 saddr->spkt_device[sizeof(saddr->spkt_device) - 1] = 0;
1571 dev = dev_get_by_name_rcu(sock_net(sk), saddr->spkt_device);
1577 if (!(dev->flags & IFF_UP))
1581 * You may not queue a frame bigger than the mtu. This is the lowest level
1582 * raw protocol and you must do your own fragmentation at this level.
1585 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
1586 if (!netif_supports_nofcs(dev)) {
1587 err = -EPROTONOSUPPORT;
1590 extra_len = 4; /* We're doing our own CRC */
1594 if (len > dev->mtu + dev->hard_header_len + VLAN_HLEN + extra_len)
1598 size_t reserved = LL_RESERVED_SPACE(dev);
1599 int tlen = dev->needed_tailroom;
1600 unsigned int hhlen = dev->header_ops ? dev->hard_header_len : 0;
1603 skb = sock_wmalloc(sk, len + reserved + tlen, 0, GFP_KERNEL);
1606 /* FIXME: Save some space for broken drivers that write a hard
1607 * header at transmission time by themselves. PPP is the notable
1608 * one here. This should really be fixed at the driver level.
1610 skb_reserve(skb, reserved);
1611 skb_reset_network_header(skb);
1613 /* Try to align data part correctly */
1618 skb_reset_network_header(skb);
1620 err = memcpy_fromiovec(skb_put(skb, len), msg->msg_iov, len);
1626 if (len > (dev->mtu + dev->hard_header_len + extra_len)) {
1627 /* Earlier code assumed this would be a VLAN pkt,
1628 * double-check this now that we have the actual
1631 struct ethhdr *ehdr;
1632 skb_reset_mac_header(skb);
1633 ehdr = eth_hdr(skb);
1634 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
1640 skb->protocol = proto;
1642 skb->priority = sk->sk_priority;
1643 skb->mark = sk->sk_mark;
1645 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
1647 if (unlikely(extra_len == 4))
1650 skb_probe_transport_header(skb, 0);
1652 dev_queue_xmit(skb);
1663 static unsigned int run_filter(const struct sk_buff *skb,
1664 const struct sock *sk,
1667 struct sk_filter *filter;
1670 filter = rcu_dereference(sk->sk_filter);
1672 res = SK_RUN_FILTER(filter, skb);
1679 * This function makes lazy skb cloning in hope that most of packets
1680 * are discarded by BPF.
1682 * Note tricky part: we DO mangle shared skb! skb->data, skb->len
1683 * and skb->cb are mangled. It works because (and until) packets
1684 * falling here are owned by current CPU. Output packets are cloned
1685 * by dev_queue_xmit_nit(), input packets are processed by net_bh
1686 * sequencially, so that if we return skb to original state on exit,
1687 * we will not harm anyone.
1690 static int packet_rcv(struct sk_buff *skb, struct net_device *dev,
1691 struct packet_type *pt, struct net_device *orig_dev)
1694 struct sockaddr_ll *sll;
1695 struct packet_sock *po;
1696 u8 *skb_head = skb->data;
1697 int skb_len = skb->len;
1698 unsigned int snaplen, res;
1700 if (skb->pkt_type == PACKET_LOOPBACK)
1703 sk = pt->af_packet_priv;
1706 if (!net_eq(dev_net(dev), sock_net(sk)))
1711 if (dev->header_ops) {
1712 /* The device has an explicit notion of ll header,
1713 * exported to higher levels.
1715 * Otherwise, the device hides details of its frame
1716 * structure, so that corresponding packet head is
1717 * never delivered to user.
1719 if (sk->sk_type != SOCK_DGRAM)
1720 skb_push(skb, skb->data - skb_mac_header(skb));
1721 else if (skb->pkt_type == PACKET_OUTGOING) {
1722 /* Special case: outgoing packets have ll header at head */
1723 skb_pull(skb, skb_network_offset(skb));
1729 res = run_filter(skb, sk, snaplen);
1731 goto drop_n_restore;
1735 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf)
1738 if (skb_shared(skb)) {
1739 struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
1743 if (skb_head != skb->data) {
1744 skb->data = skb_head;
1751 BUILD_BUG_ON(sizeof(*PACKET_SKB_CB(skb)) + MAX_ADDR_LEN - 8 >
1754 sll = &PACKET_SKB_CB(skb)->sa.ll;
1755 sll->sll_family = AF_PACKET;
1756 sll->sll_hatype = dev->type;
1757 sll->sll_protocol = skb->protocol;
1758 sll->sll_pkttype = skb->pkt_type;
1759 if (unlikely(po->origdev))
1760 sll->sll_ifindex = orig_dev->ifindex;
1762 sll->sll_ifindex = dev->ifindex;
1764 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1766 PACKET_SKB_CB(skb)->origlen = skb->len;
1768 if (pskb_trim(skb, snaplen))
1771 skb_set_owner_r(skb, sk);
1775 /* drop conntrack reference */
1778 spin_lock(&sk->sk_receive_queue.lock);
1779 po->stats.stats1.tp_packets++;
1780 skb->dropcount = atomic_read(&sk->sk_drops);
1781 __skb_queue_tail(&sk->sk_receive_queue, skb);
1782 spin_unlock(&sk->sk_receive_queue.lock);
1783 sk->sk_data_ready(sk, skb->len);
1787 spin_lock(&sk->sk_receive_queue.lock);
1788 po->stats.stats1.tp_drops++;
1789 atomic_inc(&sk->sk_drops);
1790 spin_unlock(&sk->sk_receive_queue.lock);
1793 if (skb_head != skb->data && skb_shared(skb)) {
1794 skb->data = skb_head;
1802 static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
1803 struct packet_type *pt, struct net_device *orig_dev)
1806 struct packet_sock *po;
1807 struct sockaddr_ll *sll;
1808 union tpacket_uhdr h;
1809 u8 *skb_head = skb->data;
1810 int skb_len = skb->len;
1811 unsigned int snaplen, res;
1812 unsigned long status = TP_STATUS_USER;
1813 unsigned short macoff, netoff, hdrlen;
1814 struct sk_buff *copy_skb = NULL;
1818 /* struct tpacket{2,3}_hdr is aligned to a multiple of TPACKET_ALIGNMENT.
1819 * We may add members to them until current aligned size without forcing
1820 * userspace to call getsockopt(..., PACKET_HDRLEN, ...).
1822 BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h2)) != 32);
1823 BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h3)) != 48);
1825 if (skb->pkt_type == PACKET_LOOPBACK)
1828 sk = pt->af_packet_priv;
1831 if (!net_eq(dev_net(dev), sock_net(sk)))
1834 if (dev->header_ops) {
1835 if (sk->sk_type != SOCK_DGRAM)
1836 skb_push(skb, skb->data - skb_mac_header(skb));
1837 else if (skb->pkt_type == PACKET_OUTGOING) {
1838 /* Special case: outgoing packets have ll header at head */
1839 skb_pull(skb, skb_network_offset(skb));
1843 if (skb->ip_summed == CHECKSUM_PARTIAL)
1844 status |= TP_STATUS_CSUMNOTREADY;
1848 res = run_filter(skb, sk, snaplen);
1850 goto drop_n_restore;
1854 if (sk->sk_type == SOCK_DGRAM) {
1855 macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 +
1858 unsigned int maclen = skb_network_offset(skb);
1859 netoff = TPACKET_ALIGN(po->tp_hdrlen +
1860 (maclen < 16 ? 16 : maclen)) +
1862 macoff = netoff - maclen;
1864 if (po->tp_version <= TPACKET_V2) {
1865 if (macoff + snaplen > po->rx_ring.frame_size) {
1866 if (po->copy_thresh &&
1867 atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf) {
1868 if (skb_shared(skb)) {
1869 copy_skb = skb_clone(skb, GFP_ATOMIC);
1871 copy_skb = skb_get(skb);
1872 skb_head = skb->data;
1875 skb_set_owner_r(copy_skb, sk);
1877 snaplen = po->rx_ring.frame_size - macoff;
1878 if ((int)snaplen < 0)
1882 spin_lock(&sk->sk_receive_queue.lock);
1883 h.raw = packet_current_rx_frame(po, skb,
1884 TP_STATUS_KERNEL, (macoff+snaplen));
1887 if (po->tp_version <= TPACKET_V2) {
1888 packet_increment_rx_head(po, &po->rx_ring);
1890 * LOSING will be reported till you read the stats,
1891 * because it's COR - Clear On Read.
1892 * Anyways, moving it for V1/V2 only as V3 doesn't need this
1895 if (po->stats.stats1.tp_drops)
1896 status |= TP_STATUS_LOSING;
1898 po->stats.stats1.tp_packets++;
1900 status |= TP_STATUS_COPY;
1901 __skb_queue_tail(&sk->sk_receive_queue, copy_skb);
1903 spin_unlock(&sk->sk_receive_queue.lock);
1905 skb_copy_bits(skb, 0, h.raw + macoff, snaplen);
1907 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
1908 getnstimeofday(&ts);
1910 status |= ts_status;
1912 switch (po->tp_version) {
1914 h.h1->tp_len = skb->len;
1915 h.h1->tp_snaplen = snaplen;
1916 h.h1->tp_mac = macoff;
1917 h.h1->tp_net = netoff;
1918 h.h1->tp_sec = ts.tv_sec;
1919 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
1920 hdrlen = sizeof(*h.h1);
1923 h.h2->tp_len = skb->len;
1924 h.h2->tp_snaplen = snaplen;
1925 h.h2->tp_mac = macoff;
1926 h.h2->tp_net = netoff;
1927 h.h2->tp_sec = ts.tv_sec;
1928 h.h2->tp_nsec = ts.tv_nsec;
1929 if (vlan_tx_tag_present(skb)) {
1930 h.h2->tp_vlan_tci = vlan_tx_tag_get(skb);
1931 h.h2->tp_vlan_tpid = ntohs(skb->vlan_proto);
1932 status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
1934 h.h2->tp_vlan_tci = 0;
1935 h.h2->tp_vlan_tpid = 0;
1937 memset(h.h2->tp_padding, 0, sizeof(h.h2->tp_padding));
1938 hdrlen = sizeof(*h.h2);
1941 /* tp_nxt_offset,vlan are already populated above.
1942 * So DONT clear those fields here
1944 h.h3->tp_status |= status;
1945 h.h3->tp_len = skb->len;
1946 h.h3->tp_snaplen = snaplen;
1947 h.h3->tp_mac = macoff;
1948 h.h3->tp_net = netoff;
1949 h.h3->tp_sec = ts.tv_sec;
1950 h.h3->tp_nsec = ts.tv_nsec;
1951 memset(h.h3->tp_padding, 0, sizeof(h.h3->tp_padding));
1952 hdrlen = sizeof(*h.h3);
1958 sll = h.raw + TPACKET_ALIGN(hdrlen);
1959 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1960 sll->sll_family = AF_PACKET;
1961 sll->sll_hatype = dev->type;
1962 sll->sll_protocol = skb->protocol;
1963 sll->sll_pkttype = skb->pkt_type;
1964 if (unlikely(po->origdev))
1965 sll->sll_ifindex = orig_dev->ifindex;
1967 sll->sll_ifindex = dev->ifindex;
1970 #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
1974 if (po->tp_version <= TPACKET_V2) {
1975 end = (u8 *)PAGE_ALIGN((unsigned long)h.raw
1976 + macoff + snaplen);
1977 for (start = h.raw; start < end; start += PAGE_SIZE)
1978 flush_dcache_page(pgv_to_page(start));
1983 if (po->tp_version <= TPACKET_V2)
1984 __packet_set_status(po, h.raw, status);
1986 prb_clear_blk_fill_status(&po->rx_ring);
1988 sk->sk_data_ready(sk, 0);
1991 if (skb_head != skb->data && skb_shared(skb)) {
1992 skb->data = skb_head;
2000 po->stats.stats1.tp_drops++;
2001 spin_unlock(&sk->sk_receive_queue.lock);
2003 sk->sk_data_ready(sk, 0);
2004 kfree_skb(copy_skb);
2005 goto drop_n_restore;
2008 static void tpacket_destruct_skb(struct sk_buff *skb)
2010 struct packet_sock *po = pkt_sk(skb->sk);
2013 if (likely(po->tx_ring.pg_vec)) {
2016 ph = skb_shinfo(skb)->destructor_arg;
2017 BUG_ON(atomic_read(&po->tx_ring.pending) == 0);
2018 atomic_dec(&po->tx_ring.pending);
2020 ts = __packet_set_timestamp(po, ph, skb);
2021 __packet_set_status(po, ph, TP_STATUS_AVAILABLE | ts);
2027 static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb,
2028 void *frame, struct net_device *dev, int size_max,
2029 __be16 proto, unsigned char *addr, int hlen)
2031 union tpacket_uhdr ph;
2032 int to_write, offset, len, tp_len, nr_frags, len_max;
2033 struct socket *sock = po->sk.sk_socket;
2040 skb->protocol = proto;
2042 skb->priority = po->sk.sk_priority;
2043 skb->mark = po->sk.sk_mark;
2044 sock_tx_timestamp(&po->sk, &skb_shinfo(skb)->tx_flags);
2045 skb_shinfo(skb)->destructor_arg = ph.raw;
2047 switch (po->tp_version) {
2049 tp_len = ph.h2->tp_len;
2052 tp_len = ph.h1->tp_len;
2055 if (unlikely(tp_len > size_max)) {
2056 pr_err("packet size is too long (%d > %d)\n", tp_len, size_max);
2060 skb_reserve(skb, hlen);
2061 skb_reset_network_header(skb);
2063 if (!packet_use_direct_xmit(po))
2064 skb_probe_transport_header(skb, 0);
2065 if (unlikely(po->tp_tx_has_off)) {
2066 int off_min, off_max, off;
2067 off_min = po->tp_hdrlen - sizeof(struct sockaddr_ll);
2068 off_max = po->tx_ring.frame_size - tp_len;
2069 if (sock->type == SOCK_DGRAM) {
2070 switch (po->tp_version) {
2072 off = ph.h2->tp_net;
2075 off = ph.h1->tp_net;
2079 switch (po->tp_version) {
2081 off = ph.h2->tp_mac;
2084 off = ph.h1->tp_mac;
2088 if (unlikely((off < off_min) || (off_max < off)))
2090 data = ph.raw + off;
2092 data = ph.raw + po->tp_hdrlen - sizeof(struct sockaddr_ll);
2096 if (sock->type == SOCK_DGRAM) {
2097 err = dev_hard_header(skb, dev, ntohs(proto), addr,
2099 if (unlikely(err < 0))
2101 } else if (dev->hard_header_len) {
2102 /* net device doesn't like empty head */
2103 if (unlikely(tp_len <= dev->hard_header_len)) {
2104 pr_err("packet size is too short (%d < %d)\n",
2105 tp_len, dev->hard_header_len);
2109 skb_push(skb, dev->hard_header_len);
2110 err = skb_store_bits(skb, 0, data,
2111 dev->hard_header_len);
2115 data += dev->hard_header_len;
2116 to_write -= dev->hard_header_len;
2119 offset = offset_in_page(data);
2120 len_max = PAGE_SIZE - offset;
2121 len = ((to_write > len_max) ? len_max : to_write);
2123 skb->data_len = to_write;
2124 skb->len += to_write;
2125 skb->truesize += to_write;
2126 atomic_add(to_write, &po->sk.sk_wmem_alloc);
2128 while (likely(to_write)) {
2129 nr_frags = skb_shinfo(skb)->nr_frags;
2131 if (unlikely(nr_frags >= MAX_SKB_FRAGS)) {
2132 pr_err("Packet exceed the number of skb frags(%lu)\n",
2137 page = pgv_to_page(data);
2139 flush_dcache_page(page);
2141 skb_fill_page_desc(skb, nr_frags, page, offset, len);
2144 len_max = PAGE_SIZE;
2145 len = ((to_write > len_max) ? len_max : to_write);
2151 static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
2153 struct sk_buff *skb;
2154 struct net_device *dev;
2156 int err, reserve = 0;
2158 struct sockaddr_ll *saddr = (struct sockaddr_ll *)msg->msg_name;
2159 int tp_len, size_max;
2160 unsigned char *addr;
2162 int status = TP_STATUS_AVAILABLE;
2165 mutex_lock(&po->pg_vec_lock);
2167 if (likely(saddr == NULL)) {
2168 dev = packet_cached_dev_get(po);
2173 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2175 if (msg->msg_namelen < (saddr->sll_halen
2176 + offsetof(struct sockaddr_ll,
2179 proto = saddr->sll_protocol;
2180 addr = saddr->sll_addr;
2181 dev = dev_get_by_index(sock_net(&po->sk), saddr->sll_ifindex);
2185 if (unlikely(dev == NULL))
2188 if (unlikely(!(dev->flags & IFF_UP)))
2191 reserve = dev->hard_header_len;
2193 size_max = po->tx_ring.frame_size
2194 - (po->tp_hdrlen - sizeof(struct sockaddr_ll));
2196 if (size_max > dev->mtu + reserve)
2197 size_max = dev->mtu + reserve;
2200 ph = packet_current_frame(po, &po->tx_ring,
2201 TP_STATUS_SEND_REQUEST);
2203 if (unlikely(ph == NULL)) {
2208 status = TP_STATUS_SEND_REQUEST;
2209 hlen = LL_RESERVED_SPACE(dev);
2210 tlen = dev->needed_tailroom;
2211 skb = sock_alloc_send_skb(&po->sk,
2212 hlen + tlen + sizeof(struct sockaddr_ll),
2215 if (unlikely(skb == NULL))
2218 tp_len = tpacket_fill_skb(po, skb, ph, dev, size_max, proto,
2221 if (unlikely(tp_len < 0)) {
2223 __packet_set_status(po, ph,
2224 TP_STATUS_AVAILABLE);
2225 packet_increment_head(&po->tx_ring);
2229 status = TP_STATUS_WRONG_FORMAT;
2235 skb_set_queue_mapping(skb, packet_pick_tx_queue(dev));
2236 skb->destructor = tpacket_destruct_skb;
2237 __packet_set_status(po, ph, TP_STATUS_SENDING);
2238 atomic_inc(&po->tx_ring.pending);
2240 status = TP_STATUS_SEND_REQUEST;
2241 err = po->xmit(skb);
2242 if (unlikely(err > 0)) {
2243 err = net_xmit_errno(err);
2244 if (err && __packet_get_status(po, ph) ==
2245 TP_STATUS_AVAILABLE) {
2246 /* skb was destructed already */
2251 * skb was dropped but not destructed yet;
2252 * let's treat it like congestion or err < 0
2256 packet_increment_head(&po->tx_ring);
2258 } while (likely((ph != NULL) ||
2259 ((!(msg->msg_flags & MSG_DONTWAIT)) &&
2260 (atomic_read(&po->tx_ring.pending))))
2267 __packet_set_status(po, ph, status);
2272 mutex_unlock(&po->pg_vec_lock);
2276 static struct sk_buff *packet_alloc_skb(struct sock *sk, size_t prepad,
2277 size_t reserve, size_t len,
2278 size_t linear, int noblock,
2281 struct sk_buff *skb;
2283 /* Under a page? Don't bother with paged skb. */
2284 if (prepad + len < PAGE_SIZE || !linear)
2287 skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock,
2292 skb_reserve(skb, reserve);
2293 skb_put(skb, linear);
2294 skb->data_len = len - linear;
2295 skb->len += len - linear;
2300 static int packet_snd(struct socket *sock, struct msghdr *msg, size_t len)
2302 struct sock *sk = sock->sk;
2303 struct sockaddr_ll *saddr = (struct sockaddr_ll *)msg->msg_name;
2304 struct sk_buff *skb;
2305 struct net_device *dev;
2307 unsigned char *addr;
2308 int err, reserve = 0;
2309 struct virtio_net_hdr vnet_hdr = { 0 };
2312 struct packet_sock *po = pkt_sk(sk);
2313 unsigned short gso_type = 0;
2318 * Get and verify the address.
2321 if (likely(saddr == NULL)) {
2322 dev = packet_cached_dev_get(po);
2327 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2329 if (msg->msg_namelen < (saddr->sll_halen + offsetof(struct sockaddr_ll, sll_addr)))
2331 proto = saddr->sll_protocol;
2332 addr = saddr->sll_addr;
2333 dev = dev_get_by_index(sock_net(sk), saddr->sll_ifindex);
2337 if (unlikely(dev == NULL))
2340 if (unlikely(!(dev->flags & IFF_UP)))
2343 if (sock->type == SOCK_RAW)
2344 reserve = dev->hard_header_len;
2345 if (po->has_vnet_hdr) {
2346 vnet_hdr_len = sizeof(vnet_hdr);
2349 if (len < vnet_hdr_len)
2352 len -= vnet_hdr_len;
2354 err = memcpy_fromiovec((void *)&vnet_hdr, msg->msg_iov,
2359 if ((vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
2360 (vnet_hdr.csum_start + vnet_hdr.csum_offset + 2 >
2362 vnet_hdr.hdr_len = vnet_hdr.csum_start +
2363 vnet_hdr.csum_offset + 2;
2366 if (vnet_hdr.hdr_len > len)
2369 if (vnet_hdr.gso_type != VIRTIO_NET_HDR_GSO_NONE) {
2370 switch (vnet_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
2371 case VIRTIO_NET_HDR_GSO_TCPV4:
2372 gso_type = SKB_GSO_TCPV4;
2374 case VIRTIO_NET_HDR_GSO_TCPV6:
2375 gso_type = SKB_GSO_TCPV6;
2377 case VIRTIO_NET_HDR_GSO_UDP:
2378 gso_type = SKB_GSO_UDP;
2384 if (vnet_hdr.gso_type & VIRTIO_NET_HDR_GSO_ECN)
2385 gso_type |= SKB_GSO_TCP_ECN;
2387 if (vnet_hdr.gso_size == 0)
2393 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
2394 if (!netif_supports_nofcs(dev)) {
2395 err = -EPROTONOSUPPORT;
2398 extra_len = 4; /* We're doing our own CRC */
2402 if (!gso_type && (len > dev->mtu + reserve + VLAN_HLEN + extra_len))
2406 hlen = LL_RESERVED_SPACE(dev);
2407 tlen = dev->needed_tailroom;
2408 skb = packet_alloc_skb(sk, hlen + tlen, hlen, len, vnet_hdr.hdr_len,
2409 msg->msg_flags & MSG_DONTWAIT, &err);
2413 skb_set_network_header(skb, reserve);
2416 if (sock->type == SOCK_DGRAM &&
2417 (offset = dev_hard_header(skb, dev, ntohs(proto), addr, NULL, len)) < 0)
2420 /* Returns -EFAULT on error */
2421 err = skb_copy_datagram_from_iovec(skb, offset, msg->msg_iov, 0, len);
2425 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
2427 if (!gso_type && (len > dev->mtu + reserve + extra_len)) {
2428 /* Earlier code assumed this would be a VLAN pkt,
2429 * double-check this now that we have the actual
2432 struct ethhdr *ehdr;
2433 skb_reset_mac_header(skb);
2434 ehdr = eth_hdr(skb);
2435 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
2441 skb->protocol = proto;
2443 skb->priority = sk->sk_priority;
2444 skb->mark = sk->sk_mark;
2445 skb_set_queue_mapping(skb, packet_pick_tx_queue(dev));
2447 if (po->has_vnet_hdr) {
2448 if (vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
2449 if (!skb_partial_csum_set(skb, vnet_hdr.csum_start,
2450 vnet_hdr.csum_offset)) {
2456 skb_shinfo(skb)->gso_size = vnet_hdr.gso_size;
2457 skb_shinfo(skb)->gso_type = gso_type;
2459 /* Header must be checked, and gso_segs computed. */
2460 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
2461 skb_shinfo(skb)->gso_segs = 0;
2463 len += vnet_hdr_len;
2466 if (!packet_use_direct_xmit(po))
2467 skb_probe_transport_header(skb, reserve);
2468 if (unlikely(extra_len == 4))
2471 err = po->xmit(skb);
2472 if (err > 0 && (err = net_xmit_errno(err)) != 0)
2488 static int packet_sendmsg(struct kiocb *iocb, struct socket *sock,
2489 struct msghdr *msg, size_t len)
2491 struct sock *sk = sock->sk;
2492 struct packet_sock *po = pkt_sk(sk);
2494 if (po->tx_ring.pg_vec)
2495 return tpacket_snd(po, msg);
2497 return packet_snd(sock, msg, len);
2501 * Close a PACKET socket. This is fairly simple. We immediately go
2502 * to 'closed' state and remove our protocol entry in the device list.
2505 static int packet_release(struct socket *sock)
2507 struct sock *sk = sock->sk;
2508 struct packet_sock *po;
2510 union tpacket_req_u req_u;
2518 mutex_lock(&net->packet.sklist_lock);
2519 sk_del_node_init_rcu(sk);
2520 mutex_unlock(&net->packet.sklist_lock);
2523 sock_prot_inuse_add(net, sk->sk_prot, -1);
2526 spin_lock(&po->bind_lock);
2527 unregister_prot_hook(sk, false);
2528 packet_cached_dev_reset(po);
2530 if (po->prot_hook.dev) {
2531 dev_put(po->prot_hook.dev);
2532 po->prot_hook.dev = NULL;
2534 spin_unlock(&po->bind_lock);
2536 packet_flush_mclist(sk);
2538 if (po->rx_ring.pg_vec) {
2539 memset(&req_u, 0, sizeof(req_u));
2540 packet_set_ring(sk, &req_u, 1, 0);
2543 if (po->tx_ring.pg_vec) {
2544 memset(&req_u, 0, sizeof(req_u));
2545 packet_set_ring(sk, &req_u, 1, 1);
2552 * Now the socket is dead. No more input will appear.
2559 skb_queue_purge(&sk->sk_receive_queue);
2560 sk_refcnt_debug_release(sk);
2567 * Attach a packet hook.
2570 static int packet_do_bind(struct sock *sk, struct net_device *dev, __be16 protocol)
2572 struct packet_sock *po = pkt_sk(sk);
2583 spin_lock(&po->bind_lock);
2584 unregister_prot_hook(sk, true);
2587 po->prot_hook.type = protocol;
2588 if (po->prot_hook.dev)
2589 dev_put(po->prot_hook.dev);
2591 po->prot_hook.dev = dev;
2592 po->ifindex = dev ? dev->ifindex : 0;
2594 packet_cached_dev_assign(po, dev);
2599 if (!dev || (dev->flags & IFF_UP)) {
2600 register_prot_hook(sk);
2602 sk->sk_err = ENETDOWN;
2603 if (!sock_flag(sk, SOCK_DEAD))
2604 sk->sk_error_report(sk);
2608 spin_unlock(&po->bind_lock);
2614 * Bind a packet socket to a device
2617 static int packet_bind_spkt(struct socket *sock, struct sockaddr *uaddr,
2620 struct sock *sk = sock->sk;
2622 struct net_device *dev;
2629 if (addr_len != sizeof(struct sockaddr))
2631 strlcpy(name, uaddr->sa_data, sizeof(name));
2633 dev = dev_get_by_name(sock_net(sk), name);
2635 err = packet_do_bind(sk, dev, pkt_sk(sk)->num);
2639 static int packet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
2641 struct sockaddr_ll *sll = (struct sockaddr_ll *)uaddr;
2642 struct sock *sk = sock->sk;
2643 struct net_device *dev = NULL;
2651 if (addr_len < sizeof(struct sockaddr_ll))
2653 if (sll->sll_family != AF_PACKET)
2656 if (sll->sll_ifindex) {
2658 dev = dev_get_by_index(sock_net(sk), sll->sll_ifindex);
2662 err = packet_do_bind(sk, dev, sll->sll_protocol ? : pkt_sk(sk)->num);
2668 static struct proto packet_proto = {
2670 .owner = THIS_MODULE,
2671 .obj_size = sizeof(struct packet_sock),
2675 * Create a packet of type SOCK_PACKET.
2678 static int packet_create(struct net *net, struct socket *sock, int protocol,
2682 struct packet_sock *po;
2683 __be16 proto = (__force __be16)protocol; /* weird, but documented */
2686 if (!ns_capable(net->user_ns, CAP_NET_RAW))
2688 if (sock->type != SOCK_DGRAM && sock->type != SOCK_RAW &&
2689 sock->type != SOCK_PACKET)
2690 return -ESOCKTNOSUPPORT;
2692 sock->state = SS_UNCONNECTED;
2695 sk = sk_alloc(net, PF_PACKET, GFP_KERNEL, &packet_proto);
2699 sock->ops = &packet_ops;
2700 if (sock->type == SOCK_PACKET)
2701 sock->ops = &packet_ops_spkt;
2703 sock_init_data(sock, sk);
2706 sk->sk_family = PF_PACKET;
2708 po->xmit = dev_queue_xmit;
2710 packet_cached_dev_reset(po);
2712 sk->sk_destruct = packet_sock_destruct;
2713 sk_refcnt_debug_inc(sk);
2716 * Attach a protocol block
2719 spin_lock_init(&po->bind_lock);
2720 mutex_init(&po->pg_vec_lock);
2721 po->prot_hook.func = packet_rcv;
2723 if (sock->type == SOCK_PACKET)
2724 po->prot_hook.func = packet_rcv_spkt;
2726 po->prot_hook.af_packet_priv = sk;
2729 po->prot_hook.type = proto;
2730 register_prot_hook(sk);
2733 mutex_lock(&net->packet.sklist_lock);
2734 sk_add_node_rcu(sk, &net->packet.sklist);
2735 mutex_unlock(&net->packet.sklist_lock);
2738 sock_prot_inuse_add(net, &packet_proto, 1);
2747 * Pull a packet from our receive queue and hand it to the user.
2748 * If necessary we block.
2751 static int packet_recvmsg(struct kiocb *iocb, struct socket *sock,
2752 struct msghdr *msg, size_t len, int flags)
2754 struct sock *sk = sock->sk;
2755 struct sk_buff *skb;
2757 int vnet_hdr_len = 0;
2760 if (flags & ~(MSG_PEEK|MSG_DONTWAIT|MSG_TRUNC|MSG_CMSG_COMPAT|MSG_ERRQUEUE))
2764 /* What error should we return now? EUNATTACH? */
2765 if (pkt_sk(sk)->ifindex < 0)
2769 if (flags & MSG_ERRQUEUE) {
2770 err = sock_recv_errqueue(sk, msg, len,
2771 SOL_PACKET, PACKET_TX_TIMESTAMP);
2776 * Call the generic datagram receiver. This handles all sorts
2777 * of horrible races and re-entrancy so we can forget about it
2778 * in the protocol layers.
2780 * Now it will return ENETDOWN, if device have just gone down,
2781 * but then it will block.
2784 skb = skb_recv_datagram(sk, flags, flags & MSG_DONTWAIT, &err);
2787 * An error occurred so return it. Because skb_recv_datagram()
2788 * handles the blocking we don't see and worry about blocking
2795 if (pkt_sk(sk)->has_vnet_hdr) {
2796 struct virtio_net_hdr vnet_hdr = { 0 };
2799 vnet_hdr_len = sizeof(vnet_hdr);
2800 if (len < vnet_hdr_len)
2803 len -= vnet_hdr_len;
2805 if (skb_is_gso(skb)) {
2806 struct skb_shared_info *sinfo = skb_shinfo(skb);
2808 /* This is a hint as to how much should be linear. */
2809 vnet_hdr.hdr_len = skb_headlen(skb);
2810 vnet_hdr.gso_size = sinfo->gso_size;
2811 if (sinfo->gso_type & SKB_GSO_TCPV4)
2812 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
2813 else if (sinfo->gso_type & SKB_GSO_TCPV6)
2814 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
2815 else if (sinfo->gso_type & SKB_GSO_UDP)
2816 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_UDP;
2817 else if (sinfo->gso_type & SKB_GSO_FCOE)
2821 if (sinfo->gso_type & SKB_GSO_TCP_ECN)
2822 vnet_hdr.gso_type |= VIRTIO_NET_HDR_GSO_ECN;
2824 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
2826 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2827 vnet_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
2828 vnet_hdr.csum_start = skb_checksum_start_offset(skb);
2829 vnet_hdr.csum_offset = skb->csum_offset;
2830 } else if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
2831 vnet_hdr.flags = VIRTIO_NET_HDR_F_DATA_VALID;
2832 } /* else everything is zero */
2834 err = memcpy_toiovec(msg->msg_iov, (void *)&vnet_hdr,
2840 /* You lose any data beyond the buffer you gave. If it worries
2841 * a user program they can ask the device for its MTU
2847 msg->msg_flags |= MSG_TRUNC;
2850 err = skb_copy_datagram_iovec(skb, 0, msg->msg_iov, copied);
2854 sock_recv_ts_and_drops(msg, sk, skb);
2856 if (msg->msg_name) {
2857 /* If the address length field is there to be filled
2858 * in, we fill it in now.
2860 if (sock->type == SOCK_PACKET) {
2861 msg->msg_namelen = sizeof(struct sockaddr_pkt);
2863 struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;
2864 msg->msg_namelen = sll->sll_halen +
2865 offsetof(struct sockaddr_ll, sll_addr);
2867 memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa,
2871 if (pkt_sk(sk)->auxdata) {
2872 struct tpacket_auxdata aux;
2874 aux.tp_status = TP_STATUS_USER;
2875 if (skb->ip_summed == CHECKSUM_PARTIAL)
2876 aux.tp_status |= TP_STATUS_CSUMNOTREADY;
2877 aux.tp_len = PACKET_SKB_CB(skb)->origlen;
2878 aux.tp_snaplen = skb->len;
2880 aux.tp_net = skb_network_offset(skb);
2881 if (vlan_tx_tag_present(skb)) {
2882 aux.tp_vlan_tci = vlan_tx_tag_get(skb);
2883 aux.tp_vlan_tpid = ntohs(skb->vlan_proto);
2884 aux.tp_status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
2886 aux.tp_vlan_tci = 0;
2887 aux.tp_vlan_tpid = 0;
2889 put_cmsg(msg, SOL_PACKET, PACKET_AUXDATA, sizeof(aux), &aux);
2893 * Free or return the buffer as appropriate. Again this
2894 * hides all the races and re-entrancy issues from us.
2896 err = vnet_hdr_len + ((flags&MSG_TRUNC) ? skb->len : copied);
2899 skb_free_datagram(sk, skb);
2904 static int packet_getname_spkt(struct socket *sock, struct sockaddr *uaddr,
2905 int *uaddr_len, int peer)
2907 struct net_device *dev;
2908 struct sock *sk = sock->sk;
2913 uaddr->sa_family = AF_PACKET;
2914 memset(uaddr->sa_data, 0, sizeof(uaddr->sa_data));
2916 dev = dev_get_by_index_rcu(sock_net(sk), pkt_sk(sk)->ifindex);
2918 strlcpy(uaddr->sa_data, dev->name, sizeof(uaddr->sa_data));
2920 *uaddr_len = sizeof(*uaddr);
2925 static int packet_getname(struct socket *sock, struct sockaddr *uaddr,
2926 int *uaddr_len, int peer)
2928 struct net_device *dev;
2929 struct sock *sk = sock->sk;
2930 struct packet_sock *po = pkt_sk(sk);
2931 DECLARE_SOCKADDR(struct sockaddr_ll *, sll, uaddr);
2936 sll->sll_family = AF_PACKET;
2937 sll->sll_ifindex = po->ifindex;
2938 sll->sll_protocol = po->num;
2939 sll->sll_pkttype = 0;
2941 dev = dev_get_by_index_rcu(sock_net(sk), po->ifindex);
2943 sll->sll_hatype = dev->type;
2944 sll->sll_halen = dev->addr_len;
2945 memcpy(sll->sll_addr, dev->dev_addr, dev->addr_len);
2947 sll->sll_hatype = 0; /* Bad: we have no ARPHRD_UNSPEC */
2951 *uaddr_len = offsetof(struct sockaddr_ll, sll_addr) + sll->sll_halen;
2956 static int packet_dev_mc(struct net_device *dev, struct packet_mclist *i,
2960 case PACKET_MR_MULTICAST:
2961 if (i->alen != dev->addr_len)
2964 return dev_mc_add(dev, i->addr);
2966 return dev_mc_del(dev, i->addr);
2968 case PACKET_MR_PROMISC:
2969 return dev_set_promiscuity(dev, what);
2971 case PACKET_MR_ALLMULTI:
2972 return dev_set_allmulti(dev, what);
2974 case PACKET_MR_UNICAST:
2975 if (i->alen != dev->addr_len)
2978 return dev_uc_add(dev, i->addr);
2980 return dev_uc_del(dev, i->addr);
2988 static void packet_dev_mclist(struct net_device *dev, struct packet_mclist *i, int what)
2990 for ( ; i; i = i->next) {
2991 if (i->ifindex == dev->ifindex)
2992 packet_dev_mc(dev, i, what);
2996 static int packet_mc_add(struct sock *sk, struct packet_mreq_max *mreq)
2998 struct packet_sock *po = pkt_sk(sk);
2999 struct packet_mclist *ml, *i;
3000 struct net_device *dev;
3006 dev = __dev_get_by_index(sock_net(sk), mreq->mr_ifindex);
3011 if (mreq->mr_alen > dev->addr_len)
3015 i = kmalloc(sizeof(*i), GFP_KERNEL);
3020 for (ml = po->mclist; ml; ml = ml->next) {
3021 if (ml->ifindex == mreq->mr_ifindex &&
3022 ml->type == mreq->mr_type &&
3023 ml->alen == mreq->mr_alen &&
3024 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3026 /* Free the new element ... */
3032 i->type = mreq->mr_type;
3033 i->ifindex = mreq->mr_ifindex;
3034 i->alen = mreq->mr_alen;
3035 memcpy(i->addr, mreq->mr_address, i->alen);
3037 i->next = po->mclist;
3039 err = packet_dev_mc(dev, i, 1);
3041 po->mclist = i->next;
3050 static int packet_mc_drop(struct sock *sk, struct packet_mreq_max *mreq)
3052 struct packet_mclist *ml, **mlp;
3056 for (mlp = &pkt_sk(sk)->mclist; (ml = *mlp) != NULL; mlp = &ml->next) {
3057 if (ml->ifindex == mreq->mr_ifindex &&
3058 ml->type == mreq->mr_type &&
3059 ml->alen == mreq->mr_alen &&
3060 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3061 if (--ml->count == 0) {
3062 struct net_device *dev;
3064 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3066 packet_dev_mc(dev, ml, -1);
3074 return -EADDRNOTAVAIL;
3077 static void packet_flush_mclist(struct sock *sk)
3079 struct packet_sock *po = pkt_sk(sk);
3080 struct packet_mclist *ml;
3086 while ((ml = po->mclist) != NULL) {
3087 struct net_device *dev;
3089 po->mclist = ml->next;
3090 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3092 packet_dev_mc(dev, ml, -1);
3099 packet_setsockopt(struct socket *sock, int level, int optname, char __user *optval, unsigned int optlen)
3101 struct sock *sk = sock->sk;
3102 struct packet_sock *po = pkt_sk(sk);
3105 if (level != SOL_PACKET)
3106 return -ENOPROTOOPT;
3109 case PACKET_ADD_MEMBERSHIP:
3110 case PACKET_DROP_MEMBERSHIP:
3112 struct packet_mreq_max mreq;
3114 memset(&mreq, 0, sizeof(mreq));
3115 if (len < sizeof(struct packet_mreq))
3117 if (len > sizeof(mreq))
3119 if (copy_from_user(&mreq, optval, len))
3121 if (len < (mreq.mr_alen + offsetof(struct packet_mreq, mr_address)))
3123 if (optname == PACKET_ADD_MEMBERSHIP)
3124 ret = packet_mc_add(sk, &mreq);
3126 ret = packet_mc_drop(sk, &mreq);
3130 case PACKET_RX_RING:
3131 case PACKET_TX_RING:
3133 union tpacket_req_u req_u;
3136 switch (po->tp_version) {
3139 len = sizeof(req_u.req);
3143 len = sizeof(req_u.req3);
3148 if (pkt_sk(sk)->has_vnet_hdr)
3150 if (copy_from_user(&req_u.req, optval, len))
3152 return packet_set_ring(sk, &req_u, 0,
3153 optname == PACKET_TX_RING);
3155 case PACKET_COPY_THRESH:
3159 if (optlen != sizeof(val))
3161 if (copy_from_user(&val, optval, sizeof(val)))
3164 pkt_sk(sk)->copy_thresh = val;
3167 case PACKET_VERSION:
3171 if (optlen != sizeof(val))
3173 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3175 if (copy_from_user(&val, optval, sizeof(val)))
3181 po->tp_version = val;
3187 case PACKET_RESERVE:
3191 if (optlen != sizeof(val))
3193 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3195 if (copy_from_user(&val, optval, sizeof(val)))
3197 po->tp_reserve = val;
3204 if (optlen != sizeof(val))
3206 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3208 if (copy_from_user(&val, optval, sizeof(val)))
3210 po->tp_loss = !!val;
3213 case PACKET_AUXDATA:
3217 if (optlen < sizeof(val))
3219 if (copy_from_user(&val, optval, sizeof(val)))
3222 po->auxdata = !!val;
3225 case PACKET_ORIGDEV:
3229 if (optlen < sizeof(val))
3231 if (copy_from_user(&val, optval, sizeof(val)))
3234 po->origdev = !!val;
3237 case PACKET_VNET_HDR:
3241 if (sock->type != SOCK_RAW)
3243 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3245 if (optlen < sizeof(val))
3247 if (copy_from_user(&val, optval, sizeof(val)))
3250 po->has_vnet_hdr = !!val;
3253 case PACKET_TIMESTAMP:
3257 if (optlen != sizeof(val))
3259 if (copy_from_user(&val, optval, sizeof(val)))
3262 po->tp_tstamp = val;
3269 if (optlen != sizeof(val))
3271 if (copy_from_user(&val, optval, sizeof(val)))
3274 return fanout_add(sk, val & 0xffff, val >> 16);
3276 case PACKET_TX_HAS_OFF:
3280 if (optlen != sizeof(val))
3282 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3284 if (copy_from_user(&val, optval, sizeof(val)))
3286 po->tp_tx_has_off = !!val;
3289 case PACKET_QDISC_BYPASS:
3293 if (optlen != sizeof(val))
3295 if (copy_from_user(&val, optval, sizeof(val)))
3298 po->xmit = val ? packet_direct_xmit : dev_queue_xmit;
3302 return -ENOPROTOOPT;
3306 static int packet_getsockopt(struct socket *sock, int level, int optname,
3307 char __user *optval, int __user *optlen)
3310 int val, lv = sizeof(val);
3311 struct sock *sk = sock->sk;
3312 struct packet_sock *po = pkt_sk(sk);
3314 union tpacket_stats_u st;
3316 if (level != SOL_PACKET)
3317 return -ENOPROTOOPT;
3319 if (get_user(len, optlen))
3326 case PACKET_STATISTICS:
3327 spin_lock_bh(&sk->sk_receive_queue.lock);
3328 memcpy(&st, &po->stats, sizeof(st));
3329 memset(&po->stats, 0, sizeof(po->stats));
3330 spin_unlock_bh(&sk->sk_receive_queue.lock);
3332 if (po->tp_version == TPACKET_V3) {
3333 lv = sizeof(struct tpacket_stats_v3);
3334 st.stats3.tp_packets += st.stats3.tp_drops;
3337 lv = sizeof(struct tpacket_stats);
3338 st.stats1.tp_packets += st.stats1.tp_drops;
3343 case PACKET_AUXDATA:
3346 case PACKET_ORIGDEV:
3349 case PACKET_VNET_HDR:
3350 val = po->has_vnet_hdr;
3352 case PACKET_VERSION:
3353 val = po->tp_version;
3356 if (len > sizeof(int))
3358 if (copy_from_user(&val, optval, len))
3362 val = sizeof(struct tpacket_hdr);
3365 val = sizeof(struct tpacket2_hdr);
3368 val = sizeof(struct tpacket3_hdr);
3374 case PACKET_RESERVE:
3375 val = po->tp_reserve;
3380 case PACKET_TIMESTAMP:
3381 val = po->tp_tstamp;
3385 ((u32)po->fanout->id |
3386 ((u32)po->fanout->type << 16) |
3387 ((u32)po->fanout->flags << 24)) :
3390 case PACKET_TX_HAS_OFF:
3391 val = po->tp_tx_has_off;
3393 case PACKET_QDISC_BYPASS:
3394 val = packet_use_direct_xmit(po);
3397 return -ENOPROTOOPT;
3402 if (put_user(len, optlen))
3404 if (copy_to_user(optval, data, len))
3410 static int packet_notifier(struct notifier_block *this,
3411 unsigned long msg, void *ptr)
3414 struct net_device *dev = netdev_notifier_info_to_dev(ptr);
3415 struct net *net = dev_net(dev);
3418 sk_for_each_rcu(sk, &net->packet.sklist) {
3419 struct packet_sock *po = pkt_sk(sk);
3422 case NETDEV_UNREGISTER:
3424 packet_dev_mclist(dev, po->mclist, -1);
3428 if (dev->ifindex == po->ifindex) {
3429 spin_lock(&po->bind_lock);
3431 __unregister_prot_hook(sk, false);
3432 sk->sk_err = ENETDOWN;
3433 if (!sock_flag(sk, SOCK_DEAD))
3434 sk->sk_error_report(sk);
3436 if (msg == NETDEV_UNREGISTER) {
3437 packet_cached_dev_reset(po);
3439 if (po->prot_hook.dev)
3440 dev_put(po->prot_hook.dev);
3441 po->prot_hook.dev = NULL;
3443 spin_unlock(&po->bind_lock);
3447 if (dev->ifindex == po->ifindex) {
3448 spin_lock(&po->bind_lock);
3450 register_prot_hook(sk);
3451 spin_unlock(&po->bind_lock);
3461 static int packet_ioctl(struct socket *sock, unsigned int cmd,
3464 struct sock *sk = sock->sk;
3469 int amount = sk_wmem_alloc_get(sk);
3471 return put_user(amount, (int __user *)arg);
3475 struct sk_buff *skb;
3478 spin_lock_bh(&sk->sk_receive_queue.lock);
3479 skb = skb_peek(&sk->sk_receive_queue);
3482 spin_unlock_bh(&sk->sk_receive_queue.lock);
3483 return put_user(amount, (int __user *)arg);
3486 return sock_get_timestamp(sk, (struct timeval __user *)arg);
3488 return sock_get_timestampns(sk, (struct timespec __user *)arg);
3498 case SIOCGIFBRDADDR:
3499 case SIOCSIFBRDADDR:
3500 case SIOCGIFNETMASK:
3501 case SIOCSIFNETMASK:
3502 case SIOCGIFDSTADDR:
3503 case SIOCSIFDSTADDR:
3505 return inet_dgram_ops.ioctl(sock, cmd, arg);
3509 return -ENOIOCTLCMD;
3514 static unsigned int packet_poll(struct file *file, struct socket *sock,
3517 struct sock *sk = sock->sk;
3518 struct packet_sock *po = pkt_sk(sk);
3519 unsigned int mask = datagram_poll(file, sock, wait);
3521 spin_lock_bh(&sk->sk_receive_queue.lock);
3522 if (po->rx_ring.pg_vec) {
3523 if (!packet_previous_rx_frame(po, &po->rx_ring,
3525 mask |= POLLIN | POLLRDNORM;
3527 spin_unlock_bh(&sk->sk_receive_queue.lock);
3528 spin_lock_bh(&sk->sk_write_queue.lock);
3529 if (po->tx_ring.pg_vec) {
3530 if (packet_current_frame(po, &po->tx_ring, TP_STATUS_AVAILABLE))
3531 mask |= POLLOUT | POLLWRNORM;
3533 spin_unlock_bh(&sk->sk_write_queue.lock);
3538 /* Dirty? Well, I still did not learn better way to account
3542 static void packet_mm_open(struct vm_area_struct *vma)
3544 struct file *file = vma->vm_file;
3545 struct socket *sock = file->private_data;
3546 struct sock *sk = sock->sk;
3549 atomic_inc(&pkt_sk(sk)->mapped);
3552 static void packet_mm_close(struct vm_area_struct *vma)
3554 struct file *file = vma->vm_file;
3555 struct socket *sock = file->private_data;
3556 struct sock *sk = sock->sk;
3559 atomic_dec(&pkt_sk(sk)->mapped);
3562 static const struct vm_operations_struct packet_mmap_ops = {
3563 .open = packet_mm_open,
3564 .close = packet_mm_close,
3567 static void free_pg_vec(struct pgv *pg_vec, unsigned int order,
3572 for (i = 0; i < len; i++) {
3573 if (likely(pg_vec[i].buffer)) {
3574 if (is_vmalloc_addr(pg_vec[i].buffer))
3575 vfree(pg_vec[i].buffer);
3577 free_pages((unsigned long)pg_vec[i].buffer,
3579 pg_vec[i].buffer = NULL;
3585 static char *alloc_one_pg_vec_page(unsigned long order)
3587 char *buffer = NULL;
3588 gfp_t gfp_flags = GFP_KERNEL | __GFP_COMP |
3589 __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY;
3591 buffer = (char *) __get_free_pages(gfp_flags, order);
3597 * __get_free_pages failed, fall back to vmalloc
3599 buffer = vzalloc((1 << order) * PAGE_SIZE);
3605 * vmalloc failed, lets dig into swap here
3607 gfp_flags &= ~__GFP_NORETRY;
3608 buffer = (char *)__get_free_pages(gfp_flags, order);
3613 * complete and utter failure
3618 static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order)
3620 unsigned int block_nr = req->tp_block_nr;
3624 pg_vec = kcalloc(block_nr, sizeof(struct pgv), GFP_KERNEL);
3625 if (unlikely(!pg_vec))
3628 for (i = 0; i < block_nr; i++) {
3629 pg_vec[i].buffer = alloc_one_pg_vec_page(order);
3630 if (unlikely(!pg_vec[i].buffer))
3631 goto out_free_pgvec;
3638 free_pg_vec(pg_vec, order, block_nr);
3643 static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
3644 int closing, int tx_ring)
3646 struct pgv *pg_vec = NULL;
3647 struct packet_sock *po = pkt_sk(sk);
3648 int was_running, order = 0;
3649 struct packet_ring_buffer *rb;
3650 struct sk_buff_head *rb_queue;
3653 /* Added to avoid minimal code churn */
3654 struct tpacket_req *req = &req_u->req;
3656 /* Opening a Tx-ring is NOT supported in TPACKET_V3 */
3657 if (!closing && tx_ring && (po->tp_version > TPACKET_V2)) {
3658 WARN(1, "Tx-ring is not supported.\n");
3662 rb = tx_ring ? &po->tx_ring : &po->rx_ring;
3663 rb_queue = tx_ring ? &sk->sk_write_queue : &sk->sk_receive_queue;
3667 if (atomic_read(&po->mapped))
3669 if (atomic_read(&rb->pending))
3673 if (req->tp_block_nr) {
3674 /* Sanity tests and some calculations */
3676 if (unlikely(rb->pg_vec))
3679 switch (po->tp_version) {
3681 po->tp_hdrlen = TPACKET_HDRLEN;
3684 po->tp_hdrlen = TPACKET2_HDRLEN;
3687 po->tp_hdrlen = TPACKET3_HDRLEN;
3692 if (unlikely((int)req->tp_block_size <= 0))
3694 if (unlikely(req->tp_block_size & (PAGE_SIZE - 1)))
3696 if (unlikely(req->tp_frame_size < po->tp_hdrlen +
3699 if (unlikely(req->tp_frame_size & (TPACKET_ALIGNMENT - 1)))
3702 rb->frames_per_block = req->tp_block_size/req->tp_frame_size;
3703 if (unlikely(rb->frames_per_block <= 0))
3705 if (unlikely((rb->frames_per_block * req->tp_block_nr) !=
3710 order = get_order(req->tp_block_size);
3711 pg_vec = alloc_pg_vec(req, order);
3712 if (unlikely(!pg_vec))
3714 switch (po->tp_version) {
3716 /* Transmit path is not supported. We checked
3717 * it above but just being paranoid
3720 init_prb_bdqc(po, rb, pg_vec, req_u, tx_ring);
3729 if (unlikely(req->tp_frame_nr))
3735 /* Detach socket from network */
3736 spin_lock(&po->bind_lock);
3737 was_running = po->running;
3741 __unregister_prot_hook(sk, false);
3743 spin_unlock(&po->bind_lock);
3748 mutex_lock(&po->pg_vec_lock);
3749 if (closing || atomic_read(&po->mapped) == 0) {
3751 spin_lock_bh(&rb_queue->lock);
3752 swap(rb->pg_vec, pg_vec);
3753 rb->frame_max = (req->tp_frame_nr - 1);
3755 rb->frame_size = req->tp_frame_size;
3756 spin_unlock_bh(&rb_queue->lock);
3758 swap(rb->pg_vec_order, order);
3759 swap(rb->pg_vec_len, req->tp_block_nr);
3761 rb->pg_vec_pages = req->tp_block_size/PAGE_SIZE;
3762 po->prot_hook.func = (po->rx_ring.pg_vec) ?
3763 tpacket_rcv : packet_rcv;
3764 skb_queue_purge(rb_queue);
3765 if (atomic_read(&po->mapped))
3766 pr_err("packet_mmap: vma is busy: %d\n",
3767 atomic_read(&po->mapped));
3769 mutex_unlock(&po->pg_vec_lock);
3771 spin_lock(&po->bind_lock);
3774 register_prot_hook(sk);
3776 spin_unlock(&po->bind_lock);
3777 if (closing && (po->tp_version > TPACKET_V2)) {
3778 /* Because we don't support block-based V3 on tx-ring */
3780 prb_shutdown_retire_blk_timer(po, tx_ring, rb_queue);
3785 free_pg_vec(pg_vec, order, req->tp_block_nr);
3790 static int packet_mmap(struct file *file, struct socket *sock,
3791 struct vm_area_struct *vma)
3793 struct sock *sk = sock->sk;
3794 struct packet_sock *po = pkt_sk(sk);
3795 unsigned long size, expected_size;
3796 struct packet_ring_buffer *rb;
3797 unsigned long start;
3804 mutex_lock(&po->pg_vec_lock);
3807 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3809 expected_size += rb->pg_vec_len
3815 if (expected_size == 0)
3818 size = vma->vm_end - vma->vm_start;
3819 if (size != expected_size)
3822 start = vma->vm_start;
3823 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3824 if (rb->pg_vec == NULL)
3827 for (i = 0; i < rb->pg_vec_len; i++) {
3829 void *kaddr = rb->pg_vec[i].buffer;
3832 for (pg_num = 0; pg_num < rb->pg_vec_pages; pg_num++) {
3833 page = pgv_to_page(kaddr);
3834 err = vm_insert_page(vma, start, page);
3843 atomic_inc(&po->mapped);
3844 vma->vm_ops = &packet_mmap_ops;
3848 mutex_unlock(&po->pg_vec_lock);
3852 static const struct proto_ops packet_ops_spkt = {
3853 .family = PF_PACKET,
3854 .owner = THIS_MODULE,
3855 .release = packet_release,
3856 .bind = packet_bind_spkt,
3857 .connect = sock_no_connect,
3858 .socketpair = sock_no_socketpair,
3859 .accept = sock_no_accept,
3860 .getname = packet_getname_spkt,
3861 .poll = datagram_poll,
3862 .ioctl = packet_ioctl,
3863 .listen = sock_no_listen,
3864 .shutdown = sock_no_shutdown,
3865 .setsockopt = sock_no_setsockopt,
3866 .getsockopt = sock_no_getsockopt,
3867 .sendmsg = packet_sendmsg_spkt,
3868 .recvmsg = packet_recvmsg,
3869 .mmap = sock_no_mmap,
3870 .sendpage = sock_no_sendpage,
3873 static const struct proto_ops packet_ops = {
3874 .family = PF_PACKET,
3875 .owner = THIS_MODULE,
3876 .release = packet_release,
3877 .bind = packet_bind,
3878 .connect = sock_no_connect,
3879 .socketpair = sock_no_socketpair,
3880 .accept = sock_no_accept,
3881 .getname = packet_getname,
3882 .poll = packet_poll,
3883 .ioctl = packet_ioctl,
3884 .listen = sock_no_listen,
3885 .shutdown = sock_no_shutdown,
3886 .setsockopt = packet_setsockopt,
3887 .getsockopt = packet_getsockopt,
3888 .sendmsg = packet_sendmsg,
3889 .recvmsg = packet_recvmsg,
3890 .mmap = packet_mmap,
3891 .sendpage = sock_no_sendpage,
3894 static const struct net_proto_family packet_family_ops = {
3895 .family = PF_PACKET,
3896 .create = packet_create,
3897 .owner = THIS_MODULE,
3900 static struct notifier_block packet_netdev_notifier = {
3901 .notifier_call = packet_notifier,
3904 #ifdef CONFIG_PROC_FS
3906 static void *packet_seq_start(struct seq_file *seq, loff_t *pos)
3909 struct net *net = seq_file_net(seq);
3912 return seq_hlist_start_head_rcu(&net->packet.sklist, *pos);
3915 static void *packet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
3917 struct net *net = seq_file_net(seq);
3918 return seq_hlist_next_rcu(v, &net->packet.sklist, pos);
3921 static void packet_seq_stop(struct seq_file *seq, void *v)
3927 static int packet_seq_show(struct seq_file *seq, void *v)
3929 if (v == SEQ_START_TOKEN)
3930 seq_puts(seq, "sk RefCnt Type Proto Iface R Rmem User Inode\n");
3932 struct sock *s = sk_entry(v);
3933 const struct packet_sock *po = pkt_sk(s);
3936 "%pK %-6d %-4d %04x %-5d %1d %-6u %-6u %-6lu\n",
3938 atomic_read(&s->sk_refcnt),
3943 atomic_read(&s->sk_rmem_alloc),
3944 from_kuid_munged(seq_user_ns(seq), sock_i_uid(s)),
3951 static const struct seq_operations packet_seq_ops = {
3952 .start = packet_seq_start,
3953 .next = packet_seq_next,
3954 .stop = packet_seq_stop,
3955 .show = packet_seq_show,
3958 static int packet_seq_open(struct inode *inode, struct file *file)
3960 return seq_open_net(inode, file, &packet_seq_ops,
3961 sizeof(struct seq_net_private));
3964 static const struct file_operations packet_seq_fops = {
3965 .owner = THIS_MODULE,
3966 .open = packet_seq_open,
3968 .llseek = seq_lseek,
3969 .release = seq_release_net,
3974 static int __net_init packet_net_init(struct net *net)
3976 mutex_init(&net->packet.sklist_lock);
3977 INIT_HLIST_HEAD(&net->packet.sklist);
3979 if (!proc_create("packet", 0, net->proc_net, &packet_seq_fops))
3985 static void __net_exit packet_net_exit(struct net *net)
3987 remove_proc_entry("packet", net->proc_net);
3990 static struct pernet_operations packet_net_ops = {
3991 .init = packet_net_init,
3992 .exit = packet_net_exit,
3996 static void __exit packet_exit(void)
3998 unregister_netdevice_notifier(&packet_netdev_notifier);
3999 unregister_pernet_subsys(&packet_net_ops);
4000 sock_unregister(PF_PACKET);
4001 proto_unregister(&packet_proto);
4004 static int __init packet_init(void)
4006 int rc = proto_register(&packet_proto, 0);
4011 sock_register(&packet_family_ops);
4012 register_pernet_subsys(&packet_net_ops);
4013 register_netdevice_notifier(&packet_netdev_notifier);
4018 module_init(packet_init);
4019 module_exit(packet_exit);
4020 MODULE_LICENSE("GPL");
4021 MODULE_ALIAS_NETPROTO(PF_PACKET);