Merge tag 'omap-for-v4.7-dts-fixes1' of git://git.kernel.org/pub/scm/linux/kernel...
[cascardo/linux.git] / drivers / staging / octeon / ethernet-rx.c
1 /*
2  * This file is based on code from OCTEON SDK by Cavium Networks.
3  *
4  * Copyright (c) 2003-2010 Cavium Networks
5  *
6  * This file is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License, Version 2, as
8  * published by the Free Software Foundation.
9  */
10
11 #include <linux/module.h>
12 #include <linux/kernel.h>
13 #include <linux/cache.h>
14 #include <linux/cpumask.h>
15 #include <linux/netdevice.h>
16 #include <linux/etherdevice.h>
17 #include <linux/ip.h>
18 #include <linux/string.h>
19 #include <linux/prefetch.h>
20 #include <linux/ratelimit.h>
21 #include <linux/smp.h>
22 #include <linux/interrupt.h>
23 #include <net/dst.h>
24 #ifdef CONFIG_XFRM
25 #include <linux/xfrm.h>
26 #include <net/xfrm.h>
27 #endif /* CONFIG_XFRM */
28
29 #include <asm/octeon/octeon.h>
30
31 #include "ethernet-defines.h"
32 #include "ethernet-mem.h"
33 #include "ethernet-rx.h"
34 #include "octeon-ethernet.h"
35 #include "ethernet-util.h"
36
37 #include <asm/octeon/cvmx-helper.h>
38 #include <asm/octeon/cvmx-wqe.h>
39 #include <asm/octeon/cvmx-fau.h>
40 #include <asm/octeon/cvmx-pow.h>
41 #include <asm/octeon/cvmx-pip.h>
42 #include <asm/octeon/cvmx-scratch.h>
43
44 #include <asm/octeon/cvmx-gmxx-defs.h>
45
46 static struct napi_struct cvm_oct_napi;
47
48 /**
49  * cvm_oct_do_interrupt - interrupt handler.
50  * @cpl: Interrupt number. Unused
51  * @dev_id: Cookie to identify the device. Unused
52  *
53  * The interrupt occurs whenever the POW has packets in our group.
54  *
55  */
56 static irqreturn_t cvm_oct_do_interrupt(int cpl, void *dev_id)
57 {
58         /* Disable the IRQ and start napi_poll. */
59         disable_irq_nosync(OCTEON_IRQ_WORKQ0 + pow_receive_group);
60         napi_schedule(&cvm_oct_napi);
61
62         return IRQ_HANDLED;
63 }
64
65 /**
66  * cvm_oct_check_rcv_error - process receive errors
67  * @work: Work queue entry pointing to the packet.
68  *
69  * Returns Non-zero if the packet can be dropped, zero otherwise.
70  */
71 static inline int cvm_oct_check_rcv_error(cvmx_wqe_t *work)
72 {
73         int port;
74
75         if (octeon_has_feature(OCTEON_FEATURE_PKND))
76                 port = work->word0.pip.cn68xx.pknd;
77         else
78                 port = work->word1.cn38xx.ipprt;
79
80         if ((work->word2.snoip.err_code == 10) && (work->word1.len <= 64)) {
81                 /*
82                  * Ignore length errors on min size packets. Some
83                  * equipment incorrectly pads packets to 64+4FCS
84                  * instead of 60+4FCS.  Note these packets still get
85                  * counted as frame errors.
86                  */
87         } else if (work->word2.snoip.err_code == 5 ||
88                    work->word2.snoip.err_code == 7) {
89                 /*
90                  * We received a packet with either an alignment error
91                  * or a FCS error. This may be signalling that we are
92                  * running 10Mbps with GMXX_RXX_FRM_CTL[PRE_CHK]
93                  * off. If this is the case we need to parse the
94                  * packet to determine if we can remove a non spec
95                  * preamble and generate a correct packet.
96                  */
97                 int interface = cvmx_helper_get_interface_num(port);
98                 int index = cvmx_helper_get_interface_index_num(port);
99                 union cvmx_gmxx_rxx_frm_ctl gmxx_rxx_frm_ctl;
100
101                 gmxx_rxx_frm_ctl.u64 =
102                     cvmx_read_csr(CVMX_GMXX_RXX_FRM_CTL(index, interface));
103                 if (gmxx_rxx_frm_ctl.s.pre_chk == 0) {
104                         u8 *ptr =
105                             cvmx_phys_to_ptr(work->packet_ptr.s.addr);
106                         int i = 0;
107
108                         while (i < work->word1.len - 1) {
109                                 if (*ptr != 0x55)
110                                         break;
111                                 ptr++;
112                                 i++;
113                         }
114
115                         if (*ptr == 0xd5) {
116                                 /* Port received 0xd5 preamble */
117                                 work->packet_ptr.s.addr += i + 1;
118                                 work->word1.len -= i + 5;
119                         } else if ((*ptr & 0xf) == 0xd) {
120                                 /* Port received 0xd preamble */
121                                 work->packet_ptr.s.addr += i;
122                                 work->word1.len -= i + 4;
123                                 for (i = 0; i < work->word1.len; i++) {
124                                         *ptr =
125                                             ((*ptr & 0xf0) >> 4) |
126                                             ((*(ptr + 1) & 0xf) << 4);
127                                         ptr++;
128                                 }
129                         } else {
130                                 printk_ratelimited("Port %d unknown preamble, packet dropped\n",
131                                                    port);
132                                 cvm_oct_free_work(work);
133                                 return 1;
134                         }
135                 }
136         } else {
137                 printk_ratelimited("Port %d receive error code %d, packet dropped\n",
138                                    port, work->word2.snoip.err_code);
139                 cvm_oct_free_work(work);
140                 return 1;
141         }
142
143         return 0;
144 }
145
146 /**
147  * cvm_oct_napi_poll - the NAPI poll function.
148  * @napi: The NAPI instance, or null if called from cvm_oct_poll_controller
149  * @budget: Maximum number of packets to receive.
150  *
151  * Returns the number of packets processed.
152  */
153 static int cvm_oct_napi_poll(struct napi_struct *napi, int budget)
154 {
155         const int       coreid = cvmx_get_core_num();
156         u64     old_group_mask;
157         u64     old_scratch;
158         int             rx_count = 0;
159         int             did_work_request = 0;
160         int             packet_not_copied;
161
162         /* Prefetch cvm_oct_device since we know we need it soon */
163         prefetch(cvm_oct_device);
164
165         if (USE_ASYNC_IOBDMA) {
166                 /* Save scratch in case userspace is using it */
167                 CVMX_SYNCIOBDMA;
168                 old_scratch = cvmx_scratch_read64(CVMX_SCR_SCRATCH);
169         }
170
171         /* Only allow work for our group (and preserve priorities) */
172         if (OCTEON_IS_MODEL(OCTEON_CN68XX)) {
173                 old_group_mask = cvmx_read_csr(CVMX_SSO_PPX_GRP_MSK(coreid));
174                 cvmx_write_csr(CVMX_SSO_PPX_GRP_MSK(coreid),
175                                1ull << pow_receive_group);
176                 cvmx_read_csr(CVMX_SSO_PPX_GRP_MSK(coreid)); /* Flush */
177         } else {
178                 old_group_mask = cvmx_read_csr(CVMX_POW_PP_GRP_MSKX(coreid));
179                 cvmx_write_csr(CVMX_POW_PP_GRP_MSKX(coreid),
180                                (old_group_mask & ~0xFFFFull) |
181                                1 << pow_receive_group);
182         }
183
184         if (USE_ASYNC_IOBDMA) {
185                 cvmx_pow_work_request_async(CVMX_SCR_SCRATCH, CVMX_POW_NO_WAIT);
186                 did_work_request = 1;
187         }
188
189         while (rx_count < budget) {
190                 struct sk_buff *skb = NULL;
191                 struct sk_buff **pskb = NULL;
192                 int skb_in_hw;
193                 cvmx_wqe_t *work;
194                 int port;
195
196                 if (USE_ASYNC_IOBDMA && did_work_request)
197                         work = cvmx_pow_work_response_async(CVMX_SCR_SCRATCH);
198                 else
199                         work = cvmx_pow_work_request_sync(CVMX_POW_NO_WAIT);
200
201                 prefetch(work);
202                 did_work_request = 0;
203                 if (!work) {
204                         if (OCTEON_IS_MODEL(OCTEON_CN68XX)) {
205                                 cvmx_write_csr(CVMX_SSO_WQ_IQ_DIS,
206                                                1ull << pow_receive_group);
207                                 cvmx_write_csr(CVMX_SSO_WQ_INT,
208                                                1ull << pow_receive_group);
209                         } else {
210                                 union cvmx_pow_wq_int wq_int;
211
212                                 wq_int.u64 = 0;
213                                 wq_int.s.iq_dis = 1 << pow_receive_group;
214                                 wq_int.s.wq_int = 1 << pow_receive_group;
215                                 cvmx_write_csr(CVMX_POW_WQ_INT, wq_int.u64);
216                         }
217                         break;
218                 }
219                 pskb = (struct sk_buff **)
220                         (cvm_oct_get_buffer_ptr(work->packet_ptr) -
221                         sizeof(void *));
222                 prefetch(pskb);
223
224                 if (USE_ASYNC_IOBDMA && rx_count < (budget - 1)) {
225                         cvmx_pow_work_request_async_nocheck(CVMX_SCR_SCRATCH,
226                                                             CVMX_POW_NO_WAIT);
227                         did_work_request = 1;
228                 }
229                 rx_count++;
230
231                 skb_in_hw = work->word2.s.bufs == 1;
232                 if (likely(skb_in_hw)) {
233                         skb = *pskb;
234                         prefetch(&skb->head);
235                         prefetch(&skb->len);
236                 }
237
238                 if (octeon_has_feature(OCTEON_FEATURE_PKND))
239                         port = work->word0.pip.cn68xx.pknd;
240                 else
241                         port = work->word1.cn38xx.ipprt;
242
243                 prefetch(cvm_oct_device[port]);
244
245                 /* Immediately throw away all packets with receive errors */
246                 if (unlikely(work->word2.snoip.rcv_error)) {
247                         if (cvm_oct_check_rcv_error(work))
248                                 continue;
249                 }
250
251                 /*
252                  * We can only use the zero copy path if skbuffs are
253                  * in the FPA pool and the packet fits in a single
254                  * buffer.
255                  */
256                 if (likely(skb_in_hw)) {
257                         skb->data = skb->head + work->packet_ptr.s.addr -
258                                 cvmx_ptr_to_phys(skb->head);
259                         prefetch(skb->data);
260                         skb->len = work->word1.len;
261                         skb_set_tail_pointer(skb, skb->len);
262                         packet_not_copied = 1;
263                 } else {
264                         /*
265                          * We have to copy the packet. First allocate
266                          * an skbuff for it.
267                          */
268                         skb = dev_alloc_skb(work->word1.len);
269                         if (!skb) {
270                                 cvm_oct_free_work(work);
271                                 continue;
272                         }
273
274                         /*
275                          * Check if we've received a packet that was
276                          * entirely stored in the work entry.
277                          */
278                         if (unlikely(work->word2.s.bufs == 0)) {
279                                 u8 *ptr = work->packet_data;
280
281                                 if (likely(!work->word2.s.not_IP)) {
282                                         /*
283                                          * The beginning of the packet
284                                          * moves for IP packets.
285                                          */
286                                         if (work->word2.s.is_v6)
287                                                 ptr += 2;
288                                         else
289                                                 ptr += 6;
290                                 }
291                                 memcpy(skb_put(skb, work->word1.len), ptr,
292                                        work->word1.len);
293                                 /* No packet buffers to free */
294                         } else {
295                                 int segments = work->word2.s.bufs;
296                                 union cvmx_buf_ptr segment_ptr =
297                                     work->packet_ptr;
298                                 int len = work->word1.len;
299
300                                 while (segments--) {
301                                         union cvmx_buf_ptr next_ptr =
302                                             *(union cvmx_buf_ptr *)
303                                               cvmx_phys_to_ptr(
304                                               segment_ptr.s.addr - 8);
305
306                         /*
307                          * Octeon Errata PKI-100: The segment size is
308                          * wrong. Until it is fixed, calculate the
309                          * segment size based on the packet pool
310                          * buffer size. When it is fixed, the
311                          * following line should be replaced with this
312                          * one: int segment_size =
313                          * segment_ptr.s.size;
314                          */
315                                         int segment_size =
316                                             CVMX_FPA_PACKET_POOL_SIZE -
317                                             (segment_ptr.s.addr -
318                                              (((segment_ptr.s.addr >> 7) -
319                                                segment_ptr.s.back) << 7));
320                                         /*
321                                          * Don't copy more than what
322                                          * is left in the packet.
323                                          */
324                                         if (segment_size > len)
325                                                 segment_size = len;
326                                         /* Copy the data into the packet */
327                                         memcpy(skb_put(skb, segment_size),
328                                                cvmx_phys_to_ptr(
329                                                segment_ptr.s.addr),
330                                                segment_size);
331                                         len -= segment_size;
332                                         segment_ptr = next_ptr;
333                                 }
334                         }
335                         packet_not_copied = 0;
336                 }
337                 if (likely((port < TOTAL_NUMBER_OF_PORTS) &&
338                            cvm_oct_device[port])) {
339                         struct net_device *dev = cvm_oct_device[port];
340                         struct octeon_ethernet *priv = netdev_priv(dev);
341
342                         /*
343                          * Only accept packets for devices that are
344                          * currently up.
345                          */
346                         if (likely(dev->flags & IFF_UP)) {
347                                 skb->protocol = eth_type_trans(skb, dev);
348                                 skb->dev = dev;
349
350                                 if (unlikely(work->word2.s.not_IP ||
351                                              work->word2.s.IP_exc ||
352                                              work->word2.s.L4_error ||
353                                              !work->word2.s.tcp_or_udp))
354                                         skb->ip_summed = CHECKSUM_NONE;
355                                 else
356                                         skb->ip_summed = CHECKSUM_UNNECESSARY;
357
358                                 /* Increment RX stats for virtual ports */
359                                 if (port >= CVMX_PIP_NUM_INPUT_PORTS) {
360                                         priv->stats.rx_packets++;
361                                         priv->stats.rx_bytes += skb->len;
362                                 }
363                                 netif_receive_skb(skb);
364                         } else {
365                                 /*
366                                  * Drop any packet received for a device that
367                                  * isn't up.
368                                  */
369                                 priv->stats.rx_dropped++;
370                                 dev_kfree_skb_irq(skb);
371                         }
372                 } else {
373                         /*
374                          * Drop any packet received for a device that
375                          * doesn't exist.
376                          */
377                         printk_ratelimited("Port %d not controlled by Linux, packet dropped\n",
378                                            port);
379                         dev_kfree_skb_irq(skb);
380                 }
381                 /*
382                  * Check to see if the skbuff and work share the same
383                  * packet buffer.
384                  */
385                 if (likely(packet_not_copied)) {
386                         /*
387                          * This buffer needs to be replaced, increment
388                          * the number of buffers we need to free by
389                          * one.
390                          */
391                         cvmx_fau_atomic_add32(FAU_NUM_PACKET_BUFFERS_TO_FREE,
392                                               1);
393
394                         cvmx_fpa_free(work, CVMX_FPA_WQE_POOL, 1);
395                 } else {
396                         cvm_oct_free_work(work);
397                 }
398         }
399         /* Restore the original POW group mask */
400         if (OCTEON_IS_MODEL(OCTEON_CN68XX)) {
401                 cvmx_write_csr(CVMX_SSO_PPX_GRP_MSK(coreid), old_group_mask);
402                 cvmx_read_csr(CVMX_SSO_PPX_GRP_MSK(coreid)); /* Flush */
403         } else {
404                 cvmx_write_csr(CVMX_POW_PP_GRP_MSKX(coreid), old_group_mask);
405         }
406
407         if (USE_ASYNC_IOBDMA) {
408                 /* Restore the scratch area */
409                 cvmx_scratch_write64(CVMX_SCR_SCRATCH, old_scratch);
410         }
411         cvm_oct_rx_refill_pool(0);
412
413         if (rx_count < budget && napi) {
414                 /* No more work */
415                 napi_complete(napi);
416                 enable_irq(OCTEON_IRQ_WORKQ0 + pow_receive_group);
417         }
418         return rx_count;
419 }
420
421 #ifdef CONFIG_NET_POLL_CONTROLLER
422 /**
423  * cvm_oct_poll_controller - poll for receive packets
424  * device.
425  *
426  * @dev:    Device to poll. Unused
427  */
428 void cvm_oct_poll_controller(struct net_device *dev)
429 {
430         cvm_oct_napi_poll(NULL, 16);
431 }
432 #endif
433
434 void cvm_oct_rx_initialize(void)
435 {
436         int i;
437         struct net_device *dev_for_napi = NULL;
438
439         for (i = 0; i < TOTAL_NUMBER_OF_PORTS; i++) {
440                 if (cvm_oct_device[i]) {
441                         dev_for_napi = cvm_oct_device[i];
442                         break;
443                 }
444         }
445
446         if (!dev_for_napi)
447                 panic("No net_devices were allocated.");
448
449         netif_napi_add(dev_for_napi, &cvm_oct_napi, cvm_oct_napi_poll,
450                        rx_napi_weight);
451         napi_enable(&cvm_oct_napi);
452
453         /* Register an IRQ handler to receive POW interrupts */
454         i = request_irq(OCTEON_IRQ_WORKQ0 + pow_receive_group,
455                         cvm_oct_do_interrupt, 0, "Ethernet", cvm_oct_device);
456
457         if (i)
458                 panic("Could not acquire Ethernet IRQ %d\n",
459                       OCTEON_IRQ_WORKQ0 + pow_receive_group);
460
461         disable_irq_nosync(OCTEON_IRQ_WORKQ0 + pow_receive_group);
462
463         /* Enable POW interrupt when our port has at least one packet */
464         if (OCTEON_IS_MODEL(OCTEON_CN68XX)) {
465                 union cvmx_sso_wq_int_thrx int_thr;
466                 union cvmx_pow_wq_int_pc int_pc;
467
468                 int_thr.u64 = 0;
469                 int_thr.s.tc_en = 1;
470                 int_thr.s.tc_thr = 1;
471                 cvmx_write_csr(CVMX_SSO_WQ_INT_THRX(pow_receive_group),
472                                int_thr.u64);
473
474                 int_pc.u64 = 0;
475                 int_pc.s.pc_thr = 5;
476                 cvmx_write_csr(CVMX_SSO_WQ_INT_PC, int_pc.u64);
477         } else {
478                 union cvmx_pow_wq_int_thrx int_thr;
479                 union cvmx_pow_wq_int_pc int_pc;
480
481                 int_thr.u64 = 0;
482                 int_thr.s.tc_en = 1;
483                 int_thr.s.tc_thr = 1;
484                 cvmx_write_csr(CVMX_POW_WQ_INT_THRX(pow_receive_group),
485                                int_thr.u64);
486
487                 int_pc.u64 = 0;
488                 int_pc.s.pc_thr = 5;
489                 cvmx_write_csr(CVMX_POW_WQ_INT_PC, int_pc.u64);
490         }
491
492         /* Schedule NAPI now. This will indirectly enable the interrupt. */
493         napi_schedule(&cvm_oct_napi);
494 }
495
496 void cvm_oct_rx_shutdown(void)
497 {
498         netif_napi_del(&cvm_oct_napi);
499 }