drivers/net/ethernet/cavium/liquidio/octeon_droq.c

   1 /**********************************************************************
   2 * Author: Cavium, Inc.
   3 *
   4 * Contact: support@cavium.com
   5 *          Please include "LiquidIO" in the subject.
   6 *
   7 * Copyright (c) 2003-2015 Cavium, Inc.
   8 *
   9 * This file is free software; you can redistribute it and/or modify
  10 * it under the terms of the GNU General Public License, Version 2, as
  11 * published by the Free Software Foundation.
  12 *
  13 * This file is distributed in the hope that it will be useful, but
  14 * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty
  15 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or
  16 * NONINFRINGEMENT.  See the GNU General Public License for more
  17 * details.
  18 *
  19 * This file may also be available under a different license from Cavium.
  20 * Contact Cavium, Inc. for more information
  21 **********************************************************************/
  22 #include <linux/pci.h>
  23 #include <linux/netdevice.h>
  24 #include <linux/vmalloc.h>
  25 #include "liquidio_common.h"
  26 #include "octeon_droq.h"
  27 #include "octeon_iq.h"
  28 #include "response_manager.h"
  29 #include "octeon_device.h"
  30 #include "octeon_main.h"
  31 #include "octeon_network.h"
  32 #include "cn66xx_regs.h"
  33 #include "cn66xx_device.h"
  34 #include "cn23xx_pf_device.h"
  35
  36 #define     CVM_MIN(d1, d2)           (((d1) < (d2)) ? (d1) : (d2))
  37 #define     CVM_MAX(d1, d2)           (((d1) > (d2)) ? (d1) : (d2))
  38
  39 struct niclist {
  40         struct list_head list;
  41         void *ptr;
  42 };
  43
  44 struct __dispatch {
  45         struct list_head list;
  46         struct octeon_recv_info *rinfo;
  47         octeon_dispatch_fn_t disp_fn;
  48 };
  49
  50 /** Get the argument that the user set when registering dispatch
  51  *  function for a given opcode/subcode.
  52  *  @param  octeon_dev - the octeon device pointer.
  53  *  @param  opcode     - the opcode for which the dispatch argument
  54  *                       is to be checked.
  55  *  @param  subcode    - the subcode for which the dispatch argument
  56  *                       is to be checked.
  57  *  @return  Success: void * (argument to the dispatch function)
  58  *  @return  Failure: NULL
  59  *
  60  */
  61 static inline void *octeon_get_dispatch_arg(struct octeon_device *octeon_dev,
  62                                             u16 opcode, u16 subcode)
  63 {
  64         int idx;
  65         struct list_head *dispatch;
  66         void *fn_arg = NULL;
  67         u16 combined_opcode = OPCODE_SUBCODE(opcode, subcode);
  68
  69         idx = combined_opcode & OCTEON_OPCODE_MASK;
  70
  71         spin_lock_bh(&octeon_dev->dispatch.lock);
  72
  73         if (octeon_dev->dispatch.count == 0) {
  74                 spin_unlock_bh(&octeon_dev->dispatch.lock);
  75                 return NULL;
  76         }
  77
  78         if (octeon_dev->dispatch.dlist[idx].opcode == combined_opcode) {
  79                 fn_arg = octeon_dev->dispatch.dlist[idx].arg;
  80         } else {
  81                 list_for_each(dispatch,
  82                               &octeon_dev->dispatch.dlist[idx].list) {
  83                         if (((struct octeon_dispatch *)dispatch)->opcode ==
  84                             combined_opcode) {
  85                                 fn_arg = ((struct octeon_dispatch *)
  86                                           dispatch)->arg;
  87                                 break;
  88                         }
  89                 }
  90         }
  91
  92         spin_unlock_bh(&octeon_dev->dispatch.lock);
  93         return fn_arg;
  94 }
  95
  96 /** Check for packets on Droq. This function should be called with lock held.
  97  *  @param  droq - Droq on which count is checked.
  98  *  @return Returns packet count.
  99  */
 100 u32 octeon_droq_check_hw_for_pkts(struct octeon_droq *droq)
 101 {
 102         u32 pkt_count = 0;
 103         u32 last_count;
 104
 105         pkt_count = readl(droq->pkts_sent_reg);
 106
 107         last_count = pkt_count - droq->pkt_count;
 108         droq->pkt_count = pkt_count;
 109
 110         /* we shall write to cnts  at napi irq enable or end of droq tasklet */
 111         if (last_count)
 112                 atomic_add(last_count, &droq->pkts_pending);
 113
 114         return last_count;
 115 }
 116
 117 static void octeon_droq_compute_max_packet_bufs(struct octeon_droq *droq)
 118 {
 119         u32 count = 0;
 120
 121         /* max_empty_descs is the max. no. of descs that can have no buffers.
 122          * If the empty desc count goes beyond this value, we cannot safely
 123          * read in a 64K packet sent by Octeon
 124          * (64K is max pkt size from Octeon)
 125          */
 126         droq->max_empty_descs = 0;
 127
 128         do {
 129                 droq->max_empty_descs++;
 130                 count += droq->buffer_size;
 131         } while (count < (64 * 1024));
 132
 133         droq->max_empty_descs = droq->max_count - droq->max_empty_descs;
 134 }
 135
 136 static void octeon_droq_reset_indices(struct octeon_droq *droq)
 137 {
 138         droq->read_idx = 0;
 139         droq->write_idx = 0;
 140         droq->refill_idx = 0;
 141         droq->refill_count = 0;
 142         atomic_set(&droq->pkts_pending, 0);
 143 }
 144
 145 static void
 146 octeon_droq_destroy_ring_buffers(struct octeon_device *oct,
 147                                  struct octeon_droq *droq)
 148 {
 149         u32 i;
 150         struct octeon_skb_page_info *pg_info;
 151
 152         for (i = 0; i < droq->max_count; i++) {
 153                 pg_info = &droq->recv_buf_list[i].pg_info;
 154
 155                 if (pg_info->dma)
 156                         lio_unmap_ring(oct->pci_dev,
 157                                        (u64)pg_info->dma);
 158                 pg_info->dma = 0;
 159
 160                 if (pg_info->page)
 161                         recv_buffer_destroy(droq->recv_buf_list[i].buffer,
 162                                             pg_info);
 163
 164                 if (droq->desc_ring && droq->desc_ring[i].info_ptr)
 165                         lio_unmap_ring_info(oct->pci_dev,
 166                                             (u64)droq->
 167                                             desc_ring[i].info_ptr,
 168                                             OCT_DROQ_INFO_SIZE);
 169                 droq->recv_buf_list[i].buffer = NULL;
 170         }
 171
 172         octeon_droq_reset_indices(droq);
 173 }
 174
 175 static int
 176 octeon_droq_setup_ring_buffers(struct octeon_device *oct,
 177                                struct octeon_droq *droq)
 178 {
 179         u32 i;
 180         void *buf;
 181         struct octeon_droq_desc *desc_ring = droq->desc_ring;
 182
 183         for (i = 0; i < droq->max_count; i++) {
 184                 buf = recv_buffer_alloc(oct, &droq->recv_buf_list[i].pg_info);
 185
 186                 if (!buf) {
 187                         dev_err(&oct->pci_dev->dev, "%s buffer alloc failed\n",
 188                                 __func__);
 189                         droq->stats.rx_alloc_failure++;
 190                         return -ENOMEM;
 191                 }
 192
 193                 droq->recv_buf_list[i].buffer = buf;
 194                 droq->recv_buf_list[i].data = get_rbd(buf);
 195                 droq->info_list[i].length = 0;
 196
 197                 /* map ring buffers into memory */
 198                 desc_ring[i].info_ptr = lio_map_ring_info(droq, i);
 199                 desc_ring[i].buffer_ptr =
 200                         lio_map_ring(droq->recv_buf_list[i].buffer);
 201         }
 202
 203         octeon_droq_reset_indices(droq);
 204
 205         octeon_droq_compute_max_packet_bufs(droq);
 206
 207         return 0;
 208 }
 209
 210 int octeon_delete_droq(struct octeon_device *oct, u32 q_no)
 211 {
 212         struct octeon_droq *droq = oct->droq[q_no];
 213
 214         dev_dbg(&oct->pci_dev->dev, "%s[%d]\n", __func__, q_no);
 215
 216         octeon_droq_destroy_ring_buffers(oct, droq);
 217         vfree(droq->recv_buf_list);
 218
 219         if (droq->info_base_addr)
 220                 cnnic_free_aligned_dma(oct->pci_dev, droq->info_list,
 221                                        droq->info_alloc_size,
 222                                        droq->info_base_addr,
 223                                        droq->info_list_dma);
 224
 225         if (droq->desc_ring)
 226                 lio_dma_free(oct, (droq->max_count * OCT_DROQ_DESC_SIZE),
 227                              droq->desc_ring, droq->desc_ring_dma);
 228
 229         memset(droq, 0, OCT_DROQ_SIZE);
 230
 231         return 0;
 232 }
 233
 234 int octeon_init_droq(struct octeon_device *oct,
 235                      u32 q_no,
 236                      u32 num_descs,
 237                      u32 desc_size,
 238                      void *app_ctx)
 239 {
 240         struct octeon_droq *droq;
 241         u32 desc_ring_size = 0, c_num_descs = 0, c_buf_size = 0;
 242         u32 c_pkts_per_intr = 0, c_refill_threshold = 0;
 243         int orig_node = dev_to_node(&oct->pci_dev->dev);
 244         int numa_node = cpu_to_node(q_no % num_online_cpus());
 245
 246         dev_dbg(&oct->pci_dev->dev, "%s[%d]\n", __func__, q_no);
 247
 248         droq = oct->droq[q_no];
 249         memset(droq, 0, OCT_DROQ_SIZE);
 250
 251         droq->oct_dev = oct;
 252         droq->q_no = q_no;
 253         if (app_ctx)
 254                 droq->app_ctx = app_ctx;
 255         else
 256                 droq->app_ctx = (void *)(size_t)q_no;
 257
 258         c_num_descs = num_descs;
 259         c_buf_size = desc_size;
 260         if (OCTEON_CN6XXX(oct)) {
 261                 struct octeon_config *conf6x = CHIP_FIELD(oct, cn6xxx, conf);
 262
 263                 c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf6x);
 264                 c_refill_threshold =
 265                         (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf6x);
 266         } else if (OCTEON_CN23XX_PF(oct)) {
 267                 struct octeon_config *conf23 = CHIP_FIELD(oct, cn23xx_pf, conf);
 268
 269                 c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf23);
 270                 c_refill_threshold = (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf23);
 271         } else {
 272                 return 1;
 273         }
 274
 275         droq->max_count = c_num_descs;
 276         droq->buffer_size = c_buf_size;
 277
 278         desc_ring_size = droq->max_count * OCT_DROQ_DESC_SIZE;
 279         set_dev_node(&oct->pci_dev->dev, numa_node);
 280         droq->desc_ring = lio_dma_alloc(oct, desc_ring_size,
 281                                         (dma_addr_t *)&droq->desc_ring_dma);
 282         set_dev_node(&oct->pci_dev->dev, orig_node);
 283         if (!droq->desc_ring)
 284                 droq->desc_ring = lio_dma_alloc(oct, desc_ring_size,
 285                                         (dma_addr_t *)&droq->desc_ring_dma);
 286
 287         if (!droq->desc_ring) {
 288                 dev_err(&oct->pci_dev->dev,
 289                         "Output queue %d ring alloc failed\n", q_no);
 290                 return 1;
 291         }
 292
 293         dev_dbg(&oct->pci_dev->dev, "droq[%d]: desc_ring: virt: 0x%p, dma: %lx\n",
 294                 q_no, droq->desc_ring, droq->desc_ring_dma);
 295         dev_dbg(&oct->pci_dev->dev, "droq[%d]: num_desc: %d\n", q_no,
 296                 droq->max_count);
 297
 298         droq->info_list =
 299                 cnnic_numa_alloc_aligned_dma((droq->max_count *
 300                                               OCT_DROQ_INFO_SIZE),
 301                                              &droq->info_alloc_size,
 302                                              &droq->info_base_addr,
 303                                              numa_node);
 304         if (!droq->info_list) {
 305                 dev_err(&oct->pci_dev->dev, "Cannot allocate memory for info list.\n");
 306                 lio_dma_free(oct, (droq->max_count * OCT_DROQ_DESC_SIZE),
 307                              droq->desc_ring, droq->desc_ring_dma);
 308                 return 1;
 309         }
 310
 311         droq->recv_buf_list = (struct octeon_recv_buffer *)
 312                               vmalloc_node(droq->max_count *
 313                                                 OCT_DROQ_RECVBUF_SIZE,
 314                                                 numa_node);
 315         if (!droq->recv_buf_list)
 316                 droq->recv_buf_list = (struct octeon_recv_buffer *)
 317                                       vmalloc(droq->max_count *
 318                                                 OCT_DROQ_RECVBUF_SIZE);
 319         if (!droq->recv_buf_list) {
 320                 dev_err(&oct->pci_dev->dev, "Output queue recv buf list alloc failed\n");
 321                 goto init_droq_fail;
 322         }
 323
 324         if (octeon_droq_setup_ring_buffers(oct, droq))
 325                 goto init_droq_fail;
 326
 327         droq->pkts_per_intr = c_pkts_per_intr;
 328         droq->refill_threshold = c_refill_threshold;
 329
 330         dev_dbg(&oct->pci_dev->dev, "DROQ INIT: max_empty_descs: %d\n",
 331                 droq->max_empty_descs);
 332
 333         spin_lock_init(&droq->lock);
 334
 335         INIT_LIST_HEAD(&droq->dispatch_list);
 336
 337         /* For 56xx Pass1, this function won't be called, so no checks. */
 338         oct->fn_list.setup_oq_regs(oct, q_no);
 339
 340         oct->io_qmask.oq |= (1ULL << q_no);
 341
 342         return 0;
 343
 344 init_droq_fail:
 345         octeon_delete_droq(oct, q_no);
 346         return 1;
 347 }
 348
 349 /* octeon_create_recv_info
 350  * Parameters:
 351  *  octeon_dev - pointer to the octeon device structure
 352  *  droq       - droq in which the packet arrived.
 353  *  buf_cnt    - no. of buffers used by the packet.
 354  *  idx        - index in the descriptor for the first buffer in the packet.
 355  * Description:
 356  *  Allocates a recv_info_t and copies the buffer addresses for packet data
 357  *  into the recv_pkt space which starts at an 8B offset from recv_info_t.
 358  *  Flags the descriptors for refill later. If available descriptors go
 359  *  below the threshold to receive a 64K pkt, new buffers are first allocated
 360  *  before the recv_pkt_t is created.
 361  *  This routine will be called in interrupt context.
 362  * Returns:
 363  *  Success: Pointer to recv_info_t
 364  *  Failure: NULL.
 365  * Locks:
 366  *  The droq->lock is held when this routine is called.
 367  */
 368 static inline struct octeon_recv_info *octeon_create_recv_info(
 369                 struct octeon_device *octeon_dev,
 370                 struct octeon_droq *droq,
 371                 u32 buf_cnt,
 372                 u32 idx)
 373 {
 374         struct octeon_droq_info *info;
 375         struct octeon_recv_pkt *recv_pkt;
 376         struct octeon_recv_info *recv_info;
 377         u32 i, bytes_left;
 378         struct octeon_skb_page_info *pg_info;
 379
 380         info = &droq->info_list[idx];
 381
 382         recv_info = octeon_alloc_recv_info(sizeof(struct __dispatch));
 383         if (!recv_info)
 384                 return NULL;
 385
 386         recv_pkt = recv_info->recv_pkt;
 387         recv_pkt->rh = info->rh;
 388         recv_pkt->length = (u32)info->length;
 389         recv_pkt->buffer_count = (u16)buf_cnt;
 390         recv_pkt->octeon_id = (u16)octeon_dev->octeon_id;
 391
 392         i = 0;
 393         bytes_left = (u32)info->length;
 394
 395         while (buf_cnt) {
 396                 {
 397                         pg_info = &droq->recv_buf_list[idx].pg_info;
 398
 399                         lio_unmap_ring(octeon_dev->pci_dev,
 400                                        (u64)pg_info->dma);
 401                         pg_info->page = NULL;
 402                         pg_info->dma = 0;
 403                 }
 404
 405                 recv_pkt->buffer_size[i] =
 406                         (bytes_left >=
 407                          droq->buffer_size) ? droq->buffer_size : bytes_left;
 408
 409                 recv_pkt->buffer_ptr[i] = droq->recv_buf_list[idx].buffer;
 410                 droq->recv_buf_list[idx].buffer = NULL;
 411
 412                 INCR_INDEX_BY1(idx, droq->max_count);
 413                 bytes_left -= droq->buffer_size;
 414                 i++;
 415                 buf_cnt--;
 416         }
 417
 418         return recv_info;
 419 }
 420
 421 /* If we were not able to refill all buffers, try to move around
 422  * the buffers that were not dispatched.
 423  */
 424 static inline u32
 425 octeon_droq_refill_pullup_descs(struct octeon_droq *droq,
 426                                 struct octeon_droq_desc *desc_ring)
 427 {
 428         u32 desc_refilled = 0;
 429
 430         u32 refill_index = droq->refill_idx;
 431
 432         while (refill_index != droq->read_idx) {
 433                 if (droq->recv_buf_list[refill_index].buffer) {
 434                         droq->recv_buf_list[droq->refill_idx].buffer =
 435                                 droq->recv_buf_list[refill_index].buffer;
 436                         droq->recv_buf_list[droq->refill_idx].data =
 437                                 droq->recv_buf_list[refill_index].data;
 438                         desc_ring[droq->refill_idx].buffer_ptr =
 439                                 desc_ring[refill_index].buffer_ptr;
 440                         droq->recv_buf_list[refill_index].buffer = NULL;
 441                         desc_ring[refill_index].buffer_ptr = 0;
 442                         do {
 443                                 INCR_INDEX_BY1(droq->refill_idx,
 444                                                droq->max_count);
 445                                 desc_refilled++;
 446                                 droq->refill_count--;
 447                         } while (droq->recv_buf_list[droq->refill_idx].
 448                                  buffer);
 449                 }
 450                 INCR_INDEX_BY1(refill_index, droq->max_count);
 451         }                       /* while */
 452         return desc_refilled;
 453 }
 454
 455 /* octeon_droq_refill
 456  * Parameters:
 457  *  droq       - droq in which descriptors require new buffers.
 458  * Description:
 459  *  Called during normal DROQ processing in interrupt mode or by the poll
 460  *  thread to refill the descriptors from which buffers were dispatched
 461  *  to upper layers. Attempts to allocate new buffers. If that fails, moves
 462  *  up buffers (that were not dispatched) to form a contiguous ring.
 463  * Returns:
 464  *  No of descriptors refilled.
 465  * Locks:
 466  *  This routine is called with droq->lock held.
 467  */
 468 static u32
 469 octeon_droq_refill(struct octeon_device *octeon_dev, struct octeon_droq *droq)
 470 {
 471         struct octeon_droq_desc *desc_ring;
 472         void *buf = NULL;
 473         u8 *data;
 474         u32 desc_refilled = 0;
 475         struct octeon_skb_page_info *pg_info;
 476
 477         desc_ring = droq->desc_ring;
 478
 479         while (droq->refill_count && (desc_refilled < droq->max_count)) {
 480                 /* If a valid buffer exists (happens if there is no dispatch),
 481                  * reuse
 482                  * the buffer, else allocate.
 483                  */
 484                 if (!droq->recv_buf_list[droq->refill_idx].buffer) {
 485                         pg_info =
 486                                 &droq->recv_buf_list[droq->refill_idx].pg_info;
 487                         /* Either recycle the existing pages or go for
 488                          * new page alloc
 489                          */
 490                         if (pg_info->page)
 491                                 buf = recv_buffer_reuse(octeon_dev, pg_info);
 492                         else
 493                                 buf = recv_buffer_alloc(octeon_dev, pg_info);
 494                         /* If a buffer could not be allocated, no point in
 495                          * continuing
 496                          */
 497                         if (!buf) {
 498                                 droq->stats.rx_alloc_failure++;
 499                                 break;
 500                         }
 501                         droq->recv_buf_list[droq->refill_idx].buffer =
 502                                 buf;
 503                         data = get_rbd(buf);
 504                 } else {
 505                         data = get_rbd(droq->recv_buf_list
 506                                        [droq->refill_idx].buffer);
 507                 }
 508
 509                 droq->recv_buf_list[droq->refill_idx].data = data;
 510
 511                 desc_ring[droq->refill_idx].buffer_ptr =
 512                         lio_map_ring(droq->recv_buf_list[droq->
 513                                      refill_idx].buffer);
 514                 /* Reset any previous values in the length field. */
 515                 droq->info_list[droq->refill_idx].length = 0;
 516
 517                 INCR_INDEX_BY1(droq->refill_idx, droq->max_count);
 518                 desc_refilled++;
 519                 droq->refill_count--;
 520         }
 521
 522         if (droq->refill_count)
 523                 desc_refilled +=
 524                         octeon_droq_refill_pullup_descs(droq, desc_ring);
 525
 526         /* if droq->refill_count
 527          * The refill count would not change in pass two. We only moved buffers
 528          * to close the gap in the ring, but we would still have the same no. of
 529          * buffers to refill.
 530          */
 531         return desc_refilled;
 532 }
 533
 534 static inline u32
 535 octeon_droq_get_bufcount(u32 buf_size, u32 total_len)
 536 {
 537         u32 buf_cnt = 0;
 538
 539         while (total_len > (buf_size * buf_cnt))
 540                 buf_cnt++;
 541         return buf_cnt;
 542 }
 543
 544 static int
 545 octeon_droq_dispatch_pkt(struct octeon_device *oct,
 546                          struct octeon_droq *droq,
 547                          union octeon_rh *rh,
 548                          struct octeon_droq_info *info)
 549 {
 550         u32 cnt;
 551         octeon_dispatch_fn_t disp_fn;
 552         struct octeon_recv_info *rinfo;
 553
 554         cnt = octeon_droq_get_bufcount(droq->buffer_size, (u32)info->length);
 555
 556         disp_fn = octeon_get_dispatch(oct, (u16)rh->r.opcode,
 557                                       (u16)rh->r.subcode);
 558         if (disp_fn) {
 559                 rinfo = octeon_create_recv_info(oct, droq, cnt, droq->read_idx);
 560                 if (rinfo) {
 561                         struct __dispatch *rdisp = rinfo->rsvd;
 562
 563                         rdisp->rinfo = rinfo;
 564                         rdisp->disp_fn = disp_fn;
 565                         rinfo->recv_pkt->rh = *rh;
 566                         list_add_tail(&rdisp->list,
 567                                       &droq->dispatch_list);
 568                 } else {
 569                         droq->stats.dropped_nomem++;
 570                 }
 571         } else {
 572                 dev_err(&oct->pci_dev->dev, "DROQ: No dispatch function (opcode %u/%u)\n",
 573                         (unsigned int)rh->r.opcode,
 574                         (unsigned int)rh->r.subcode);
 575                 droq->stats.dropped_nodispatch++;
 576         }
 577
 578         return cnt;
 579 }
 580
 581 static inline void octeon_droq_drop_packets(struct octeon_device *oct,
 582                                             struct octeon_droq *droq,
 583                                             u32 cnt)
 584 {
 585         u32 i = 0, buf_cnt;
 586         struct octeon_droq_info *info;
 587
 588         for (i = 0; i < cnt; i++) {
 589                 info = &droq->info_list[droq->read_idx];
 590                 octeon_swap_8B_data((u64 *)info, 2);
 591
 592                 if (info->length) {
 593                         info->length -= OCT_RH_SIZE;
 594                         droq->stats.bytes_received += info->length;
 595                         buf_cnt = octeon_droq_get_bufcount(droq->buffer_size,
 596                                                            (u32)info->length);
 597                 } else {
 598                         dev_err(&oct->pci_dev->dev, "DROQ: In drop: pkt with len 0\n");
 599                         buf_cnt = 1;
 600                 }
 601
 602                 INCR_INDEX(droq->read_idx, buf_cnt, droq->max_count);
 603                 droq->refill_count += buf_cnt;
 604         }
 605 }
 606
 607 static u32
 608 octeon_droq_fast_process_packets(struct octeon_device *oct,
 609                                  struct octeon_droq *droq,
 610                                  u32 pkts_to_process)
 611 {
 612         struct octeon_droq_info *info;
 613         union octeon_rh *rh;
 614         u32 pkt, total_len = 0, pkt_count;
 615
 616         pkt_count = pkts_to_process;
 617
 618         for (pkt = 0; pkt < pkt_count; pkt++) {
 619                 u32 pkt_len = 0;
 620                 struct sk_buff *nicbuf = NULL;
 621                 struct octeon_skb_page_info *pg_info;
 622                 void *buf;
 623
 624                 info = &droq->info_list[droq->read_idx];
 625                 octeon_swap_8B_data((u64 *)info, 2);
 626
 627                 if (!info->length) {
 628                         dev_err(&oct->pci_dev->dev,
 629                                 "DROQ[%d] idx: %d len:0, pkt_cnt: %d\n",
 630                                 droq->q_no, droq->read_idx, pkt_count);
 631                         print_hex_dump_bytes("", DUMP_PREFIX_ADDRESS,
 632                                              (u8 *)info,
 633                                              OCT_DROQ_INFO_SIZE);
 634                         break;
 635                 }
 636
 637                 /* Len of resp hdr in included in the received data len. */
 638                 info->length -= OCT_RH_SIZE;
 639                 rh = &info->rh;
 640
 641                 total_len += (u32)info->length;
 642                 if (OPCODE_SLOW_PATH(rh)) {
 643                         u32 buf_cnt;
 644
 645                         buf_cnt = octeon_droq_dispatch_pkt(oct, droq, rh, info);
 646                         INCR_INDEX(droq->read_idx, buf_cnt, droq->max_count);
 647                         droq->refill_count += buf_cnt;
 648                 } else {
 649                         if (info->length <= droq->buffer_size) {
 650                                 pkt_len = (u32)info->length;
 651                                 nicbuf = droq->recv_buf_list[
 652                                         droq->read_idx].buffer;
 653                                 pg_info = &droq->recv_buf_list[
 654                                         droq->read_idx].pg_info;
 655                                 if (recv_buffer_recycle(oct, pg_info))
 656                                         pg_info->page = NULL;
 657                                 droq->recv_buf_list[droq->read_idx].buffer =
 658                                         NULL;
 659
 660                                 INCR_INDEX_BY1(droq->read_idx, droq->max_count);
 661                                 droq->refill_count++;
 662                         } else {
 663                                 nicbuf = octeon_fast_packet_alloc((u32)
 664                                                                   info->length);
 665                                 pkt_len = 0;
 666                                 /* nicbuf allocation can fail. We'll handle it
 667                                  * inside the loop.
 668                                  */
 669                                 while (pkt_len < info->length) {
 670                                         int cpy_len, idx = droq->read_idx;
 671
 672                                         cpy_len = ((pkt_len + droq->buffer_size)
 673                                                    > info->length) ?
 674                                                 ((u32)info->length - pkt_len) :
 675                                                 droq->buffer_size;
 676
 677                                         if (nicbuf) {
 678                                                 octeon_fast_packet_next(droq,
 679                                                                         nicbuf,
 680                                                                         cpy_len,
 681                                                                         idx);
 682                                                 buf = droq->recv_buf_list[idx].
 683                                                         buffer;
 684                                                 recv_buffer_fast_free(buf);
 685                                                 droq->recv_buf_list[idx].buffer
 686                                                         = NULL;
 687                                         } else {
 688                                                 droq->stats.rx_alloc_failure++;
 689                                         }
 690
 691                                         pkt_len += cpy_len;
 692                                         INCR_INDEX_BY1(droq->read_idx,
 693                                                        droq->max_count);
 694                                         droq->refill_count++;
 695                                 }
 696                         }
 697
 698                         if (nicbuf) {
 699                                 if (droq->ops.fptr) {
 700                                         droq->ops.fptr(oct->octeon_id,
 701                                                        nicbuf, pkt_len,
 702                                                        rh, &droq->napi,
 703                                                        droq->ops.farg);
 704                                 } else {
 705                                         recv_buffer_free(nicbuf);
 706                                 }
 707                         }
 708                 }
 709
 710                 if (droq->refill_count >= droq->refill_threshold) {
 711                         int desc_refilled = octeon_droq_refill(oct, droq);
 712
 713                         /* Flush the droq descriptor data to memory to be sure
 714                          * that when we update the credits the data in memory
 715                          * is accurate.
 716                          */
 717                         wmb();
 718                         writel((desc_refilled), droq->pkts_credit_reg);
 719                         /* make sure mmio write completes */
 720                         mmiowb();
 721                 }
 722
 723         }                       /* for (each packet)... */
 724
 725         /* Increment refill_count by the number of buffers processed. */
 726         droq->stats.pkts_received += pkt;
 727         droq->stats.bytes_received += total_len;
 728
 729         if ((droq->ops.drop_on_max) && (pkts_to_process - pkt)) {
 730                 octeon_droq_drop_packets(oct, droq, (pkts_to_process - pkt));
 731
 732                 droq->stats.dropped_toomany += (pkts_to_process - pkt);
 733                 return pkts_to_process;
 734         }
 735
 736         return pkt;
 737 }
 738
 739 int
 740 octeon_droq_process_packets(struct octeon_device *oct,
 741                             struct octeon_droq *droq,
 742                             u32 budget)
 743 {
 744         u32 pkt_count = 0, pkts_processed = 0;
 745         struct list_head *tmp, *tmp2;
 746
 747         /* Grab the droq lock */
 748         spin_lock(&droq->lock);
 749
 750         octeon_droq_check_hw_for_pkts(droq);
 751         pkt_count = atomic_read(&droq->pkts_pending);
 752
 753         if (!pkt_count) {
 754                 spin_unlock(&droq->lock);
 755                 return 0;
 756         }
 757
 758         if (pkt_count > budget)
 759                 pkt_count = budget;
 760
 761         pkts_processed = octeon_droq_fast_process_packets(oct, droq, pkt_count);
 762
 763         atomic_sub(pkts_processed, &droq->pkts_pending);
 764
 765         /* Release the spin lock */
 766         spin_unlock(&droq->lock);
 767
 768         list_for_each_safe(tmp, tmp2, &droq->dispatch_list) {
 769                 struct __dispatch *rdisp = (struct __dispatch *)tmp;
 770
 771                 list_del(tmp);
 772                 rdisp->disp_fn(rdisp->rinfo,
 773                                octeon_get_dispatch_arg
 774                                (oct,
 775                                 (u16)rdisp->rinfo->recv_pkt->rh.r.opcode,
 776                                 (u16)rdisp->rinfo->recv_pkt->rh.r.subcode));
 777         }
 778
 779         /* If there are packets pending. schedule tasklet again */
 780         if (atomic_read(&droq->pkts_pending))
 781                 return 1;
 782
 783         return 0;
 784 }
 785
 786 /**
 787  * Utility function to poll for packets. check_hw_for_packets must be
 788  * called before calling this routine.
 789  */
 790
 791 static int
 792 octeon_droq_process_poll_pkts(struct octeon_device *oct,
 793                               struct octeon_droq *droq, u32 budget)
 794 {
 795         struct list_head *tmp, *tmp2;
 796         u32 pkts_available = 0, pkts_processed = 0;
 797         u32 total_pkts_processed = 0;
 798
 799         if (budget > droq->max_count)
 800                 budget = droq->max_count;
 801
 802         spin_lock(&droq->lock);
 803
 804         while (total_pkts_processed < budget) {
 805                 octeon_droq_check_hw_for_pkts(droq);
 806
 807                 pkts_available =
 808                         CVM_MIN((budget - total_pkts_processed),
 809                                 (u32)(atomic_read(&droq->pkts_pending)));
 810
 811                 if (pkts_available == 0)
 812                         break;
 813
 814                 pkts_processed =
 815                         octeon_droq_fast_process_packets(oct, droq,
 816                                                          pkts_available);
 817
 818                 atomic_sub(pkts_processed, &droq->pkts_pending);
 819
 820                 total_pkts_processed += pkts_processed;
 821         }
 822
 823         spin_unlock(&droq->lock);
 824
 825         list_for_each_safe(tmp, tmp2, &droq->dispatch_list) {
 826                 struct __dispatch *rdisp = (struct __dispatch *)tmp;
 827
 828                 list_del(tmp);
 829                 rdisp->disp_fn(rdisp->rinfo,
 830                                octeon_get_dispatch_arg
 831                                (oct,
 832                                 (u16)rdisp->rinfo->recv_pkt->rh.r.opcode,
 833                                 (u16)rdisp->rinfo->recv_pkt->rh.r.subcode));
 834         }
 835
 836         return total_pkts_processed;
 837 }
 838
 839 int
 840 octeon_process_droq_poll_cmd(struct octeon_device *oct, u32 q_no, int cmd,
 841                              u32 arg)
 842 {
 843         struct octeon_droq *droq;
 844
 845         droq = oct->droq[q_no];
 846
 847         if (cmd == POLL_EVENT_PROCESS_PKTS)
 848                 return octeon_droq_process_poll_pkts(oct, droq, arg);
 849
 850         if (cmd == POLL_EVENT_PENDING_PKTS) {
 851                 u32 pkt_cnt = atomic_read(&droq->pkts_pending);
 852
 853                 return  octeon_droq_process_packets(oct, droq, pkt_cnt);
 854         }
 855
 856         if (cmd == POLL_EVENT_ENABLE_INTR) {
 857                 u32 value;
 858                 unsigned long flags;
 859
 860                 /* Enable Pkt Interrupt */
 861                 switch (oct->chip_id) {
 862                 case OCTEON_CN66XX:
 863                 case OCTEON_CN68XX: {
 864                         struct octeon_cn6xxx *cn6xxx =
 865                                 (struct octeon_cn6xxx *)oct->chip;
 866                         spin_lock_irqsave
 867                                 (&cn6xxx->lock_for_droq_int_enb_reg, flags);
 868                         value =
 869                                 octeon_read_csr(oct,
 870                                                 CN6XXX_SLI_PKT_TIME_INT_ENB);
 871                         value |= (1 << q_no);
 872                         octeon_write_csr(oct,
 873                                          CN6XXX_SLI_PKT_TIME_INT_ENB,
 874                                          value);
 875                         value =
 876                                 octeon_read_csr(oct,
 877                                                 CN6XXX_SLI_PKT_CNT_INT_ENB);
 878                         value |= (1 << q_no);
 879                         octeon_write_csr(oct,
 880                                          CN6XXX_SLI_PKT_CNT_INT_ENB,
 881                                          value);
 882
 883                         /* don't bother flushing the enables */
 884
 885                         spin_unlock_irqrestore
 886                                 (&cn6xxx->lock_for_droq_int_enb_reg, flags);
 887                         return 0;
 888                 }
 889                 break;
 890                 case OCTEON_CN23XX_PF_VID: {
 891                         lio_enable_irq(oct->droq[q_no], oct->instr_queue[q_no]);
 892                 }
 893                 break;
 894                 }
 895                 return 0;
 896         }
 897
 898         dev_err(&oct->pci_dev->dev, "%s Unknown command: %d\n", __func__, cmd);
 899         return -EINVAL;
 900 }
 901
 902 int octeon_register_droq_ops(struct octeon_device *oct, u32 q_no,
 903                              struct octeon_droq_ops *ops)
 904 {
 905         struct octeon_droq *droq;
 906         unsigned long flags;
 907         struct octeon_config *oct_cfg = NULL;
 908
 909         oct_cfg = octeon_get_conf(oct);
 910
 911         if (!oct_cfg)
 912                 return -EINVAL;
 913
 914         if (!(ops)) {
 915                 dev_err(&oct->pci_dev->dev, "%s: droq_ops pointer is NULL\n",
 916                         __func__);
 917                 return -EINVAL;
 918         }
 919
 920         if (q_no >= CFG_GET_OQ_MAX_Q(oct_cfg)) {
 921                 dev_err(&oct->pci_dev->dev, "%s: droq id (%d) exceeds MAX (%d)\n",
 922                         __func__, q_no, (oct->num_oqs - 1));
 923                 return -EINVAL;
 924         }
 925
 926         droq = oct->droq[q_no];
 927
 928         spin_lock_irqsave(&droq->lock, flags);
 929
 930         memcpy(&droq->ops, ops, sizeof(struct octeon_droq_ops));
 931
 932         spin_unlock_irqrestore(&droq->lock, flags);
 933
 934         return 0;
 935 }
 936
 937 int octeon_unregister_droq_ops(struct octeon_device *oct, u32 q_no)
 938 {
 939         unsigned long flags;
 940         struct octeon_droq *droq;
 941         struct octeon_config *oct_cfg = NULL;
 942
 943         oct_cfg = octeon_get_conf(oct);
 944
 945         if (!oct_cfg)
 946                 return -EINVAL;
 947
 948         if (q_no >= CFG_GET_OQ_MAX_Q(oct_cfg)) {
 949                 dev_err(&oct->pci_dev->dev, "%s: droq id (%d) exceeds MAX (%d)\n",
 950                         __func__, q_no, oct->num_oqs - 1);
 951                 return -EINVAL;
 952         }
 953
 954         droq = oct->droq[q_no];
 955
 956         if (!droq) {
 957                 dev_info(&oct->pci_dev->dev,
 958                          "Droq id (%d) not available.\n", q_no);
 959                 return 0;
 960         }
 961
 962         spin_lock_irqsave(&droq->lock, flags);
 963
 964         droq->ops.fptr = NULL;
 965         droq->ops.farg = NULL;
 966         droq->ops.drop_on_max = 0;
 967
 968         spin_unlock_irqrestore(&droq->lock, flags);
 969
 970         return 0;
 971 }
 972
 973 int octeon_create_droq(struct octeon_device *oct,
 974                        u32 q_no, u32 num_descs,
 975                        u32 desc_size, void *app_ctx)
 976 {
 977         struct octeon_droq *droq;
 978         int numa_node = cpu_to_node(q_no % num_online_cpus());
 979
 980         if (oct->droq[q_no]) {
 981                 dev_dbg(&oct->pci_dev->dev, "Droq already in use. Cannot create droq %d again\n",
 982                         q_no);
 983                 return 1;
 984         }
 985
 986         /* Allocate the DS for the new droq. */
 987         droq = vmalloc_node(sizeof(*droq), numa_node);
 988         if (!droq)
 989                 droq = vmalloc(sizeof(*droq));
 990         if (!droq)
 991                 goto create_droq_fail;
 992         memset(droq, 0, sizeof(struct octeon_droq));
 993
 994         /*Disable the pkt o/p for this Q  */
 995         octeon_set_droq_pkt_op(oct, q_no, 0);
 996         oct->droq[q_no] = droq;
 997
 998         /* Initialize the Droq */
 999         octeon_init_droq(oct, q_no, num_descs, desc_size, app_ctx);
1000
1001         oct->num_oqs++;
1002
1003         dev_dbg(&oct->pci_dev->dev, "%s: Total number of OQ: %d\n", __func__,
1004                 oct->num_oqs);
1005
1006         /* Global Droq register settings */
1007
1008         /* As of now not required, as setting are done for all 32 Droqs at
1009          * the same time.
1010          */
1011         return 0;
1012
1013 create_droq_fail:
1014         octeon_delete_droq(oct, q_no);
1015         return -ENOMEM;
1016 }