drivers/net/ethernet/cavium/liquidio/cn23xx_pf_device.c

   1 /**********************************************************************
   2 * Author: Cavium, Inc.
   3 *
   4 * Contact: support@cavium.com
   5 *          Please include "LiquidIO" in the subject.
   6 *
   7 * Copyright (c) 2003-2015 Cavium, Inc.
   8 *
   9 * This file is free software; you can redistribute it and/or modify
  10 * it under the terms of the GNU General Public License, Version 2, as
  11 * published by the Free Software Foundation.
  12 *
  13 * This file is distributed in the hope that it will be useful, but
  14 * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty
  15 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or
  16 * NONINFRINGEMENT.  See the GNU General Public License for more
  17 * details.
  18 *
  19 * This file may also be available under a different license from Cavium.
  20 * Contact Cavium, Inc. for more information
  21 **********************************************************************/
  22
  23 #include <linux/pci.h>
  24 #include <linux/netdevice.h>
  25 #include <linux/vmalloc.h>
  26 #include "liquidio_common.h"
  27 #include "octeon_droq.h"
  28 #include "octeon_iq.h"
  29 #include "response_manager.h"
  30 #include "octeon_device.h"
  31 #include "cn23xx_pf_device.h"
  32 #include "octeon_main.h"
  33
  34 #define RESET_NOTDONE 0
  35 #define RESET_DONE 1
  36
  37 /* Change the value of SLI Packet Input Jabber Register to allow
  38  * VXLAN TSO packets which can be 64424 bytes, exceeding the
  39  * MAX_GSO_SIZE we supplied to the kernel
  40  */
  41 #define CN23XX_INPUT_JABBER 64600
  42
  43 #define LIOLUT_RING_DISTRIBUTION 9
  44 const int liolut_num_vfs_to_rings_per_vf[LIOLUT_RING_DISTRIBUTION] = {
  45         0, 8, 4, 2, 2, 2, 1, 1, 1
  46 };
  47
  48 void cn23xx_dump_pf_initialized_regs(struct octeon_device *oct)
  49 {
  50         int i = 0;
  51         u32 regval = 0;
  52         struct octeon_cn23xx_pf *cn23xx = (struct octeon_cn23xx_pf *)oct->chip;
  53
  54         /*In cn23xx_soft_reset*/
  55         dev_dbg(&oct->pci_dev->dev, "%s[%llx] : 0x%llx\n",
  56                 "CN23XX_WIN_WR_MASK_REG", CVM_CAST64(CN23XX_WIN_WR_MASK_REG),
  57                 CVM_CAST64(octeon_read_csr64(oct, CN23XX_WIN_WR_MASK_REG)));
  58         dev_dbg(&oct->pci_dev->dev, "%s[%llx] : 0x%016llx\n",
  59                 "CN23XX_SLI_SCRATCH1", CVM_CAST64(CN23XX_SLI_SCRATCH1),
  60                 CVM_CAST64(octeon_read_csr64(oct, CN23XX_SLI_SCRATCH1)));
  61         dev_dbg(&oct->pci_dev->dev, "%s[%llx] : 0x%016llx\n",
  62                 "CN23XX_RST_SOFT_RST", CN23XX_RST_SOFT_RST,
  63                 lio_pci_readq(oct, CN23XX_RST_SOFT_RST));
  64
  65         /*In cn23xx_set_dpi_regs*/
  66         dev_dbg(&oct->pci_dev->dev, "%s[%llx] : 0x%016llx\n",
  67                 "CN23XX_DPI_DMA_CONTROL", CN23XX_DPI_DMA_CONTROL,
  68                 lio_pci_readq(oct, CN23XX_DPI_DMA_CONTROL));
  69
  70         for (i = 0; i < 6; i++) {
  71                 dev_dbg(&oct->pci_dev->dev, "%s(%d)[%llx] : 0x%016llx\n",
  72                         "CN23XX_DPI_DMA_ENG_ENB", i,
  73                         CN23XX_DPI_DMA_ENG_ENB(i),
  74                         lio_pci_readq(oct, CN23XX_DPI_DMA_ENG_ENB(i)));
  75                 dev_dbg(&oct->pci_dev->dev, "%s(%d)[%llx] : 0x%016llx\n",
  76                         "CN23XX_DPI_DMA_ENG_BUF", i,
  77                         CN23XX_DPI_DMA_ENG_BUF(i),
  78                         lio_pci_readq(oct, CN23XX_DPI_DMA_ENG_BUF(i)));
  79         }
  80
  81         dev_dbg(&oct->pci_dev->dev, "%s[%llx] : 0x%016llx\n", "CN23XX_DPI_CTL",
  82                 CN23XX_DPI_CTL, lio_pci_readq(oct, CN23XX_DPI_CTL));
  83
  84         /*In cn23xx_setup_pcie_mps and cn23xx_setup_pcie_mrrs */
  85         pci_read_config_dword(oct->pci_dev, CN23XX_CONFIG_PCIE_DEVCTL, &regval);
  86         dev_dbg(&oct->pci_dev->dev, "%s[%llx] : 0x%016llx\n",
  87                 "CN23XX_CONFIG_PCIE_DEVCTL",
  88                 CVM_CAST64(CN23XX_CONFIG_PCIE_DEVCTL), CVM_CAST64(regval));
  89
  90         dev_dbg(&oct->pci_dev->dev, "%s(%d)[%llx] : 0x%016llx\n",
  91                 "CN23XX_DPI_SLI_PRTX_CFG", oct->pcie_port,
  92                 CN23XX_DPI_SLI_PRTX_CFG(oct->pcie_port),
  93                 lio_pci_readq(oct, CN23XX_DPI_SLI_PRTX_CFG(oct->pcie_port)));
  94
  95         /*In cn23xx_specific_regs_setup */
  96         dev_dbg(&oct->pci_dev->dev, "%s(%d)[%llx] : 0x%016llx\n",
  97                 "CN23XX_SLI_S2M_PORTX_CTL", oct->pcie_port,
  98                 CVM_CAST64(CN23XX_SLI_S2M_PORTX_CTL(oct->pcie_port)),
  99                 CVM_CAST64(octeon_read_csr64(
 100                         oct, CN23XX_SLI_S2M_PORTX_CTL(oct->pcie_port))));
 101
 102         dev_dbg(&oct->pci_dev->dev, "%s[%llx] : 0x%016llx\n",
 103                 "CN23XX_SLI_RING_RST", CVM_CAST64(CN23XX_SLI_PKT_IOQ_RING_RST),
 104                 (u64)octeon_read_csr64(oct, CN23XX_SLI_PKT_IOQ_RING_RST));
 105
 106         /*In cn23xx_setup_global_mac_regs*/
 107         for (i = 0; i < CN23XX_MAX_MACS; i++) {
 108                 dev_dbg(&oct->pci_dev->dev, "%s(%d)[%llx] : 0x%016llx\n",
 109                         "CN23XX_SLI_PKT_MAC_RINFO64", i,
 110                         CVM_CAST64(CN23XX_SLI_PKT_MAC_RINFO64(i, oct->pf_num)),
 111                         CVM_CAST64(octeon_read_csr64
 112                                 (oct, CN23XX_SLI_PKT_MAC_RINFO64
 113                                         (i, oct->pf_num))));
 114         }
 115
 116         /*In cn23xx_setup_global_input_regs*/
 117         for (i = 0; i < CN23XX_MAX_INPUT_QUEUES; i++) {
 118                 dev_dbg(&oct->pci_dev->dev, "%s(%d)[%llx] : 0x%016llx\n",
 119                         "CN23XX_SLI_IQ_PKT_CONTROL64", i,
 120                         CVM_CAST64(CN23XX_SLI_IQ_PKT_CONTROL64(i)),
 121                         CVM_CAST64(octeon_read_csr64
 122                                 (oct, CN23XX_SLI_IQ_PKT_CONTROL64(i))));
 123         }
 124
 125         /*In cn23xx_setup_global_output_regs*/
 126         dev_dbg(&oct->pci_dev->dev, "%s[%llx] : 0x%016llx\n",
 127                 "CN23XX_SLI_OQ_WMARK", CVM_CAST64(CN23XX_SLI_OQ_WMARK),
 128                 CVM_CAST64(octeon_read_csr64(oct, CN23XX_SLI_OQ_WMARK)));
 129
 130         for (i = 0; i < CN23XX_MAX_OUTPUT_QUEUES; i++) {
 131                 dev_dbg(&oct->pci_dev->dev, "%s(%d)[%llx] : 0x%016llx\n",
 132                         "CN23XX_SLI_OQ_PKT_CONTROL", i,
 133                         CVM_CAST64(CN23XX_SLI_OQ_PKT_CONTROL(i)),
 134                         CVM_CAST64(octeon_read_csr(
 135                                 oct, CN23XX_SLI_OQ_PKT_CONTROL(i))));
 136                 dev_dbg(&oct->pci_dev->dev, "%s(%d)[%llx] : 0x%016llx\n",
 137                         "CN23XX_SLI_OQ_PKT_INT_LEVELS", i,
 138                         CVM_CAST64(CN23XX_SLI_OQ_PKT_INT_LEVELS(i)),
 139                         CVM_CAST64(octeon_read_csr64(
 140                                 oct, CN23XX_SLI_OQ_PKT_INT_LEVELS(i))));
 141         }
 142
 143         /*In cn23xx_enable_interrupt and cn23xx_disable_interrupt*/
 144         dev_dbg(&oct->pci_dev->dev, "%s[%llx] : 0x%016llx\n",
 145                 "cn23xx->intr_enb_reg64",
 146                 CVM_CAST64((long)(cn23xx->intr_enb_reg64)),
 147                 CVM_CAST64(readq(cn23xx->intr_enb_reg64)));
 148
 149         dev_dbg(&oct->pci_dev->dev, "%s[%llx] : 0x%016llx\n",
 150                 "cn23xx->intr_sum_reg64",
 151                 CVM_CAST64((long)(cn23xx->intr_sum_reg64)),
 152                 CVM_CAST64(readq(cn23xx->intr_sum_reg64)));
 153
 154         /*In cn23xx_setup_iq_regs*/
 155         for (i = 0; i < CN23XX_MAX_INPUT_QUEUES; i++) {
 156                 dev_dbg(&oct->pci_dev->dev, "%s(%d)[%llx] : 0x%016llx\n",
 157                         "CN23XX_SLI_IQ_BASE_ADDR64", i,
 158                         CVM_CAST64(CN23XX_SLI_IQ_BASE_ADDR64(i)),
 159                         CVM_CAST64(octeon_read_csr64(
 160                                 oct, CN23XX_SLI_IQ_BASE_ADDR64(i))));
 161                 dev_dbg(&oct->pci_dev->dev, "%s(%d)[%llx] : 0x%016llx\n",
 162                         "CN23XX_SLI_IQ_SIZE", i,
 163                         CVM_CAST64(CN23XX_SLI_IQ_SIZE(i)),
 164                         CVM_CAST64(octeon_read_csr
 165                                 (oct, CN23XX_SLI_IQ_SIZE(i))));
 166                 dev_dbg(&oct->pci_dev->dev, "%s(%d)[%llx] : 0x%016llx\n",
 167                         "CN23XX_SLI_IQ_DOORBELL", i,
 168                         CVM_CAST64(CN23XX_SLI_IQ_DOORBELL(i)),
 169                         CVM_CAST64(octeon_read_csr64(
 170                                 oct, CN23XX_SLI_IQ_DOORBELL(i))));
 171                 dev_dbg(&oct->pci_dev->dev, "%s(%d)[%llx] : 0x%016llx\n",
 172                         "CN23XX_SLI_IQ_INSTR_COUNT64", i,
 173                         CVM_CAST64(CN23XX_SLI_IQ_INSTR_COUNT64(i)),
 174                         CVM_CAST64(octeon_read_csr64(
 175                                 oct, CN23XX_SLI_IQ_INSTR_COUNT64(i))));
 176         }
 177
 178         /*In cn23xx_setup_oq_regs*/
 179         for (i = 0; i < CN23XX_MAX_OUTPUT_QUEUES; i++) {
 180                 dev_dbg(&oct->pci_dev->dev, "%s(%d)[%llx] : 0x%016llx\n",
 181                         "CN23XX_SLI_OQ_BASE_ADDR64", i,
 182                         CVM_CAST64(CN23XX_SLI_OQ_BASE_ADDR64(i)),
 183                         CVM_CAST64(octeon_read_csr64(
 184                                 oct, CN23XX_SLI_OQ_BASE_ADDR64(i))));
 185                 dev_dbg(&oct->pci_dev->dev, "%s(%d)[%llx] : 0x%016llx\n",
 186                         "CN23XX_SLI_OQ_SIZE", i,
 187                         CVM_CAST64(CN23XX_SLI_OQ_SIZE(i)),
 188                         CVM_CAST64(octeon_read_csr
 189                                 (oct, CN23XX_SLI_OQ_SIZE(i))));
 190                 dev_dbg(&oct->pci_dev->dev, "%s(%d)[%llx] : 0x%016llx\n",
 191                         "CN23XX_SLI_OQ_BUFF_INFO_SIZE", i,
 192                         CVM_CAST64(CN23XX_SLI_OQ_BUFF_INFO_SIZE(i)),
 193                         CVM_CAST64(octeon_read_csr(
 194                                 oct, CN23XX_SLI_OQ_BUFF_INFO_SIZE(i))));
 195                 dev_dbg(&oct->pci_dev->dev, "%s(%d)[%llx] : 0x%016llx\n",
 196                         "CN23XX_SLI_OQ_PKTS_SENT", i,
 197                         CVM_CAST64(CN23XX_SLI_OQ_PKTS_SENT(i)),
 198                         CVM_CAST64(octeon_read_csr64(
 199                                 oct, CN23XX_SLI_OQ_PKTS_SENT(i))));
 200                 dev_dbg(&oct->pci_dev->dev, "%s(%d)[%llx] : 0x%016llx\n",
 201                         "CN23XX_SLI_OQ_PKTS_CREDIT", i,
 202                         CVM_CAST64(CN23XX_SLI_OQ_PKTS_CREDIT(i)),
 203                         CVM_CAST64(octeon_read_csr64(
 204                                 oct, CN23XX_SLI_OQ_PKTS_CREDIT(i))));
 205         }
 206
 207         dev_dbg(&oct->pci_dev->dev, "%s[%llx] : 0x%016llx\n",
 208                 "CN23XX_SLI_PKT_TIME_INT",
 209                 CVM_CAST64(CN23XX_SLI_PKT_TIME_INT),
 210                 CVM_CAST64(octeon_read_csr64(oct, CN23XX_SLI_PKT_TIME_INT)));
 211         dev_dbg(&oct->pci_dev->dev, "%s[%llx] : 0x%016llx\n",
 212                 "CN23XX_SLI_PKT_CNT_INT",
 213                 CVM_CAST64(CN23XX_SLI_PKT_CNT_INT),
 214                 CVM_CAST64(octeon_read_csr64(oct, CN23XX_SLI_PKT_CNT_INT)));
 215 }
 216
 217 static int cn23xx_pf_soft_reset(struct octeon_device *oct)
 218 {
 219         octeon_write_csr64(oct, CN23XX_WIN_WR_MASK_REG, 0xFF);
 220
 221         dev_dbg(&oct->pci_dev->dev, "OCTEON[%d]: BIST enabled for CN23XX soft reset\n",
 222                 oct->octeon_id);
 223
 224         octeon_write_csr64(oct, CN23XX_SLI_SCRATCH1, 0x1234ULL);
 225
 226         /* Initiate chip-wide soft reset */
 227         lio_pci_readq(oct, CN23XX_RST_SOFT_RST);
 228         lio_pci_writeq(oct, 1, CN23XX_RST_SOFT_RST);
 229
 230         /* Wait for 100ms as Octeon resets. */
 231         mdelay(100);
 232
 233         if (octeon_read_csr64(oct, CN23XX_SLI_SCRATCH1) == 0x1234ULL) {
 234                 dev_err(&oct->pci_dev->dev, "OCTEON[%d]: Soft reset failed\n",
 235                         oct->octeon_id);
 236                 return 1;
 237         }
 238
 239         dev_dbg(&oct->pci_dev->dev, "OCTEON[%d]: Reset completed\n",
 240                 oct->octeon_id);
 241
 242         /* restore the  reset value*/
 243         octeon_write_csr64(oct, CN23XX_WIN_WR_MASK_REG, 0xFF);
 244
 245         return 0;
 246 }
 247
 248 static void cn23xx_enable_error_reporting(struct octeon_device *oct)
 249 {
 250         u32 regval;
 251         u32 uncorrectable_err_mask, corrtable_err_status;
 252
 253         pci_read_config_dword(oct->pci_dev, CN23XX_CONFIG_PCIE_DEVCTL, &regval);
 254         if (regval & CN23XX_CONFIG_PCIE_DEVCTL_MASK) {
 255                 uncorrectable_err_mask = 0;
 256                 corrtable_err_status = 0;
 257                 pci_read_config_dword(oct->pci_dev,
 258                                       CN23XX_CONFIG_PCIE_UNCORRECT_ERR_MASK,
 259                                       &uncorrectable_err_mask);
 260                 pci_read_config_dword(oct->pci_dev,
 261                                       CN23XX_CONFIG_PCIE_CORRECT_ERR_STATUS,
 262                                       &corrtable_err_status);
 263                 dev_err(&oct->pci_dev->dev, "PCI-E Fatal error detected;\n"
 264                                  "\tdev_ctl_status_reg = 0x%08x\n"
 265                                  "\tuncorrectable_error_mask_reg = 0x%08x\n"
 266                                  "\tcorrectable_error_status_reg = 0x%08x\n",
 267                             regval, uncorrectable_err_mask,
 268                             corrtable_err_status);
 269         }
 270
 271         regval |= 0xf; /* Enable Link error reporting */
 272
 273         dev_dbg(&oct->pci_dev->dev, "OCTEON[%d]: Enabling PCI-E error reporting..\n",
 274                 oct->octeon_id);
 275         pci_write_config_dword(oct->pci_dev, CN23XX_CONFIG_PCIE_DEVCTL, regval);
 276 }
 277
 278 static u32 cn23xx_coprocessor_clock(struct octeon_device *oct)
 279 {
 280         /* Bits 29:24 of RST_BOOT[PNR_MUL] holds the ref.clock MULTIPLIER
 281          * for SLI.
 282          */
 283
 284         /* TBD: get the info in Hand-shake */
 285         return (((lio_pci_readq(oct, CN23XX_RST_BOOT) >> 24) & 0x3f) * 50);
 286 }
 287
 288 u32 cn23xx_pf_get_oq_ticks(struct octeon_device *oct, u32 time_intr_in_us)
 289 {
 290         /* This gives the SLI clock per microsec */
 291         u32 oqticks_per_us = cn23xx_coprocessor_clock(oct);
 292
 293         oct->pfvf_hsword.coproc_tics_per_us = oqticks_per_us;
 294
 295         /* This gives the clock cycles per millisecond */
 296         oqticks_per_us *= 1000;
 297
 298         /* This gives the oq ticks (1024 core clock cycles) per millisecond */
 299         oqticks_per_us /= 1024;
 300
 301         /* time_intr is in microseconds. The next 2 steps gives the oq ticks
 302          *  corressponding to time_intr.
 303          */
 304         oqticks_per_us *= time_intr_in_us;
 305         oqticks_per_us /= 1000;
 306
 307         return oqticks_per_us;
 308 }
 309
 310 static void cn23xx_setup_global_mac_regs(struct octeon_device *oct)
 311 {
 312         u64 reg_val;
 313         u16 mac_no = oct->pcie_port;
 314         u16 pf_num = oct->pf_num;
 315
 316         /* programming SRN and TRS for each MAC(0..3)  */
 317
 318         dev_dbg(&oct->pci_dev->dev, "%s:Using pcie port %d\n",
 319                 __func__, mac_no);
 320         /* By default, mapping all 64 IOQs to  a single MACs */
 321
 322         reg_val =
 323             octeon_read_csr64(oct, CN23XX_SLI_PKT_MAC_RINFO64(mac_no, pf_num));
 324
 325         if (oct->rev_id == OCTEON_CN23XX_REV_1_1) {
 326                 /* setting SRN <6:0>  */
 327                 reg_val = pf_num * CN23XX_MAX_RINGS_PER_PF_PASS_1_1;
 328         } else {
 329                 /* setting SRN <6:0>  */
 330                 reg_val = pf_num * CN23XX_MAX_RINGS_PER_PF;
 331         }
 332
 333         /* setting TRS <23:16> */
 334         reg_val = reg_val |
 335                   (oct->sriov_info.trs << CN23XX_PKT_MAC_CTL_RINFO_TRS_BIT_POS);
 336         /* write these settings to MAC register */
 337         octeon_write_csr64(oct, CN23XX_SLI_PKT_MAC_RINFO64(mac_no, pf_num),
 338                            reg_val);
 339
 340         dev_dbg(&oct->pci_dev->dev, "SLI_PKT_MAC(%d)_PF(%d)_RINFO : 0x%016llx\n",
 341                 mac_no, pf_num, (u64)octeon_read_csr64
 342                 (oct, CN23XX_SLI_PKT_MAC_RINFO64(mac_no, pf_num)));
 343 }
 344
 345 static int cn23xx_reset_io_queues(struct octeon_device *oct)
 346 {
 347         int ret_val = 0;
 348         u64 d64;
 349         u32 q_no, srn, ern;
 350         u32 loop = 1000;
 351
 352         srn = oct->sriov_info.pf_srn;
 353         ern = srn + oct->sriov_info.num_pf_rings;
 354
 355         /*As per HRM reg description, s/w cant write 0 to ENB. */
 356         /*to make the queue off, need to set the RST bit. */
 357
 358         /* Reset the Enable bit for all the 64 IQs.  */
 359         for (q_no = srn; q_no < ern; q_no++) {
 360                 /* set RST bit to 1. This bit applies to both IQ and OQ */
 361                 d64 = octeon_read_csr64(oct, CN23XX_SLI_IQ_PKT_CONTROL64(q_no));
 362                 d64 = d64 | CN23XX_PKT_INPUT_CTL_RST;
 363                 octeon_write_csr64(oct, CN23XX_SLI_IQ_PKT_CONTROL64(q_no), d64);
 364         }
 365
 366         /*wait until the RST bit is clear or the RST and quite bits are set*/
 367         for (q_no = srn; q_no < ern; q_no++) {
 368                 u64 reg_val = octeon_read_csr64(oct,
 369                                         CN23XX_SLI_IQ_PKT_CONTROL64(q_no));
 370                 while ((READ_ONCE(reg_val) & CN23XX_PKT_INPUT_CTL_RST) &&
 371                        !(READ_ONCE(reg_val) & CN23XX_PKT_INPUT_CTL_QUIET) &&
 372                        loop--) {
 373                         WRITE_ONCE(reg_val, octeon_read_csr64(
 374                             oct, CN23XX_SLI_IQ_PKT_CONTROL64(q_no)));
 375                 }
 376                 if (!loop) {
 377                         dev_err(&oct->pci_dev->dev,
 378                                 "clearing the reset reg failed or setting the quiet reg failed for qno: %u\n",
 379                                 q_no);
 380                         return -1;
 381                 }
 382                 WRITE_ONCE(reg_val, READ_ONCE(reg_val) &
 383                         ~CN23XX_PKT_INPUT_CTL_RST);
 384                 octeon_write_csr64(oct, CN23XX_SLI_IQ_PKT_CONTROL64(q_no),
 385                                    READ_ONCE(reg_val));
 386
 387                 WRITE_ONCE(reg_val, octeon_read_csr64(
 388                            oct, CN23XX_SLI_IQ_PKT_CONTROL64(q_no)));
 389                 if (READ_ONCE(reg_val) & CN23XX_PKT_INPUT_CTL_RST) {
 390                         dev_err(&oct->pci_dev->dev,
 391                                 "clearing the reset failed for qno: %u\n",
 392                                 q_no);
 393                         ret_val = -1;
 394                 }
 395         }
 396
 397         return ret_val;
 398 }
 399
 400 static int cn23xx_pf_setup_global_input_regs(struct octeon_device *oct)
 401 {
 402         u32 q_no, ern, srn;
 403         u64 pf_num;
 404         u64 intr_threshold, reg_val;
 405         struct octeon_instr_queue *iq;
 406         struct octeon_cn23xx_pf *cn23xx = (struct octeon_cn23xx_pf *)oct->chip;
 407
 408         pf_num = oct->pf_num;
 409
 410         srn = oct->sriov_info.pf_srn;
 411         ern = srn + oct->sriov_info.num_pf_rings;
 412
 413         if (cn23xx_reset_io_queues(oct))
 414                 return -1;
 415
 416         /** Set the MAC_NUM and PVF_NUM in IQ_PKT_CONTROL reg
 417         * for all queues.Only PF can set these bits.
 418         * bits 29:30 indicate the MAC num.
 419         * bits 32:47 indicate the PVF num.
 420         */
 421         for (q_no = 0; q_no < ern; q_no++) {
 422                 reg_val = oct->pcie_port << CN23XX_PKT_INPUT_CTL_MAC_NUM_POS;
 423                 reg_val |= pf_num << CN23XX_PKT_INPUT_CTL_PF_NUM_POS;
 424
 425                 octeon_write_csr64(oct, CN23XX_SLI_IQ_PKT_CONTROL64(q_no),
 426                                    reg_val);
 427         }
 428
 429         /* Select ES, RO, NS, RDSIZE,DPTR Fomat#0 for
 430          * pf queues
 431          */
 432         for (q_no = srn; q_no < ern; q_no++) {
 433                 void __iomem *inst_cnt_reg;
 434
 435                 iq = oct->instr_queue[q_no];
 436                 if (iq)
 437                         inst_cnt_reg = iq->inst_cnt_reg;
 438                 else
 439                         inst_cnt_reg = (u8 *)oct->mmio[0].hw_addr +
 440                                        CN23XX_SLI_IQ_INSTR_COUNT64(q_no);
 441
 442                 reg_val =
 443                     octeon_read_csr64(oct, CN23XX_SLI_IQ_PKT_CONTROL64(q_no));
 444
 445                 reg_val |= CN23XX_PKT_INPUT_CTL_MASK;
 446
 447                 octeon_write_csr64(oct, CN23XX_SLI_IQ_PKT_CONTROL64(q_no),
 448                                    reg_val);
 449
 450                 /* Set WMARK level for triggering PI_INT */
 451                 /* intr_threshold = CN23XX_DEF_IQ_INTR_THRESHOLD & */
 452                 intr_threshold = CFG_GET_IQ_INTR_PKT(cn23xx->conf) &
 453                                  CN23XX_PKT_IN_DONE_WMARK_MASK;
 454
 455                 writeq((readq(inst_cnt_reg) &
 456                         ~(CN23XX_PKT_IN_DONE_WMARK_MASK <<
 457                           CN23XX_PKT_IN_DONE_WMARK_BIT_POS)) |
 458                        (intr_threshold << CN23XX_PKT_IN_DONE_WMARK_BIT_POS),
 459                        inst_cnt_reg);
 460         }
 461         return 0;
 462 }
 463
 464 static void cn23xx_pf_setup_global_output_regs(struct octeon_device *oct)
 465 {
 466         u32 reg_val;
 467         u32 q_no, ern, srn;
 468         u64 time_threshold;
 469
 470         struct octeon_cn23xx_pf *cn23xx = (struct octeon_cn23xx_pf *)oct->chip;
 471
 472         srn = oct->sriov_info.pf_srn;
 473         ern = srn + oct->sriov_info.num_pf_rings;
 474
 475         if (CFG_GET_IS_SLI_BP_ON(cn23xx->conf)) {
 476                 octeon_write_csr64(oct, CN23XX_SLI_OQ_WMARK, 32);
 477         } else {
 478                 /** Set Output queue watermark to 0 to disable backpressure */
 479                 octeon_write_csr64(oct, CN23XX_SLI_OQ_WMARK, 0);
 480         }
 481
 482         for (q_no = srn; q_no < ern; q_no++) {
 483                 reg_val = octeon_read_csr(oct, CN23XX_SLI_OQ_PKT_CONTROL(q_no));
 484
 485                 /* set IPTR & DPTR */
 486                 reg_val |=
 487                     (CN23XX_PKT_OUTPUT_CTL_IPTR | CN23XX_PKT_OUTPUT_CTL_DPTR);
 488
 489                 /* reset BMODE */
 490                 reg_val &= ~(CN23XX_PKT_OUTPUT_CTL_BMODE);
 491
 492                 /* No Relaxed Ordering, No Snoop, 64-bit Byte swap
 493                  * for Output Queue ScatterList
 494                  * reset ROR_P, NSR_P
 495                  */
 496                 reg_val &= ~(CN23XX_PKT_OUTPUT_CTL_ROR_P);
 497                 reg_val &= ~(CN23XX_PKT_OUTPUT_CTL_NSR_P);
 498
 499 #ifdef __LITTLE_ENDIAN_BITFIELD
 500                 reg_val &= ~(CN23XX_PKT_OUTPUT_CTL_ES_P);
 501 #else
 502                 reg_val |= (CN23XX_PKT_OUTPUT_CTL_ES_P);
 503 #endif
 504                 /* No Relaxed Ordering, No Snoop, 64-bit Byte swap
 505                  * for Output Queue Data
 506                  * reset ROR, NSR
 507                  */
 508                 reg_val &= ~(CN23XX_PKT_OUTPUT_CTL_ROR);
 509                 reg_val &= ~(CN23XX_PKT_OUTPUT_CTL_NSR);
 510                 /* set the ES bit */
 511                 reg_val |= (CN23XX_PKT_OUTPUT_CTL_ES);
 512
 513                 /* write all the selected settings */
 514                 octeon_write_csr(oct, CN23XX_SLI_OQ_PKT_CONTROL(q_no), reg_val);
 515
 516                 /* Enabling these interrupt in oct->fn_list.enable_interrupt()
 517                  * routine which called after IOQ init.
 518                  * Set up interrupt packet and time thresholds
 519                  * for all the OQs
 520                  */
 521                 time_threshold = cn23xx_pf_get_oq_ticks(
 522                     oct, (u32)CFG_GET_OQ_INTR_TIME(cn23xx->conf));
 523
 524                 octeon_write_csr64(oct, CN23XX_SLI_OQ_PKT_INT_LEVELS(q_no),
 525                                    (CFG_GET_OQ_INTR_PKT(cn23xx->conf) |
 526                                     (time_threshold << 32)));
 527         }
 528
 529         /** Setting the water mark level for pko back pressure **/
 530         writeq(0x40, (u8 *)oct->mmio[0].hw_addr + CN23XX_SLI_OQ_WMARK);
 531
 532         /** Disabling setting OQs in reset when ring has no dorebells
 533           * enabling this will cause of head of line blocking
 534           */
 535         /* Do it only for pass1.1. and pass1.2 */
 536         if ((oct->rev_id == OCTEON_CN23XX_REV_1_0) ||
 537             (oct->rev_id == OCTEON_CN23XX_REV_1_1))
 538                 writeq(readq((u8 *)oct->mmio[0].hw_addr +
 539                                      CN23XX_SLI_GBL_CONTROL) | 0x2,
 540                        (u8 *)oct->mmio[0].hw_addr + CN23XX_SLI_GBL_CONTROL);
 541
 542         /** Enable channel-level backpressure */
 543         if (oct->pf_num)
 544                 writeq(0xffffffffffffffffULL,
 545                        (u8 *)oct->mmio[0].hw_addr + CN23XX_SLI_OUT_BP_EN2_W1S);
 546         else
 547                 writeq(0xffffffffffffffffULL,
 548                        (u8 *)oct->mmio[0].hw_addr + CN23XX_SLI_OUT_BP_EN_W1S);
 549 }
 550
 551 static int cn23xx_setup_pf_device_regs(struct octeon_device *oct)
 552 {
 553         cn23xx_enable_error_reporting(oct);
 554
 555         /* program the MAC(0..3)_RINFO before setting up input/output regs */
 556         cn23xx_setup_global_mac_regs(oct);
 557
 558         if (cn23xx_pf_setup_global_input_regs(oct))
 559                 return -1;
 560
 561         cn23xx_pf_setup_global_output_regs(oct);
 562
 563         /* Default error timeout value should be 0x200000 to avoid host hang
 564          * when reads invalid register
 565          */
 566         octeon_write_csr64(oct, CN23XX_SLI_WINDOW_CTL,
 567                            CN23XX_SLI_WINDOW_CTL_DEFAULT);
 568
 569         /* set SLI_PKT_IN_JABBER to handle large VXLAN packets */
 570         octeon_write_csr64(oct, CN23XX_SLI_PKT_IN_JABBER, CN23XX_INPUT_JABBER);
 571         return 0;
 572 }
 573
 574 static void cn23xx_setup_iq_regs(struct octeon_device *oct, u32 iq_no)
 575 {
 576         struct octeon_instr_queue *iq = oct->instr_queue[iq_no];
 577         u64 pkt_in_done;
 578
 579         iq_no += oct->sriov_info.pf_srn;
 580
 581         /* Write the start of the input queue's ring and its size  */
 582         octeon_write_csr64(oct, CN23XX_SLI_IQ_BASE_ADDR64(iq_no),
 583                            iq->base_addr_dma);
 584         octeon_write_csr(oct, CN23XX_SLI_IQ_SIZE(iq_no), iq->max_count);
 585
 586         /* Remember the doorbell & instruction count register addr
 587          * for this queue
 588          */
 589         iq->doorbell_reg =
 590             (u8 *)oct->mmio[0].hw_addr + CN23XX_SLI_IQ_DOORBELL(iq_no);
 591         iq->inst_cnt_reg =
 592             (u8 *)oct->mmio[0].hw_addr + CN23XX_SLI_IQ_INSTR_COUNT64(iq_no);
 593         dev_dbg(&oct->pci_dev->dev, "InstQ[%d]:dbell reg @ 0x%p instcnt_reg @ 0x%p\n",
 594                 iq_no, iq->doorbell_reg, iq->inst_cnt_reg);
 595
 596         /* Store the current instruction counter (used in flush_iq
 597          * calculation)
 598          */
 599         pkt_in_done = readq(iq->inst_cnt_reg);
 600
 601         if (oct->msix_on) {
 602                 /* Set CINT_ENB to enable IQ interrupt   */
 603                 writeq((pkt_in_done | CN23XX_INTR_CINT_ENB),
 604                        iq->inst_cnt_reg);
 605         } else {
 606                 /* Clear the count by writing back what we read, but don't
 607                  * enable interrupts
 608                  */
 609                 writeq(pkt_in_done, iq->inst_cnt_reg);
 610         }
 611
 612         iq->reset_instr_cnt = 0;
 613 }
 614
 615 static void cn23xx_setup_oq_regs(struct octeon_device *oct, u32 oq_no)
 616 {
 617         u32 reg_val;
 618         struct octeon_droq *droq = oct->droq[oq_no];
 619         struct octeon_cn23xx_pf *cn23xx = (struct octeon_cn23xx_pf *)oct->chip;
 620         u64 time_threshold;
 621         u64 cnt_threshold;
 622
 623         oq_no += oct->sriov_info.pf_srn;
 624
 625         octeon_write_csr64(oct, CN23XX_SLI_OQ_BASE_ADDR64(oq_no),
 626                            droq->desc_ring_dma);
 627         octeon_write_csr(oct, CN23XX_SLI_OQ_SIZE(oq_no), droq->max_count);
 628
 629         octeon_write_csr(oct, CN23XX_SLI_OQ_BUFF_INFO_SIZE(oq_no),
 630                          (droq->buffer_size | (OCT_RH_SIZE << 16)));
 631
 632         /* Get the mapped address of the pkt_sent and pkts_credit regs */
 633         droq->pkts_sent_reg =
 634             (u8 *)oct->mmio[0].hw_addr + CN23XX_SLI_OQ_PKTS_SENT(oq_no);
 635         droq->pkts_credit_reg =
 636             (u8 *)oct->mmio[0].hw_addr + CN23XX_SLI_OQ_PKTS_CREDIT(oq_no);
 637
 638         if (!oct->msix_on) {
 639                 /* Enable this output queue to generate Packet Timer Interrupt
 640                  */
 641                 reg_val =
 642                     octeon_read_csr(oct, CN23XX_SLI_OQ_PKT_CONTROL(oq_no));
 643                 reg_val |= CN23XX_PKT_OUTPUT_CTL_TENB;
 644                 octeon_write_csr(oct, CN23XX_SLI_OQ_PKT_CONTROL(oq_no),
 645                                  reg_val);
 646
 647                 /* Enable this output queue to generate Packet Count Interrupt
 648                  */
 649                 reg_val =
 650                     octeon_read_csr(oct, CN23XX_SLI_OQ_PKT_CONTROL(oq_no));
 651                 reg_val |= CN23XX_PKT_OUTPUT_CTL_CENB;
 652                 octeon_write_csr(oct, CN23XX_SLI_OQ_PKT_CONTROL(oq_no),
 653                                  reg_val);
 654         } else {
 655                 time_threshold = cn23xx_pf_get_oq_ticks(
 656                     oct, (u32)CFG_GET_OQ_INTR_TIME(cn23xx->conf));
 657                 cnt_threshold = (u32)CFG_GET_OQ_INTR_PKT(cn23xx->conf);
 658
 659                 octeon_write_csr64(
 660                     oct, CN23XX_SLI_OQ_PKT_INT_LEVELS(oq_no),
 661                     ((time_threshold << 32 | cnt_threshold)));
 662         }
 663 }
 664
 665 static int cn23xx_enable_io_queues(struct octeon_device *oct)
 666 {
 667         u64 reg_val;
 668         u32 srn, ern, q_no;
 669         u32 loop = 1000;
 670
 671         srn = oct->sriov_info.pf_srn;
 672         ern = srn + oct->num_iqs;
 673
 674         for (q_no = srn; q_no < ern; q_no++) {
 675                 /* set the corresponding IQ IS_64B bit */
 676                 if (oct->io_qmask.iq64B & BIT_ULL(q_no - srn)) {
 677                         reg_val = octeon_read_csr64(
 678                             oct, CN23XX_SLI_IQ_PKT_CONTROL64(q_no));
 679                         reg_val = reg_val | CN23XX_PKT_INPUT_CTL_IS_64B;
 680                         octeon_write_csr64(
 681                             oct, CN23XX_SLI_IQ_PKT_CONTROL64(q_no), reg_val);
 682                 }
 683
 684                 /* set the corresponding IQ ENB bit */
 685                 if (oct->io_qmask.iq & BIT_ULL(q_no - srn)) {
 686                         /* IOQs are in reset by default in PEM2 mode,
 687                          * clearing reset bit
 688                          */
 689                         reg_val = octeon_read_csr64(
 690                             oct, CN23XX_SLI_IQ_PKT_CONTROL64(q_no));
 691
 692                         if (reg_val & CN23XX_PKT_INPUT_CTL_RST) {
 693                                 while ((reg_val & CN23XX_PKT_INPUT_CTL_RST) &&
 694                                        !(reg_val &
 695                                          CN23XX_PKT_INPUT_CTL_QUIET) &&
 696                                        --loop) {
 697                                         reg_val = octeon_read_csr64(
 698                                             oct,
 699                                             CN23XX_SLI_IQ_PKT_CONTROL64(q_no));
 700                                 }
 701                                 if (!loop) {
 702                                         dev_err(&oct->pci_dev->dev,
 703                                                 "clearing the reset reg failed or setting the quiet reg failed for qno: %u\n",
 704                                                 q_no);
 705                                         return -1;
 706                                 }
 707                                 reg_val = reg_val & ~CN23XX_PKT_INPUT_CTL_RST;
 708                                 octeon_write_csr64(
 709                                     oct, CN23XX_SLI_IQ_PKT_CONTROL64(q_no),
 710                                     reg_val);
 711
 712                                 reg_val = octeon_read_csr64(
 713                                     oct, CN23XX_SLI_IQ_PKT_CONTROL64(q_no));
 714                                 if (reg_val & CN23XX_PKT_INPUT_CTL_RST) {
 715                                         dev_err(&oct->pci_dev->dev,
 716                                                 "clearing the reset failed for qno: %u\n",
 717                                                 q_no);
 718                                         return -1;
 719                                 }
 720                         }
 721                         reg_val = octeon_read_csr64(
 722                             oct, CN23XX_SLI_IQ_PKT_CONTROL64(q_no));
 723                         reg_val = reg_val | CN23XX_PKT_INPUT_CTL_RING_ENB;
 724                         octeon_write_csr64(
 725                             oct, CN23XX_SLI_IQ_PKT_CONTROL64(q_no), reg_val);
 726                 }
 727         }
 728         for (q_no = srn; q_no < ern; q_no++) {
 729                 u32 reg_val;
 730                 /* set the corresponding OQ ENB bit */
 731                 if (oct->io_qmask.oq & BIT_ULL(q_no - srn)) {
 732                         reg_val = octeon_read_csr(
 733                             oct, CN23XX_SLI_OQ_PKT_CONTROL(q_no));
 734                         reg_val = reg_val | CN23XX_PKT_OUTPUT_CTL_RING_ENB;
 735                         octeon_write_csr(oct, CN23XX_SLI_OQ_PKT_CONTROL(q_no),
 736                                          reg_val);
 737                 }
 738         }
 739         return 0;
 740 }
 741
 742 static void cn23xx_disable_io_queues(struct octeon_device *oct)
 743 {
 744         int q_no, loop;
 745         u64 d64;
 746         u32 d32;
 747         u32 srn, ern;
 748
 749         srn = oct->sriov_info.pf_srn;
 750         ern = srn + oct->num_iqs;
 751
 752         /*** Disable Input Queues. ***/
 753         for (q_no = srn; q_no < ern; q_no++) {
 754                 loop = HZ;
 755
 756                 /* start the Reset for a particular ring */
 757                 WRITE_ONCE(d64, octeon_read_csr64(
 758                            oct, CN23XX_SLI_IQ_PKT_CONTROL64(q_no)));
 759                 WRITE_ONCE(d64, READ_ONCE(d64) &
 760                                         (~(CN23XX_PKT_INPUT_CTL_RING_ENB)));
 761                 WRITE_ONCE(d64, READ_ONCE(d64) | CN23XX_PKT_INPUT_CTL_RST);
 762                 octeon_write_csr64(oct, CN23XX_SLI_IQ_PKT_CONTROL64(q_no),
 763                                    READ_ONCE(d64));
 764
 765                 /* Wait until hardware indicates that the particular IQ
 766                  * is out of reset.
 767                  */
 768                 WRITE_ONCE(d64, octeon_read_csr64(
 769                                         oct, CN23XX_SLI_PKT_IOQ_RING_RST));
 770                 while (!(READ_ONCE(d64) & BIT_ULL(q_no)) && loop--) {
 771                         WRITE_ONCE(d64, octeon_read_csr64(
 772                                         oct, CN23XX_SLI_PKT_IOQ_RING_RST));
 773                         schedule_timeout_uninterruptible(1);
 774                 }
 775
 776                 /* Reset the doorbell register for this Input Queue. */
 777                 octeon_write_csr(oct, CN23XX_SLI_IQ_DOORBELL(q_no), 0xFFFFFFFF);
 778                 while (octeon_read_csr64(oct, CN23XX_SLI_IQ_DOORBELL(q_no)) &&
 779                        loop--) {
 780                         schedule_timeout_uninterruptible(1);
 781                 }
 782         }
 783
 784         /*** Disable Output Queues. ***/
 785         for (q_no = srn; q_no < ern; q_no++) {
 786                 loop = HZ;
 787
 788                 /* Wait until hardware indicates that the particular IQ
 789                  * is out of reset.It given that SLI_PKT_RING_RST is
 790                  * common for both IQs and OQs
 791                  */
 792                 WRITE_ONCE(d64, octeon_read_csr64(
 793                                         oct, CN23XX_SLI_PKT_IOQ_RING_RST));
 794                 while (!(READ_ONCE(d64) & BIT_ULL(q_no)) && loop--) {
 795                         WRITE_ONCE(d64, octeon_read_csr64(
 796                                         oct, CN23XX_SLI_PKT_IOQ_RING_RST));
 797                         schedule_timeout_uninterruptible(1);
 798                 }
 799
 800                 /* Reset the doorbell register for this Output Queue. */
 801                 octeon_write_csr(oct, CN23XX_SLI_OQ_PKTS_CREDIT(q_no),
 802                                  0xFFFFFFFF);
 803                 while (octeon_read_csr64(oct,
 804                                          CN23XX_SLI_OQ_PKTS_CREDIT(q_no)) &&
 805                        loop--) {
 806                         schedule_timeout_uninterruptible(1);
 807                 }
 808
 809                 /* clear the SLI_PKT(0..63)_CNTS[CNT] reg value */
 810                 WRITE_ONCE(d32, octeon_read_csr(
 811                                         oct, CN23XX_SLI_OQ_PKTS_SENT(q_no)));
 812                 octeon_write_csr(oct, CN23XX_SLI_OQ_PKTS_SENT(q_no),
 813                                  READ_ONCE(d32));
 814         }
 815 }
 816
 817 static u64 cn23xx_pf_msix_interrupt_handler(void *dev)
 818 {
 819         struct octeon_ioq_vector *ioq_vector = (struct octeon_ioq_vector *)dev;
 820         struct octeon_device *oct = ioq_vector->oct_dev;
 821         u64 pkts_sent;
 822         u64 ret = 0;
 823         struct octeon_droq *droq = oct->droq[ioq_vector->droq_index];
 824
 825         dev_dbg(&oct->pci_dev->dev, "In %s octeon_dev @ %p\n", __func__, oct);
 826
 827         if (!droq) {
 828                 dev_err(&oct->pci_dev->dev, "23XX bringup FIXME: oct pfnum:%d ioq_vector->ioq_num :%d droq is NULL\n",
 829                         oct->pf_num, ioq_vector->ioq_num);
 830                 return 0;
 831         }
 832
 833         pkts_sent = readq(droq->pkts_sent_reg);
 834
 835         /* If our device has interrupted, then proceed. Also check
 836          * for all f's if interrupt was triggered on an error
 837          * and the PCI read fails.
 838          */
 839         if (!pkts_sent || (pkts_sent == 0xFFFFFFFFFFFFFFFFULL))
 840                 return ret;
 841
 842         /* Write count reg in sli_pkt_cnts to clear these int.*/
 843         if ((pkts_sent & CN23XX_INTR_PO_INT) ||
 844             (pkts_sent & CN23XX_INTR_PI_INT)) {
 845                 if (pkts_sent & CN23XX_INTR_PO_INT)
 846                         ret |= MSIX_PO_INT;
 847         }
 848
 849         if (pkts_sent & CN23XX_INTR_PI_INT)
 850                 /* We will clear the count when we update the read_index. */
 851                 ret |= MSIX_PI_INT;
 852
 853         /* Never need to handle msix mbox intr for pf. They arrive on the last
 854          * msix
 855          */
 856         return ret;
 857 }
 858
 859 static irqreturn_t cn23xx_interrupt_handler(void *dev)
 860 {
 861         struct octeon_device *oct = (struct octeon_device *)dev;
 862         struct octeon_cn23xx_pf *cn23xx = (struct octeon_cn23xx_pf *)oct->chip;
 863         u64 intr64;
 864
 865         dev_dbg(&oct->pci_dev->dev, "In %s octeon_dev @ %p\n", __func__, oct);
 866         intr64 = readq(cn23xx->intr_sum_reg64);
 867
 868         oct->int_status = 0;
 869
 870         if (intr64 & CN23XX_INTR_ERR)
 871                 dev_err(&oct->pci_dev->dev, "OCTEON[%d]: Error Intr: 0x%016llx\n",
 872                         oct->octeon_id, CVM_CAST64(intr64));
 873
 874         if (oct->msix_on != LIO_FLAG_MSIX_ENABLED) {
 875                 if (intr64 & CN23XX_INTR_PKT_DATA)
 876                         oct->int_status |= OCT_DEV_INTR_PKT_DATA;
 877         }
 878
 879         if (intr64 & (CN23XX_INTR_DMA0_FORCE))
 880                 oct->int_status |= OCT_DEV_INTR_DMA0_FORCE;
 881         if (intr64 & (CN23XX_INTR_DMA1_FORCE))
 882                 oct->int_status |= OCT_DEV_INTR_DMA1_FORCE;
 883
 884         /* Clear the current interrupts */
 885         writeq(intr64, cn23xx->intr_sum_reg64);
 886
 887         return IRQ_HANDLED;
 888 }
 889
 890 static void cn23xx_bar1_idx_setup(struct octeon_device *oct, u64 core_addr,
 891                                   u32 idx, int valid)
 892 {
 893         u64 bar1;
 894         u64 reg_adr;
 895
 896         if (!valid) {
 897                 reg_adr = lio_pci_readq(
 898                         oct, CN23XX_PEM_BAR1_INDEX_REG(oct->pcie_port, idx));
 899                 WRITE_ONCE(bar1, reg_adr);
 900                 lio_pci_writeq(oct, (READ_ONCE(bar1) & 0xFFFFFFFEULL),
 901                                CN23XX_PEM_BAR1_INDEX_REG(oct->pcie_port, idx));
 902                 reg_adr = lio_pci_readq(
 903                         oct, CN23XX_PEM_BAR1_INDEX_REG(oct->pcie_port, idx));
 904                 WRITE_ONCE(bar1, reg_adr);
 905                 return;
 906         }
 907
 908         /*  The PEM(0..3)_BAR1_INDEX(0..15)[ADDR_IDX]<23:4> stores
 909          *  bits <41:22> of the Core Addr
 910          */
 911         lio_pci_writeq(oct, (((core_addr >> 22) << 4) | PCI_BAR1_MASK),
 912                        CN23XX_PEM_BAR1_INDEX_REG(oct->pcie_port, idx));
 913
 914         WRITE_ONCE(bar1, lio_pci_readq(
 915                    oct, CN23XX_PEM_BAR1_INDEX_REG(oct->pcie_port, idx)));
 916 }
 917
 918 static void cn23xx_bar1_idx_write(struct octeon_device *oct, u32 idx, u32 mask)
 919 {
 920         lio_pci_writeq(oct, mask,
 921                        CN23XX_PEM_BAR1_INDEX_REG(oct->pcie_port, idx));
 922 }
 923
 924 static u32 cn23xx_bar1_idx_read(struct octeon_device *oct, u32 idx)
 925 {
 926         return (u32)lio_pci_readq(
 927             oct, CN23XX_PEM_BAR1_INDEX_REG(oct->pcie_port, idx));
 928 }
 929
 930 /* always call with lock held */
 931 static u32 cn23xx_update_read_index(struct octeon_instr_queue *iq)
 932 {
 933         u32 new_idx;
 934         u32 last_done;
 935         u32 pkt_in_done = readl(iq->inst_cnt_reg);
 936
 937         last_done = pkt_in_done - iq->pkt_in_done;
 938         iq->pkt_in_done = pkt_in_done;
 939
 940         /* Modulo of the new index with the IQ size will give us
 941          * the new index.  The iq->reset_instr_cnt is always zero for
 942          * cn23xx, so no extra adjustments are needed.
 943          */
 944         new_idx = (iq->octeon_read_index +
 945                    (u32)(last_done & CN23XX_PKT_IN_DONE_CNT_MASK)) %
 946                   iq->max_count;
 947
 948         return new_idx;
 949 }
 950
 951 static void cn23xx_enable_pf_interrupt(struct octeon_device *oct, u8 intr_flag)
 952 {
 953         struct octeon_cn23xx_pf *cn23xx = (struct octeon_cn23xx_pf *)oct->chip;
 954         u64 intr_val = 0;
 955
 956         /*  Divide the single write to multiple writes based on the flag. */
 957         /* Enable Interrupt */
 958         if (intr_flag == OCTEON_ALL_INTR) {
 959                 writeq(cn23xx->intr_mask64, cn23xx->intr_enb_reg64);
 960         } else if (intr_flag & OCTEON_OUTPUT_INTR) {
 961                 intr_val = readq(cn23xx->intr_enb_reg64);
 962                 intr_val |= CN23XX_INTR_PKT_DATA;
 963                 writeq(intr_val, cn23xx->intr_enb_reg64);
 964         }
 965 }
 966
 967 static void cn23xx_disable_pf_interrupt(struct octeon_device *oct, u8 intr_flag)
 968 {
 969         struct octeon_cn23xx_pf *cn23xx = (struct octeon_cn23xx_pf *)oct->chip;
 970         u64 intr_val = 0;
 971
 972         /* Disable Interrupts */
 973         if (intr_flag == OCTEON_ALL_INTR) {
 974                 writeq(0, cn23xx->intr_enb_reg64);
 975         } else if (intr_flag & OCTEON_OUTPUT_INTR) {
 976                 intr_val = readq(cn23xx->intr_enb_reg64);
 977                 intr_val &= ~CN23XX_INTR_PKT_DATA;
 978                 writeq(intr_val, cn23xx->intr_enb_reg64);
 979         }
 980 }
 981
 982 static void cn23xx_get_pcie_qlmport(struct octeon_device *oct)
 983 {
 984         oct->pcie_port = (octeon_read_csr(oct, CN23XX_SLI_MAC_NUMBER)) & 0xff;
 985
 986         dev_dbg(&oct->pci_dev->dev, "OCTEON: CN23xx uses PCIE Port %d\n",
 987                 oct->pcie_port);
 988 }
 989
 990 static void cn23xx_get_pf_num(struct octeon_device *oct)
 991 {
 992         u32 fdl_bit = 0;
 993
 994         /** Read Function Dependency Link reg to get the function number */
 995         pci_read_config_dword(oct->pci_dev, CN23XX_PCIE_SRIOV_FDL, &fdl_bit);
 996         oct->pf_num = ((fdl_bit >> CN23XX_PCIE_SRIOV_FDL_BIT_POS) &
 997                        CN23XX_PCIE_SRIOV_FDL_MASK);
 998 }
 999
1000 static void cn23xx_setup_reg_address(struct octeon_device *oct)
1001 {
1002         u8 __iomem *bar0_pciaddr = oct->mmio[0].hw_addr;
1003         struct octeon_cn23xx_pf *cn23xx = (struct octeon_cn23xx_pf *)oct->chip;
1004
1005         oct->reg_list.pci_win_wr_addr_hi =
1006             (u32 __iomem *)(bar0_pciaddr + CN23XX_WIN_WR_ADDR_HI);
1007         oct->reg_list.pci_win_wr_addr_lo =
1008             (u32 __iomem *)(bar0_pciaddr + CN23XX_WIN_WR_ADDR_LO);
1009         oct->reg_list.pci_win_wr_addr =
1010             (u64 __iomem *)(bar0_pciaddr + CN23XX_WIN_WR_ADDR64);
1011
1012         oct->reg_list.pci_win_rd_addr_hi =
1013             (u32 __iomem *)(bar0_pciaddr + CN23XX_WIN_RD_ADDR_HI);
1014         oct->reg_list.pci_win_rd_addr_lo =
1015             (u32 __iomem *)(bar0_pciaddr + CN23XX_WIN_RD_ADDR_LO);
1016         oct->reg_list.pci_win_rd_addr =
1017             (u64 __iomem *)(bar0_pciaddr + CN23XX_WIN_RD_ADDR64);
1018
1019         oct->reg_list.pci_win_wr_data_hi =
1020             (u32 __iomem *)(bar0_pciaddr + CN23XX_WIN_WR_DATA_HI);
1021         oct->reg_list.pci_win_wr_data_lo =
1022             (u32 __iomem *)(bar0_pciaddr + CN23XX_WIN_WR_DATA_LO);
1023         oct->reg_list.pci_win_wr_data =
1024             (u64 __iomem *)(bar0_pciaddr + CN23XX_WIN_WR_DATA64);
1025
1026         oct->reg_list.pci_win_rd_data_hi =
1027             (u32 __iomem *)(bar0_pciaddr + CN23XX_WIN_RD_DATA_HI);
1028         oct->reg_list.pci_win_rd_data_lo =
1029             (u32 __iomem *)(bar0_pciaddr + CN23XX_WIN_RD_DATA_LO);
1030         oct->reg_list.pci_win_rd_data =
1031             (u64 __iomem *)(bar0_pciaddr + CN23XX_WIN_RD_DATA64);
1032
1033         cn23xx_get_pcie_qlmport(oct);
1034
1035         cn23xx->intr_mask64 = CN23XX_INTR_MASK;
1036         if (!oct->msix_on)
1037                 cn23xx->intr_mask64 |= CN23XX_INTR_PKT_TIME;
1038         if (oct->rev_id >= OCTEON_CN23XX_REV_1_1)
1039                 cn23xx->intr_mask64 |= CN23XX_INTR_VF_MBOX;
1040
1041         cn23xx->intr_sum_reg64 =
1042             bar0_pciaddr +
1043             CN23XX_SLI_MAC_PF_INT_SUM64(oct->pcie_port, oct->pf_num);
1044         cn23xx->intr_enb_reg64 =
1045             bar0_pciaddr +
1046             CN23XX_SLI_MAC_PF_INT_ENB64(oct->pcie_port, oct->pf_num);
1047 }
1048
1049 static int cn23xx_sriov_config(struct octeon_device *oct)
1050 {
1051         u32 total_rings;
1052         struct octeon_cn23xx_pf *cn23xx = (struct octeon_cn23xx_pf *)oct->chip;
1053         /* num_vfs is already filled for us */
1054         u32 pf_srn, num_pf_rings;
1055
1056         cn23xx->conf =
1057             (struct octeon_config *)oct_get_config_info(oct, LIO_23XX);
1058         switch (oct->rev_id) {
1059         case OCTEON_CN23XX_REV_1_0:
1060                 total_rings = CN23XX_MAX_RINGS_PER_PF_PASS_1_0;
1061                 break;
1062         case OCTEON_CN23XX_REV_1_1:
1063                 total_rings = CN23XX_MAX_RINGS_PER_PF_PASS_1_1;
1064                 break;
1065         default:
1066                 total_rings = CN23XX_MAX_RINGS_PER_PF;
1067                 break;
1068         }
1069         if (!oct->sriov_info.num_pf_rings) {
1070                 if (total_rings > num_present_cpus())
1071                         num_pf_rings = num_present_cpus();
1072                 else
1073                         num_pf_rings = total_rings;
1074         } else {
1075                 num_pf_rings = oct->sriov_info.num_pf_rings;
1076
1077                 if (num_pf_rings > total_rings) {
1078                         dev_warn(&oct->pci_dev->dev,
1079                                  "num_queues_per_pf requested %u is more than available rings. Reducing to %u\n",
1080                                  num_pf_rings, total_rings);
1081                         num_pf_rings = total_rings;
1082                 }
1083         }
1084
1085         total_rings = num_pf_rings;
1086         /* the first ring of the pf */
1087         pf_srn = total_rings - num_pf_rings;
1088
1089         oct->sriov_info.trs = total_rings;
1090         oct->sriov_info.pf_srn = pf_srn;
1091         oct->sriov_info.num_pf_rings = num_pf_rings;
1092         dev_dbg(&oct->pci_dev->dev, "trs:%d pf_srn:%d num_pf_rings:%d\n",
1093                 oct->sriov_info.trs, oct->sriov_info.pf_srn,
1094                 oct->sriov_info.num_pf_rings);
1095         return 0;
1096 }
1097
1098 int setup_cn23xx_octeon_pf_device(struct octeon_device *oct)
1099 {
1100         if (octeon_map_pci_barx(oct, 0, 0))
1101                 return 1;
1102
1103         if (octeon_map_pci_barx(oct, 1, MAX_BAR1_IOREMAP_SIZE)) {
1104                 dev_err(&oct->pci_dev->dev, "%s CN23XX BAR1 map failed\n",
1105                         __func__);
1106                 octeon_unmap_pci_barx(oct, 0);
1107                 return 1;
1108         }
1109
1110         cn23xx_get_pf_num(oct);
1111
1112         if (cn23xx_sriov_config(oct)) {
1113                 octeon_unmap_pci_barx(oct, 0);
1114                 octeon_unmap_pci_barx(oct, 1);
1115                 return 1;
1116         }
1117
1118         octeon_write_csr64(oct, CN23XX_SLI_MAC_CREDIT_CNT, 0x3F802080802080ULL);
1119
1120         oct->fn_list.setup_iq_regs = cn23xx_setup_iq_regs;
1121         oct->fn_list.setup_oq_regs = cn23xx_setup_oq_regs;
1122         oct->fn_list.process_interrupt_regs = cn23xx_interrupt_handler;
1123         oct->fn_list.msix_interrupt_handler = cn23xx_pf_msix_interrupt_handler;
1124
1125         oct->fn_list.soft_reset = cn23xx_pf_soft_reset;
1126         oct->fn_list.setup_device_regs = cn23xx_setup_pf_device_regs;
1127         oct->fn_list.update_iq_read_idx = cn23xx_update_read_index;
1128
1129         oct->fn_list.bar1_idx_setup = cn23xx_bar1_idx_setup;
1130         oct->fn_list.bar1_idx_write = cn23xx_bar1_idx_write;
1131         oct->fn_list.bar1_idx_read = cn23xx_bar1_idx_read;
1132
1133         oct->fn_list.enable_interrupt = cn23xx_enable_pf_interrupt;
1134         oct->fn_list.disable_interrupt = cn23xx_disable_pf_interrupt;
1135
1136         oct->fn_list.enable_io_queues = cn23xx_enable_io_queues;
1137         oct->fn_list.disable_io_queues = cn23xx_disable_io_queues;
1138
1139         cn23xx_setup_reg_address(oct);
1140
1141         oct->coproc_clock_rate = 1000000ULL * cn23xx_coprocessor_clock(oct);
1142
1143         return 0;
1144 }
1145
1146 int validate_cn23xx_pf_config_info(struct octeon_device *oct,
1147                                    struct octeon_config *conf23xx)
1148 {
1149         if (CFG_GET_IQ_MAX_Q(conf23xx) > CN23XX_MAX_INPUT_QUEUES) {
1150                 dev_err(&oct->pci_dev->dev, "%s: Num IQ (%d) exceeds Max (%d)\n",
1151                         __func__, CFG_GET_IQ_MAX_Q(conf23xx),
1152                         CN23XX_MAX_INPUT_QUEUES);
1153                 return 1;
1154         }
1155
1156         if (CFG_GET_OQ_MAX_Q(conf23xx) > CN23XX_MAX_OUTPUT_QUEUES) {
1157                 dev_err(&oct->pci_dev->dev, "%s: Num OQ (%d) exceeds Max (%d)\n",
1158                         __func__, CFG_GET_OQ_MAX_Q(conf23xx),
1159                         CN23XX_MAX_OUTPUT_QUEUES);
1160                 return 1;
1161         }
1162
1163         if (CFG_GET_IQ_INSTR_TYPE(conf23xx) != OCTEON_32BYTE_INSTR &&
1164             CFG_GET_IQ_INSTR_TYPE(conf23xx) != OCTEON_64BYTE_INSTR) {
1165                 dev_err(&oct->pci_dev->dev, "%s: Invalid instr type for IQ\n",
1166                         __func__);
1167                 return 1;
1168         }
1169
1170         if (!(CFG_GET_OQ_INFO_PTR(conf23xx)) ||
1171             !(CFG_GET_OQ_REFILL_THRESHOLD(conf23xx))) {
1172                 dev_err(&oct->pci_dev->dev, "%s: Invalid parameter for OQ\n",
1173                         __func__);
1174                 return 1;
1175         }
1176
1177         if (!(CFG_GET_OQ_INTR_TIME(conf23xx))) {
1178                 dev_err(&oct->pci_dev->dev, "%s: Invalid parameter for OQ\n",
1179                         __func__);
1180                 return 1;
1181         }
1182
1183         return 0;
1184 }
1185
1186 void cn23xx_dump_iq_regs(struct octeon_device *oct)
1187 {
1188         u32 regval, q_no;
1189
1190         dev_dbg(&oct->pci_dev->dev, "SLI_IQ_DOORBELL_0 [0x%x]: 0x%016llx\n",
1191                 CN23XX_SLI_IQ_DOORBELL(0),
1192                 CVM_CAST64(octeon_read_csr64
1193                         (oct, CN23XX_SLI_IQ_DOORBELL(0))));
1194
1195         dev_dbg(&oct->pci_dev->dev, "SLI_IQ_BASEADDR_0 [0x%x]: 0x%016llx\n",
1196                 CN23XX_SLI_IQ_BASE_ADDR64(0),
1197                 CVM_CAST64(octeon_read_csr64
1198                         (oct, CN23XX_SLI_IQ_BASE_ADDR64(0))));
1199
1200         dev_dbg(&oct->pci_dev->dev, "SLI_IQ_FIFO_RSIZE_0 [0x%x]: 0x%016llx\n",
1201                 CN23XX_SLI_IQ_SIZE(0),
1202                 CVM_CAST64(octeon_read_csr64(oct, CN23XX_SLI_IQ_SIZE(0))));
1203
1204         dev_dbg(&oct->pci_dev->dev, "SLI_CTL_STATUS [0x%x]: 0x%016llx\n",
1205                 CN23XX_SLI_CTL_STATUS,
1206                 CVM_CAST64(octeon_read_csr64(oct, CN23XX_SLI_CTL_STATUS)));
1207
1208         for (q_no = 0; q_no < CN23XX_MAX_INPUT_QUEUES; q_no++) {
1209                 dev_dbg(&oct->pci_dev->dev, "SLI_PKT[%d]_INPUT_CTL [0x%x]: 0x%016llx\n",
1210                         q_no, CN23XX_SLI_IQ_PKT_CONTROL64(q_no),
1211                         CVM_CAST64(octeon_read_csr64
1212                                 (oct,
1213                                         CN23XX_SLI_IQ_PKT_CONTROL64(q_no))));
1214         }
1215
1216         pci_read_config_dword(oct->pci_dev, CN23XX_CONFIG_PCIE_DEVCTL, &regval);
1217         dev_dbg(&oct->pci_dev->dev, "Config DevCtl [0x%x]: 0x%08x\n",
1218                 CN23XX_CONFIG_PCIE_DEVCTL, regval);
1219
1220         dev_dbg(&oct->pci_dev->dev, "SLI_PRT[%d]_CFG [0x%llx]: 0x%016llx\n",
1221                 oct->pcie_port, CN23XX_DPI_SLI_PRTX_CFG(oct->pcie_port),
1222                 CVM_CAST64(lio_pci_readq(
1223                         oct, CN23XX_DPI_SLI_PRTX_CFG(oct->pcie_port))));
1224
1225         dev_dbg(&oct->pci_dev->dev, "SLI_S2M_PORT[%d]_CTL [0x%x]: 0x%016llx\n",
1226                 oct->pcie_port, CN23XX_SLI_S2M_PORTX_CTL(oct->pcie_port),
1227                 CVM_CAST64(octeon_read_csr64(
1228                         oct, CN23XX_SLI_S2M_PORTX_CTL(oct->pcie_port))));
1229 }
1230
1231 int cn23xx_fw_loaded(struct octeon_device *oct)
1232 {
1233         u64 val;
1234
1235         val = octeon_read_csr64(oct, CN23XX_SLI_SCRATCH1);
1236         return (val >> 1) & 1ULL;
1237 }