drivers/infiniband/hw/hns/hns_roce_hw_v1.c

   1 /*
   2  * Copyright (c) 2016 Hisilicon Limited.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32
  33 #include <linux/platform_device.h>
  34 #include <linux/acpi.h>
  35 #include <rdma/ib_umem.h>
  36 #include "hns_roce_common.h"
  37 #include "hns_roce_device.h"
  38 #include "hns_roce_cmd.h"
  39 #include "hns_roce_hem.h"
  40 #include "hns_roce_hw_v1.h"
  41
  42 static void set_data_seg(struct hns_roce_wqe_data_seg *dseg, struct ib_sge *sg)
  43 {
  44         dseg->lkey = cpu_to_le32(sg->lkey);
  45         dseg->addr = cpu_to_le64(sg->addr);
  46         dseg->len  = cpu_to_le32(sg->length);
  47 }
  48
  49 static void set_raddr_seg(struct hns_roce_wqe_raddr_seg *rseg, u64 remote_addr,
  50                           u32 rkey)
  51 {
  52         rseg->raddr = cpu_to_le64(remote_addr);
  53         rseg->rkey  = cpu_to_le32(rkey);
  54         rseg->len   = 0;
  55 }
  56
  57 int hns_roce_v1_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
  58                           struct ib_send_wr **bad_wr)
  59 {
  60         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
  61         struct hns_roce_ah *ah = to_hr_ah(ud_wr(wr)->ah);
  62         struct hns_roce_ud_send_wqe *ud_sq_wqe = NULL;
  63         struct hns_roce_wqe_ctrl_seg *ctrl = NULL;
  64         struct hns_roce_wqe_data_seg *dseg = NULL;
  65         struct hns_roce_qp *qp = to_hr_qp(ibqp);
  66         struct device *dev = &hr_dev->pdev->dev;
  67         struct hns_roce_sq_db sq_db;
  68         int ps_opcode = 0, i = 0;
  69         unsigned long flags = 0;
  70         void *wqe = NULL;
  71         u32 doorbell[2];
  72         int nreq = 0;
  73         u32 ind = 0;
  74         int ret = 0;
  75
  76         if (unlikely(ibqp->qp_type != IB_QPT_GSI &&
  77                 ibqp->qp_type != IB_QPT_RC)) {
  78                 dev_err(dev, "un-supported QP type\n");
  79                 *bad_wr = NULL;
  80                 return -EOPNOTSUPP;
  81         }
  82
  83         spin_lock_irqsave(&qp->sq.lock, flags);
  84         ind = qp->sq_next_wqe;
  85         for (nreq = 0; wr; ++nreq, wr = wr->next) {
  86                 if (hns_roce_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) {
  87                         ret = -ENOMEM;
  88                         *bad_wr = wr;
  89                         goto out;
  90                 }
  91
  92                 if (unlikely(wr->num_sge > qp->sq.max_gs)) {
  93                         dev_err(dev, "num_sge=%d > qp->sq.max_gs=%d\n",
  94                                 wr->num_sge, qp->sq.max_gs);
  95                         ret = -EINVAL;
  96                         *bad_wr = wr;
  97                         goto out;
  98                 }
  99
 100                 wqe = get_send_wqe(qp, ind & (qp->sq.wqe_cnt - 1));
 101                 qp->sq.wrid[(qp->sq.head + nreq) & (qp->sq.wqe_cnt - 1)] =
 102                                                                       wr->wr_id;
 103
 104                 /* Corresponding to the RC and RD type wqe process separately */
 105                 if (ibqp->qp_type == IB_QPT_GSI) {
 106                         ud_sq_wqe = wqe;
 107                         roce_set_field(ud_sq_wqe->dmac_h,
 108                                        UD_SEND_WQE_U32_4_DMAC_0_M,
 109                                        UD_SEND_WQE_U32_4_DMAC_0_S,
 110                                        ah->av.mac[0]);
 111                         roce_set_field(ud_sq_wqe->dmac_h,
 112                                        UD_SEND_WQE_U32_4_DMAC_1_M,
 113                                        UD_SEND_WQE_U32_4_DMAC_1_S,
 114                                        ah->av.mac[1]);
 115                         roce_set_field(ud_sq_wqe->dmac_h,
 116                                        UD_SEND_WQE_U32_4_DMAC_2_M,
 117                                        UD_SEND_WQE_U32_4_DMAC_2_S,
 118                                        ah->av.mac[2]);
 119                         roce_set_field(ud_sq_wqe->dmac_h,
 120                                        UD_SEND_WQE_U32_4_DMAC_3_M,
 121                                        UD_SEND_WQE_U32_4_DMAC_3_S,
 122                                        ah->av.mac[3]);
 123
 124                         roce_set_field(ud_sq_wqe->u32_8,
 125                                        UD_SEND_WQE_U32_8_DMAC_4_M,
 126                                        UD_SEND_WQE_U32_8_DMAC_4_S,
 127                                        ah->av.mac[4]);
 128                         roce_set_field(ud_sq_wqe->u32_8,
 129                                        UD_SEND_WQE_U32_8_DMAC_5_M,
 130                                        UD_SEND_WQE_U32_8_DMAC_5_S,
 131                                        ah->av.mac[5]);
 132                         roce_set_field(ud_sq_wqe->u32_8,
 133                                        UD_SEND_WQE_U32_8_OPERATION_TYPE_M,
 134                                        UD_SEND_WQE_U32_8_OPERATION_TYPE_S,
 135                                        HNS_ROCE_WQE_OPCODE_SEND);
 136                         roce_set_field(ud_sq_wqe->u32_8,
 137                                        UD_SEND_WQE_U32_8_NUMBER_OF_DATA_SEG_M,
 138                                        UD_SEND_WQE_U32_8_NUMBER_OF_DATA_SEG_S,
 139                                        2);
 140                         roce_set_bit(ud_sq_wqe->u32_8,
 141                                 UD_SEND_WQE_U32_8_SEND_GL_ROUTING_HDR_FLAG_S,
 142                                 1);
 143
 144                         ud_sq_wqe->u32_8 |= (wr->send_flags & IB_SEND_SIGNALED ?
 145                                 cpu_to_le32(HNS_ROCE_WQE_CQ_NOTIFY) : 0) |
 146                                 (wr->send_flags & IB_SEND_SOLICITED ?
 147                                 cpu_to_le32(HNS_ROCE_WQE_SE) : 0) |
 148                                 ((wr->opcode == IB_WR_SEND_WITH_IMM) ?
 149                                 cpu_to_le32(HNS_ROCE_WQE_IMM) : 0);
 150
 151                         roce_set_field(ud_sq_wqe->u32_16,
 152                                        UD_SEND_WQE_U32_16_DEST_QP_M,
 153                                        UD_SEND_WQE_U32_16_DEST_QP_S,
 154                                        ud_wr(wr)->remote_qpn);
 155                         roce_set_field(ud_sq_wqe->u32_16,
 156                                        UD_SEND_WQE_U32_16_MAX_STATIC_RATE_M,
 157                                        UD_SEND_WQE_U32_16_MAX_STATIC_RATE_S,
 158                                        ah->av.stat_rate);
 159
 160                         roce_set_field(ud_sq_wqe->u32_36,
 161                                        UD_SEND_WQE_U32_36_FLOW_LABEL_M,
 162                                        UD_SEND_WQE_U32_36_FLOW_LABEL_S, 0);
 163                         roce_set_field(ud_sq_wqe->u32_36,
 164                                        UD_SEND_WQE_U32_36_PRIORITY_M,
 165                                        UD_SEND_WQE_U32_36_PRIORITY_S,
 166                                        ah->av.sl_tclass_flowlabel >>
 167                                        HNS_ROCE_SL_SHIFT);
 168                         roce_set_field(ud_sq_wqe->u32_36,
 169                                        UD_SEND_WQE_U32_36_SGID_INDEX_M,
 170                                        UD_SEND_WQE_U32_36_SGID_INDEX_S,
 171                                        hns_get_gid_index(hr_dev, qp->phy_port,
 172                                                          ah->av.gid_index));
 173
 174                         roce_set_field(ud_sq_wqe->u32_40,
 175                                        UD_SEND_WQE_U32_40_HOP_LIMIT_M,
 176                                        UD_SEND_WQE_U32_40_HOP_LIMIT_S,
 177                                        ah->av.hop_limit);
 178                         roce_set_field(ud_sq_wqe->u32_40,
 179                                        UD_SEND_WQE_U32_40_TRAFFIC_CLASS_M,
 180                                        UD_SEND_WQE_U32_40_TRAFFIC_CLASS_S, 0);
 181
 182                         memcpy(&ud_sq_wqe->dgid[0], &ah->av.dgid[0], GID_LEN);
 183
 184                         ud_sq_wqe->va0_l = (u32)wr->sg_list[0].addr;
 185                         ud_sq_wqe->va0_h = (wr->sg_list[0].addr) >> 32;
 186                         ud_sq_wqe->l_key0 = wr->sg_list[0].lkey;
 187
 188                         ud_sq_wqe->va1_l = (u32)wr->sg_list[1].addr;
 189                         ud_sq_wqe->va1_h = (wr->sg_list[1].addr) >> 32;
 190                         ud_sq_wqe->l_key1 = wr->sg_list[1].lkey;
 191                         ind++;
 192                 } else if (ibqp->qp_type == IB_QPT_RC) {
 193                         ctrl = wqe;
 194                         memset(ctrl, 0, sizeof(struct hns_roce_wqe_ctrl_seg));
 195                         for (i = 0; i < wr->num_sge; i++)
 196                                 ctrl->msg_length += wr->sg_list[i].length;
 197
 198                         ctrl->sgl_pa_h = 0;
 199                         ctrl->flag = 0;
 200                         ctrl->imm_data = send_ieth(wr);
 201
 202                         /*Ctrl field, ctrl set type: sig, solic, imm, fence */
 203                         /* SO wait for conforming application scenarios */
 204                         ctrl->flag |= (wr->send_flags & IB_SEND_SIGNALED ?
 205                                       cpu_to_le32(HNS_ROCE_WQE_CQ_NOTIFY) : 0) |
 206                                       (wr->send_flags & IB_SEND_SOLICITED ?
 207                                       cpu_to_le32(HNS_ROCE_WQE_SE) : 0) |
 208                                       ((wr->opcode == IB_WR_SEND_WITH_IMM ||
 209                                       wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) ?
 210                                       cpu_to_le32(HNS_ROCE_WQE_IMM) : 0) |
 211                                       (wr->send_flags & IB_SEND_FENCE ?
 212                                       (cpu_to_le32(HNS_ROCE_WQE_FENCE)) : 0);
 213
 214                         wqe += sizeof(struct hns_roce_wqe_ctrl_seg);
 215
 216                         switch (wr->opcode) {
 217                         case IB_WR_RDMA_READ:
 218                                 ps_opcode = HNS_ROCE_WQE_OPCODE_RDMA_READ;
 219                                 set_raddr_seg(wqe, atomic_wr(wr)->remote_addr,
 220                                               atomic_wr(wr)->rkey);
 221                                 break;
 222                         case IB_WR_RDMA_WRITE:
 223                         case IB_WR_RDMA_WRITE_WITH_IMM:
 224                                 ps_opcode = HNS_ROCE_WQE_OPCODE_RDMA_WRITE;
 225                                 set_raddr_seg(wqe, atomic_wr(wr)->remote_addr,
 226                                               atomic_wr(wr)->rkey);
 227                                 break;
 228                         case IB_WR_SEND:
 229                         case IB_WR_SEND_WITH_INV:
 230                         case IB_WR_SEND_WITH_IMM:
 231                                 ps_opcode = HNS_ROCE_WQE_OPCODE_SEND;
 232                                 break;
 233                         case IB_WR_LOCAL_INV:
 234                                 break;
 235                         case IB_WR_ATOMIC_CMP_AND_SWP:
 236                         case IB_WR_ATOMIC_FETCH_AND_ADD:
 237                         case IB_WR_LSO:
 238                         default:
 239                                 ps_opcode = HNS_ROCE_WQE_OPCODE_MASK;
 240                                 break;
 241                         }
 242                         ctrl->flag |= cpu_to_le32(ps_opcode);
 243                         wqe += sizeof(struct hns_roce_wqe_raddr_seg);
 244
 245                         dseg = wqe;
 246                         if (wr->send_flags & IB_SEND_INLINE && wr->num_sge) {
 247                                 if (ctrl->msg_length >
 248                                         hr_dev->caps.max_sq_inline) {
 249                                         ret = -EINVAL;
 250                                         *bad_wr = wr;
 251                                         dev_err(dev, "inline len(1-%d)=%d, illegal",
 252                                                 ctrl->msg_length,
 253                                                 hr_dev->caps.max_sq_inline);
 254                                         goto out;
 255                                 }
 256                                 for (i = 0; i < wr->num_sge; i++) {
 257                                         memcpy(wqe, ((void *) (uintptr_t)
 258                                                wr->sg_list[i].addr),
 259                                                wr->sg_list[i].length);
 260                                         wqe += wr->sg_list[i].length;
 261                                 }
 262                                 ctrl->flag |= HNS_ROCE_WQE_INLINE;
 263                         } else {
 264                                 /*sqe num is two */
 265                                 for (i = 0; i < wr->num_sge; i++)
 266                                         set_data_seg(dseg + i, wr->sg_list + i);
 267
 268                                 ctrl->flag |= cpu_to_le32(wr->num_sge <<
 269                                               HNS_ROCE_WQE_SGE_NUM_BIT);
 270                         }
 271                         ind++;
 272                 }
 273         }
 274
 275 out:
 276         /* Set DB return */
 277         if (likely(nreq)) {
 278                 qp->sq.head += nreq;
 279                 /* Memory barrier */
 280                 wmb();
 281
 282                 sq_db.u32_4 = 0;
 283                 sq_db.u32_8 = 0;
 284                 roce_set_field(sq_db.u32_4, SQ_DOORBELL_U32_4_SQ_HEAD_M,
 285                                SQ_DOORBELL_U32_4_SQ_HEAD_S,
 286                               (qp->sq.head & ((qp->sq.wqe_cnt << 1) - 1)));
 287                 roce_set_field(sq_db.u32_4, SQ_DOORBELL_U32_4_PORT_M,
 288                                SQ_DOORBELL_U32_4_PORT_S, qp->phy_port);
 289                 roce_set_field(sq_db.u32_8, SQ_DOORBELL_U32_8_QPN_M,
 290                                SQ_DOORBELL_U32_8_QPN_S, qp->doorbell_qpn);
 291                 roce_set_bit(sq_db.u32_8, SQ_DOORBELL_HW_SYNC_S, 1);
 292
 293                 doorbell[0] = sq_db.u32_4;
 294                 doorbell[1] = sq_db.u32_8;
 295
 296                 hns_roce_write64_k(doorbell, qp->sq.db_reg_l);
 297                 qp->sq_next_wqe = ind;
 298         }
 299
 300         spin_unlock_irqrestore(&qp->sq.lock, flags);
 301
 302         return ret;
 303 }
 304
 305 int hns_roce_v1_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr,
 306                           struct ib_recv_wr **bad_wr)
 307 {
 308         int ret = 0;
 309         int nreq = 0;
 310         int ind = 0;
 311         int i = 0;
 312         u32 reg_val = 0;
 313         unsigned long flags = 0;
 314         struct hns_roce_rq_wqe_ctrl *ctrl = NULL;
 315         struct hns_roce_wqe_data_seg *scat = NULL;
 316         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
 317         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
 318         struct device *dev = &hr_dev->pdev->dev;
 319         struct hns_roce_rq_db rq_db;
 320         uint32_t doorbell[2] = {0};
 321
 322         spin_lock_irqsave(&hr_qp->rq.lock, flags);
 323         ind = hr_qp->rq.head & (hr_qp->rq.wqe_cnt - 1);
 324
 325         for (nreq = 0; wr; ++nreq, wr = wr->next) {
 326                 if (hns_roce_wq_overflow(&hr_qp->rq, nreq,
 327                         hr_qp->ibqp.recv_cq)) {
 328                         ret = -ENOMEM;
 329                         *bad_wr = wr;
 330                         goto out;
 331                 }
 332
 333                 if (unlikely(wr->num_sge > hr_qp->rq.max_gs)) {
 334                         dev_err(dev, "rq:num_sge=%d > qp->sq.max_gs=%d\n",
 335                                 wr->num_sge, hr_qp->rq.max_gs);
 336                         ret = -EINVAL;
 337                         *bad_wr = wr;
 338                         goto out;
 339                 }
 340
 341                 ctrl = get_recv_wqe(hr_qp, ind);
 342
 343                 roce_set_field(ctrl->rwqe_byte_12,
 344                                RQ_WQE_CTRL_RWQE_BYTE_12_RWQE_SGE_NUM_M,
 345                                RQ_WQE_CTRL_RWQE_BYTE_12_RWQE_SGE_NUM_S,
 346                                wr->num_sge);
 347
 348                 scat = (struct hns_roce_wqe_data_seg *)(ctrl + 1);
 349
 350                 for (i = 0; i < wr->num_sge; i++)
 351                         set_data_seg(scat + i, wr->sg_list + i);
 352
 353                 hr_qp->rq.wrid[ind] = wr->wr_id;
 354
 355                 ind = (ind + 1) & (hr_qp->rq.wqe_cnt - 1);
 356         }
 357
 358 out:
 359         if (likely(nreq)) {
 360                 hr_qp->rq.head += nreq;
 361                 /* Memory barrier */
 362                 wmb();
 363
 364                 if (ibqp->qp_type == IB_QPT_GSI) {
 365                         /* SW update GSI rq header */
 366                         reg_val = roce_read(to_hr_dev(ibqp->device),
 367                                             ROCEE_QP1C_CFG3_0_REG +
 368                                             QP1C_CFGN_OFFSET * hr_qp->phy_port);
 369                         roce_set_field(reg_val,
 370                                        ROCEE_QP1C_CFG3_0_ROCEE_QP1C_RQ_HEAD_M,
 371                                        ROCEE_QP1C_CFG3_0_ROCEE_QP1C_RQ_HEAD_S,
 372                                        hr_qp->rq.head);
 373                         roce_write(to_hr_dev(ibqp->device),
 374                                    ROCEE_QP1C_CFG3_0_REG +
 375                                    QP1C_CFGN_OFFSET * hr_qp->phy_port, reg_val);
 376                 } else {
 377                         rq_db.u32_4 = 0;
 378                         rq_db.u32_8 = 0;
 379
 380                         roce_set_field(rq_db.u32_4, RQ_DOORBELL_U32_4_RQ_HEAD_M,
 381                                        RQ_DOORBELL_U32_4_RQ_HEAD_S,
 382                                        hr_qp->rq.head);
 383                         roce_set_field(rq_db.u32_8, RQ_DOORBELL_U32_8_QPN_M,
 384                                        RQ_DOORBELL_U32_8_QPN_S, hr_qp->qpn);
 385                         roce_set_field(rq_db.u32_8, RQ_DOORBELL_U32_8_CMD_M,
 386                                        RQ_DOORBELL_U32_8_CMD_S, 1);
 387                         roce_set_bit(rq_db.u32_8, RQ_DOORBELL_U32_8_HW_SYNC_S,
 388                                      1);
 389
 390                         doorbell[0] = rq_db.u32_4;
 391                         doorbell[1] = rq_db.u32_8;
 392
 393                         hns_roce_write64_k(doorbell, hr_qp->rq.db_reg_l);
 394                 }
 395         }
 396         spin_unlock_irqrestore(&hr_qp->rq.lock, flags);
 397
 398         return ret;
 399 }
 400
 401 static void hns_roce_set_db_event_mode(struct hns_roce_dev *hr_dev,
 402                                        int sdb_mode, int odb_mode)
 403 {
 404         u32 val;
 405
 406         val = roce_read(hr_dev, ROCEE_GLB_CFG_REG);
 407         roce_set_bit(val, ROCEE_GLB_CFG_ROCEE_DB_SQ_MODE_S, sdb_mode);
 408         roce_set_bit(val, ROCEE_GLB_CFG_ROCEE_DB_OTH_MODE_S, odb_mode);
 409         roce_write(hr_dev, ROCEE_GLB_CFG_REG, val);
 410 }
 411
 412 static void hns_roce_set_db_ext_mode(struct hns_roce_dev *hr_dev, u32 sdb_mode,
 413                                      u32 odb_mode)
 414 {
 415         u32 val;
 416
 417         /* Configure SDB/ODB extend mode */
 418         val = roce_read(hr_dev, ROCEE_GLB_CFG_REG);
 419         roce_set_bit(val, ROCEE_GLB_CFG_SQ_EXT_DB_MODE_S, sdb_mode);
 420         roce_set_bit(val, ROCEE_GLB_CFG_OTH_EXT_DB_MODE_S, odb_mode);
 421         roce_write(hr_dev, ROCEE_GLB_CFG_REG, val);
 422 }
 423
 424 static void hns_roce_set_sdb(struct hns_roce_dev *hr_dev, u32 sdb_alept,
 425                              u32 sdb_alful)
 426 {
 427         u32 val;
 428
 429         /* Configure SDB */
 430         val = roce_read(hr_dev, ROCEE_DB_SQ_WL_REG);
 431         roce_set_field(val, ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_M,
 432                        ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_S, sdb_alful);
 433         roce_set_field(val, ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_EMPTY_M,
 434                        ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_EMPTY_S, sdb_alept);
 435         roce_write(hr_dev, ROCEE_DB_SQ_WL_REG, val);
 436 }
 437
 438 static void hns_roce_set_odb(struct hns_roce_dev *hr_dev, u32 odb_alept,
 439                              u32 odb_alful)
 440 {
 441         u32 val;
 442
 443         /* Configure ODB */
 444         val = roce_read(hr_dev, ROCEE_DB_OTHERS_WL_REG);
 445         roce_set_field(val, ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_M,
 446                        ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_S, odb_alful);
 447         roce_set_field(val, ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_EMPTY_M,
 448                        ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_EMPTY_S, odb_alept);
 449         roce_write(hr_dev, ROCEE_DB_OTHERS_WL_REG, val);
 450 }
 451
 452 static void hns_roce_set_sdb_ext(struct hns_roce_dev *hr_dev, u32 ext_sdb_alept,
 453                                  u32 ext_sdb_alful)
 454 {
 455         struct device *dev = &hr_dev->pdev->dev;
 456         struct hns_roce_v1_priv *priv;
 457         struct hns_roce_db_table *db;
 458         dma_addr_t sdb_dma_addr;
 459         u32 val;
 460
 461         priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
 462         db = &priv->db_table;
 463
 464         /* Configure extend SDB threshold */
 465         roce_write(hr_dev, ROCEE_EXT_DB_SQ_WL_EMPTY_REG, ext_sdb_alept);
 466         roce_write(hr_dev, ROCEE_EXT_DB_SQ_WL_REG, ext_sdb_alful);
 467
 468         /* Configure extend SDB base addr */
 469         sdb_dma_addr = db->ext_db->sdb_buf_list->map;
 470         roce_write(hr_dev, ROCEE_EXT_DB_SQ_REG, (u32)(sdb_dma_addr >> 12));
 471
 472         /* Configure extend SDB depth */
 473         val = roce_read(hr_dev, ROCEE_EXT_DB_SQ_H_REG);
 474         roce_set_field(val, ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_SHIFT_M,
 475                        ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_SHIFT_S,
 476                        db->ext_db->esdb_dep);
 477         /*
 478          * 44 = 32 + 12, When evaluating addr to hardware, shift 12 because of
 479          * using 4K page, and shift more 32 because of
 480          * caculating the high 32 bit value evaluated to hardware.
 481          */
 482         roce_set_field(val, ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_BA_H_M,
 483                        ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_BA_H_S, sdb_dma_addr >> 44);
 484         roce_write(hr_dev, ROCEE_EXT_DB_SQ_H_REG, val);
 485
 486         dev_dbg(dev, "ext SDB depth: 0x%x\n", db->ext_db->esdb_dep);
 487         dev_dbg(dev, "ext SDB threshold: epmty: 0x%x, ful: 0x%x\n",
 488                 ext_sdb_alept, ext_sdb_alful);
 489 }
 490
 491 static void hns_roce_set_odb_ext(struct hns_roce_dev *hr_dev, u32 ext_odb_alept,
 492                                  u32 ext_odb_alful)
 493 {
 494         struct device *dev = &hr_dev->pdev->dev;
 495         struct hns_roce_v1_priv *priv;
 496         struct hns_roce_db_table *db;
 497         dma_addr_t odb_dma_addr;
 498         u32 val;
 499
 500         priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
 501         db = &priv->db_table;
 502
 503         /* Configure extend ODB threshold */
 504         roce_write(hr_dev, ROCEE_EXT_DB_OTHERS_WL_EMPTY_REG, ext_odb_alept);
 505         roce_write(hr_dev, ROCEE_EXT_DB_OTHERS_WL_REG, ext_odb_alful);
 506
 507         /* Configure extend ODB base addr */
 508         odb_dma_addr = db->ext_db->odb_buf_list->map;
 509         roce_write(hr_dev, ROCEE_EXT_DB_OTH_REG, (u32)(odb_dma_addr >> 12));
 510
 511         /* Configure extend ODB depth */
 512         val = roce_read(hr_dev, ROCEE_EXT_DB_OTH_H_REG);
 513         roce_set_field(val, ROCEE_EXT_DB_OTH_H_EXT_DB_OTH_SHIFT_M,
 514                        ROCEE_EXT_DB_OTH_H_EXT_DB_OTH_SHIFT_S,
 515                        db->ext_db->eodb_dep);
 516         roce_set_field(val, ROCEE_EXT_DB_SQ_H_EXT_DB_OTH_BA_H_M,
 517                        ROCEE_EXT_DB_SQ_H_EXT_DB_OTH_BA_H_S,
 518                        db->ext_db->eodb_dep);
 519         roce_write(hr_dev, ROCEE_EXT_DB_OTH_H_REG, val);
 520
 521         dev_dbg(dev, "ext ODB depth: 0x%x\n", db->ext_db->eodb_dep);
 522         dev_dbg(dev, "ext ODB threshold: empty: 0x%x, ful: 0x%x\n",
 523                 ext_odb_alept, ext_odb_alful);
 524 }
 525
 526 static int hns_roce_db_ext_init(struct hns_roce_dev *hr_dev, u32 sdb_ext_mod,
 527                                 u32 odb_ext_mod)
 528 {
 529         struct device *dev = &hr_dev->pdev->dev;
 530         struct hns_roce_v1_priv *priv;
 531         struct hns_roce_db_table *db;
 532         dma_addr_t sdb_dma_addr;
 533         dma_addr_t odb_dma_addr;
 534         int ret = 0;
 535
 536         priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
 537         db = &priv->db_table;
 538
 539         db->ext_db = kmalloc(sizeof(*db->ext_db), GFP_KERNEL);
 540         if (!db->ext_db)
 541                 return -ENOMEM;
 542
 543         if (sdb_ext_mod) {
 544                 db->ext_db->sdb_buf_list = kmalloc(
 545                                 sizeof(*db->ext_db->sdb_buf_list), GFP_KERNEL);
 546                 if (!db->ext_db->sdb_buf_list) {
 547                         ret = -ENOMEM;
 548                         goto ext_sdb_buf_fail_out;
 549                 }
 550
 551                 db->ext_db->sdb_buf_list->buf = dma_alloc_coherent(dev,
 552                                                      HNS_ROCE_V1_EXT_SDB_SIZE,
 553                                                      &sdb_dma_addr, GFP_KERNEL);
 554                 if (!db->ext_db->sdb_buf_list->buf) {
 555                         ret = -ENOMEM;
 556                         goto alloc_sq_db_buf_fail;
 557                 }
 558                 db->ext_db->sdb_buf_list->map = sdb_dma_addr;
 559
 560                 db->ext_db->esdb_dep = ilog2(HNS_ROCE_V1_EXT_SDB_DEPTH);
 561                 hns_roce_set_sdb_ext(hr_dev, HNS_ROCE_V1_EXT_SDB_ALEPT,
 562                                      HNS_ROCE_V1_EXT_SDB_ALFUL);
 563         } else
 564                 hns_roce_set_sdb(hr_dev, HNS_ROCE_V1_SDB_ALEPT,
 565                                  HNS_ROCE_V1_SDB_ALFUL);
 566
 567         if (odb_ext_mod) {
 568                 db->ext_db->odb_buf_list = kmalloc(
 569                                 sizeof(*db->ext_db->odb_buf_list), GFP_KERNEL);
 570                 if (!db->ext_db->odb_buf_list) {
 571                         ret = -ENOMEM;
 572                         goto ext_odb_buf_fail_out;
 573                 }
 574
 575                 db->ext_db->odb_buf_list->buf = dma_alloc_coherent(dev,
 576                                                      HNS_ROCE_V1_EXT_ODB_SIZE,
 577                                                      &odb_dma_addr, GFP_KERNEL);
 578                 if (!db->ext_db->odb_buf_list->buf) {
 579                         ret = -ENOMEM;
 580                         goto alloc_otr_db_buf_fail;
 581                 }
 582                 db->ext_db->odb_buf_list->map = odb_dma_addr;
 583
 584                 db->ext_db->eodb_dep = ilog2(HNS_ROCE_V1_EXT_ODB_DEPTH);
 585                 hns_roce_set_odb_ext(hr_dev, HNS_ROCE_V1_EXT_ODB_ALEPT,
 586                                      HNS_ROCE_V1_EXT_ODB_ALFUL);
 587         } else
 588                 hns_roce_set_odb(hr_dev, HNS_ROCE_V1_ODB_ALEPT,
 589                                  HNS_ROCE_V1_ODB_ALFUL);
 590
 591         hns_roce_set_db_ext_mode(hr_dev, sdb_ext_mod, odb_ext_mod);
 592
 593         return 0;
 594
 595 alloc_otr_db_buf_fail:
 596         kfree(db->ext_db->odb_buf_list);
 597
 598 ext_odb_buf_fail_out:
 599         if (sdb_ext_mod) {
 600                 dma_free_coherent(dev, HNS_ROCE_V1_EXT_SDB_SIZE,
 601                                   db->ext_db->sdb_buf_list->buf,
 602                                   db->ext_db->sdb_buf_list->map);
 603         }
 604
 605 alloc_sq_db_buf_fail:
 606         if (sdb_ext_mod)
 607                 kfree(db->ext_db->sdb_buf_list);
 608
 609 ext_sdb_buf_fail_out:
 610         kfree(db->ext_db);
 611         return ret;
 612 }
 613
 614 static int hns_roce_db_init(struct hns_roce_dev *hr_dev)
 615 {
 616         struct device *dev = &hr_dev->pdev->dev;
 617         struct hns_roce_v1_priv *priv;
 618         struct hns_roce_db_table *db;
 619         u32 sdb_ext_mod;
 620         u32 odb_ext_mod;
 621         u32 sdb_evt_mod;
 622         u32 odb_evt_mod;
 623         int ret = 0;
 624
 625         priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
 626         db = &priv->db_table;
 627
 628         memset(db, 0, sizeof(*db));
 629
 630         /* Default DB mode */
 631         sdb_ext_mod = HNS_ROCE_SDB_EXTEND_MODE;
 632         odb_ext_mod = HNS_ROCE_ODB_EXTEND_MODE;
 633         sdb_evt_mod = HNS_ROCE_SDB_NORMAL_MODE;
 634         odb_evt_mod = HNS_ROCE_ODB_POLL_MODE;
 635
 636         db->sdb_ext_mod = sdb_ext_mod;
 637         db->odb_ext_mod = odb_ext_mod;
 638
 639         /* Init extend DB */
 640         ret = hns_roce_db_ext_init(hr_dev, sdb_ext_mod, odb_ext_mod);
 641         if (ret) {
 642                 dev_err(dev, "Failed in extend DB configuration.\n");
 643                 return ret;
 644         }
 645
 646         hns_roce_set_db_event_mode(hr_dev, sdb_evt_mod, odb_evt_mod);
 647
 648         return 0;
 649 }
 650
 651 static void hns_roce_db_free(struct hns_roce_dev *hr_dev)
 652 {
 653         struct device *dev = &hr_dev->pdev->dev;
 654         struct hns_roce_v1_priv *priv;
 655         struct hns_roce_db_table *db;
 656
 657         priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
 658         db = &priv->db_table;
 659
 660         if (db->sdb_ext_mod) {
 661                 dma_free_coherent(dev, HNS_ROCE_V1_EXT_SDB_SIZE,
 662                                   db->ext_db->sdb_buf_list->buf,
 663                                   db->ext_db->sdb_buf_list->map);
 664                 kfree(db->ext_db->sdb_buf_list);
 665         }
 666
 667         if (db->odb_ext_mod) {
 668                 dma_free_coherent(dev, HNS_ROCE_V1_EXT_ODB_SIZE,
 669                                   db->ext_db->odb_buf_list->buf,
 670                                   db->ext_db->odb_buf_list->map);
 671                 kfree(db->ext_db->odb_buf_list);
 672         }
 673
 674         kfree(db->ext_db);
 675 }
 676
 677 static int hns_roce_raq_init(struct hns_roce_dev *hr_dev)
 678 {
 679         int ret;
 680         int raq_shift = 0;
 681         dma_addr_t addr;
 682         u32 val;
 683         struct hns_roce_v1_priv *priv;
 684         struct hns_roce_raq_table *raq;
 685         struct device *dev = &hr_dev->pdev->dev;
 686
 687         priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
 688         raq = &priv->raq_table;
 689
 690         raq->e_raq_buf = kzalloc(sizeof(*(raq->e_raq_buf)), GFP_KERNEL);
 691         if (!raq->e_raq_buf)
 692                 return -ENOMEM;
 693
 694         raq->e_raq_buf->buf = dma_alloc_coherent(dev, HNS_ROCE_V1_RAQ_SIZE,
 695                                                  &addr, GFP_KERNEL);
 696         if (!raq->e_raq_buf->buf) {
 697                 ret = -ENOMEM;
 698                 goto err_dma_alloc_raq;
 699         }
 700         raq->e_raq_buf->map = addr;
 701
 702         /* Configure raq extended address. 48bit 4K align*/
 703         roce_write(hr_dev, ROCEE_EXT_RAQ_REG, raq->e_raq_buf->map >> 12);
 704
 705         /* Configure raq_shift */
 706         raq_shift = ilog2(HNS_ROCE_V1_RAQ_SIZE / HNS_ROCE_V1_RAQ_ENTRY);
 707         val = roce_read(hr_dev, ROCEE_EXT_RAQ_H_REG);
 708         roce_set_field(val, ROCEE_EXT_RAQ_H_EXT_RAQ_SHIFT_M,
 709                        ROCEE_EXT_RAQ_H_EXT_RAQ_SHIFT_S, raq_shift);
 710         /*
 711          * 44 = 32 + 12, When evaluating addr to hardware, shift 12 because of
 712          * using 4K page, and shift more 32 because of
 713          * caculating the high 32 bit value evaluated to hardware.
 714          */
 715         roce_set_field(val, ROCEE_EXT_RAQ_H_EXT_RAQ_BA_H_M,
 716                        ROCEE_EXT_RAQ_H_EXT_RAQ_BA_H_S,
 717                        raq->e_raq_buf->map >> 44);
 718         roce_write(hr_dev, ROCEE_EXT_RAQ_H_REG, val);
 719         dev_dbg(dev, "Configure raq_shift 0x%x.\n", val);
 720
 721         /* Configure raq threshold */
 722         val = roce_read(hr_dev, ROCEE_RAQ_WL_REG);
 723         roce_set_field(val, ROCEE_RAQ_WL_ROCEE_RAQ_WL_M,
 724                        ROCEE_RAQ_WL_ROCEE_RAQ_WL_S,
 725                        HNS_ROCE_V1_EXT_RAQ_WF);
 726         roce_write(hr_dev, ROCEE_RAQ_WL_REG, val);
 727         dev_dbg(dev, "Configure raq_wl 0x%x.\n", val);
 728
 729         /* Enable extend raq */
 730         val = roce_read(hr_dev, ROCEE_WRMS_POL_TIME_INTERVAL_REG);
 731         roce_set_field(val,
 732                        ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_POL_TIME_INTERVAL_M,
 733                        ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_POL_TIME_INTERVAL_S,
 734                        POL_TIME_INTERVAL_VAL);
 735         roce_set_bit(val, ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_EXT_RAQ_MODE, 1);
 736         roce_set_field(val,
 737                        ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_CFG_M,
 738                        ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_CFG_S,
 739                        2);
 740         roce_set_bit(val,
 741                      ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_EN_S, 1);
 742         roce_write(hr_dev, ROCEE_WRMS_POL_TIME_INTERVAL_REG, val);
 743         dev_dbg(dev, "Configure WrmsPolTimeInterval 0x%x.\n", val);
 744
 745         /* Enable raq drop */
 746         val = roce_read(hr_dev, ROCEE_GLB_CFG_REG);
 747         roce_set_bit(val, ROCEE_GLB_CFG_TRP_RAQ_DROP_EN_S, 1);
 748         roce_write(hr_dev, ROCEE_GLB_CFG_REG, val);
 749         dev_dbg(dev, "Configure GlbCfg = 0x%x.\n", val);
 750
 751         return 0;
 752
 753 err_dma_alloc_raq:
 754         kfree(raq->e_raq_buf);
 755         return ret;
 756 }
 757
 758 static void hns_roce_raq_free(struct hns_roce_dev *hr_dev)
 759 {
 760         struct device *dev = &hr_dev->pdev->dev;
 761         struct hns_roce_v1_priv *priv;
 762         struct hns_roce_raq_table *raq;
 763
 764         priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
 765         raq = &priv->raq_table;
 766
 767         dma_free_coherent(dev, HNS_ROCE_V1_RAQ_SIZE, raq->e_raq_buf->buf,
 768                           raq->e_raq_buf->map);
 769         kfree(raq->e_raq_buf);
 770 }
 771
 772 static void hns_roce_port_enable(struct hns_roce_dev *hr_dev, int enable_flag)
 773 {
 774         u32 val;
 775
 776         if (enable_flag) {
 777                 val = roce_read(hr_dev, ROCEE_GLB_CFG_REG);
 778                  /* Open all ports */
 779                 roce_set_field(val, ROCEE_GLB_CFG_ROCEE_PORT_ST_M,
 780                                ROCEE_GLB_CFG_ROCEE_PORT_ST_S,
 781                                ALL_PORT_VAL_OPEN);
 782                 roce_write(hr_dev, ROCEE_GLB_CFG_REG, val);
 783         } else {
 784                 val = roce_read(hr_dev, ROCEE_GLB_CFG_REG);
 785                 /* Close all ports */
 786                 roce_set_field(val, ROCEE_GLB_CFG_ROCEE_PORT_ST_M,
 787                                ROCEE_GLB_CFG_ROCEE_PORT_ST_S, 0x0);
 788                 roce_write(hr_dev, ROCEE_GLB_CFG_REG, val);
 789         }
 790 }
 791
 792 static int hns_roce_bt_init(struct hns_roce_dev *hr_dev)
 793 {
 794         struct device *dev = &hr_dev->pdev->dev;
 795         struct hns_roce_v1_priv *priv;
 796         int ret;
 797
 798         priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
 799
 800         priv->bt_table.qpc_buf.buf = dma_alloc_coherent(dev,
 801                 HNS_ROCE_BT_RSV_BUF_SIZE, &priv->bt_table.qpc_buf.map,
 802                 GFP_KERNEL);
 803         if (!priv->bt_table.qpc_buf.buf)
 804                 return -ENOMEM;
 805
 806         priv->bt_table.mtpt_buf.buf = dma_alloc_coherent(dev,
 807                 HNS_ROCE_BT_RSV_BUF_SIZE, &priv->bt_table.mtpt_buf.map,
 808                 GFP_KERNEL);
 809         if (!priv->bt_table.mtpt_buf.buf) {
 810                 ret = -ENOMEM;
 811                 goto err_failed_alloc_mtpt_buf;
 812         }
 813
 814         priv->bt_table.cqc_buf.buf = dma_alloc_coherent(dev,
 815                 HNS_ROCE_BT_RSV_BUF_SIZE, &priv->bt_table.cqc_buf.map,
 816                 GFP_KERNEL);
 817         if (!priv->bt_table.cqc_buf.buf) {
 818                 ret = -ENOMEM;
 819                 goto err_failed_alloc_cqc_buf;
 820         }
 821
 822         return 0;
 823
 824 err_failed_alloc_cqc_buf:
 825         dma_free_coherent(dev, HNS_ROCE_BT_RSV_BUF_SIZE,
 826                 priv->bt_table.mtpt_buf.buf, priv->bt_table.mtpt_buf.map);
 827
 828 err_failed_alloc_mtpt_buf:
 829         dma_free_coherent(dev, HNS_ROCE_BT_RSV_BUF_SIZE,
 830                 priv->bt_table.qpc_buf.buf, priv->bt_table.qpc_buf.map);
 831
 832         return ret;
 833 }
 834
 835 static void hns_roce_bt_free(struct hns_roce_dev *hr_dev)
 836 {
 837         struct device *dev = &hr_dev->pdev->dev;
 838         struct hns_roce_v1_priv *priv;
 839
 840         priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
 841
 842         dma_free_coherent(dev, HNS_ROCE_BT_RSV_BUF_SIZE,
 843                 priv->bt_table.cqc_buf.buf, priv->bt_table.cqc_buf.map);
 844
 845         dma_free_coherent(dev, HNS_ROCE_BT_RSV_BUF_SIZE,
 846                 priv->bt_table.mtpt_buf.buf, priv->bt_table.mtpt_buf.map);
 847
 848         dma_free_coherent(dev, HNS_ROCE_BT_RSV_BUF_SIZE,
 849                 priv->bt_table.qpc_buf.buf, priv->bt_table.qpc_buf.map);
 850 }
 851
 852 /**
 853  * hns_roce_v1_reset - reset RoCE
 854  * @hr_dev: RoCE device struct pointer
 855  * @enable: true -- drop reset, false -- reset
 856  * return 0 - success , negative --fail
 857  */
 858 int hns_roce_v1_reset(struct hns_roce_dev *hr_dev, bool dereset)
 859 {
 860         struct device_node *dsaf_node;
 861         struct device *dev = &hr_dev->pdev->dev;
 862         struct device_node *np = dev->of_node;
 863         struct fwnode_handle *fwnode;
 864         int ret;
 865
 866         /* check if this is DT/ACPI case */
 867         if (dev_of_node(dev)) {
 868                 dsaf_node = of_parse_phandle(np, "dsaf-handle", 0);
 869                 if (!dsaf_node) {
 870                         dev_err(dev, "could not find dsaf-handle\n");
 871                         return -EINVAL;
 872                 }
 873                 fwnode = &dsaf_node->fwnode;
 874         } else if (is_acpi_device_node(dev->fwnode)) {
 875                 struct acpi_reference_args args;
 876
 877                 ret = acpi_node_get_property_reference(dev->fwnode,
 878                                                        "dsaf-handle", 0, &args);
 879                 if (ret) {
 880                         dev_err(dev, "could not find dsaf-handle\n");
 881                         return ret;
 882                 }
 883                 fwnode = acpi_fwnode_handle(args.adev);
 884         } else {
 885                 dev_err(dev, "cannot read data from DT or ACPI\n");
 886                 return -ENXIO;
 887         }
 888
 889         ret = hns_dsaf_roce_reset(fwnode, false);
 890         if (ret)
 891                 return ret;
 892
 893         if (dereset) {
 894                 msleep(SLEEP_TIME_INTERVAL);
 895                 ret = hns_dsaf_roce_reset(fwnode, true);
 896         }
 897
 898         return ret;
 899 }
 900
 901 void hns_roce_v1_profile(struct hns_roce_dev *hr_dev)
 902 {
 903         int i = 0;
 904         struct hns_roce_caps *caps = &hr_dev->caps;
 905
 906         hr_dev->vendor_id = le32_to_cpu(roce_read(hr_dev, ROCEE_VENDOR_ID_REG));
 907         hr_dev->vendor_part_id = le32_to_cpu(roce_read(hr_dev,
 908                                              ROCEE_VENDOR_PART_ID_REG));
 909         hr_dev->hw_rev = le32_to_cpu(roce_read(hr_dev, ROCEE_HW_VERSION_REG));
 910
 911         hr_dev->sys_image_guid = le32_to_cpu(roce_read(hr_dev,
 912                                              ROCEE_SYS_IMAGE_GUID_L_REG)) |
 913                                 ((u64)le32_to_cpu(roce_read(hr_dev,
 914                                             ROCEE_SYS_IMAGE_GUID_H_REG)) << 32);
 915
 916         caps->num_qps           = HNS_ROCE_V1_MAX_QP_NUM;
 917         caps->max_wqes          = HNS_ROCE_V1_MAX_WQE_NUM;
 918         caps->num_cqs           = HNS_ROCE_V1_MAX_CQ_NUM;
 919         caps->max_cqes          = HNS_ROCE_V1_MAX_CQE_NUM;
 920         caps->max_sq_sg         = HNS_ROCE_V1_SG_NUM;
 921         caps->max_rq_sg         = HNS_ROCE_V1_SG_NUM;
 922         caps->max_sq_inline     = HNS_ROCE_V1_INLINE_SIZE;
 923         caps->num_uars          = HNS_ROCE_V1_UAR_NUM;
 924         caps->phy_num_uars      = HNS_ROCE_V1_PHY_UAR_NUM;
 925         caps->num_aeq_vectors   = HNS_ROCE_AEQE_VEC_NUM;
 926         caps->num_comp_vectors  = HNS_ROCE_COMP_VEC_NUM;
 927         caps->num_other_vectors = HNS_ROCE_AEQE_OF_VEC_NUM;
 928         caps->num_mtpts         = HNS_ROCE_V1_MAX_MTPT_NUM;
 929         caps->num_mtt_segs      = HNS_ROCE_V1_MAX_MTT_SEGS;
 930         caps->num_pds           = HNS_ROCE_V1_MAX_PD_NUM;
 931         caps->max_qp_init_rdma  = HNS_ROCE_V1_MAX_QP_INIT_RDMA;
 932         caps->max_qp_dest_rdma  = HNS_ROCE_V1_MAX_QP_DEST_RDMA;
 933         caps->max_sq_desc_sz    = HNS_ROCE_V1_MAX_SQ_DESC_SZ;
 934         caps->max_rq_desc_sz    = HNS_ROCE_V1_MAX_RQ_DESC_SZ;
 935         caps->qpc_entry_sz      = HNS_ROCE_V1_QPC_ENTRY_SIZE;
 936         caps->irrl_entry_sz     = HNS_ROCE_V1_IRRL_ENTRY_SIZE;
 937         caps->cqc_entry_sz      = HNS_ROCE_V1_CQC_ENTRY_SIZE;
 938         caps->mtpt_entry_sz     = HNS_ROCE_V1_MTPT_ENTRY_SIZE;
 939         caps->mtt_entry_sz      = HNS_ROCE_V1_MTT_ENTRY_SIZE;
 940         caps->cq_entry_sz       = HNS_ROCE_V1_CQE_ENTRY_SIZE;
 941         caps->page_size_cap     = HNS_ROCE_V1_PAGE_SIZE_SUPPORT;
 942         caps->reserved_lkey     = 0;
 943         caps->reserved_pds      = 0;
 944         caps->reserved_mrws     = 1;
 945         caps->reserved_uars     = 0;
 946         caps->reserved_cqs      = 0;
 947
 948         for (i = 0; i < caps->num_ports; i++)
 949                 caps->pkey_table_len[i] = 1;
 950
 951         for (i = 0; i < caps->num_ports; i++) {
 952                 /* Six ports shared 16 GID in v1 engine */
 953                 if (i >= (HNS_ROCE_V1_GID_NUM % caps->num_ports))
 954                         caps->gid_table_len[i] = HNS_ROCE_V1_GID_NUM /
 955                                                  caps->num_ports;
 956                 else
 957                         caps->gid_table_len[i] = HNS_ROCE_V1_GID_NUM /
 958                                                  caps->num_ports + 1;
 959         }
 960
 961         for (i = 0; i < caps->num_comp_vectors; i++)
 962                 caps->ceqe_depth[i] = HNS_ROCE_V1_NUM_COMP_EQE;
 963
 964         caps->aeqe_depth = HNS_ROCE_V1_NUM_ASYNC_EQE;
 965         caps->local_ca_ack_delay = le32_to_cpu(roce_read(hr_dev,
 966                                                          ROCEE_ACK_DELAY_REG));
 967         caps->max_mtu = IB_MTU_2048;
 968 }
 969
 970 int hns_roce_v1_init(struct hns_roce_dev *hr_dev)
 971 {
 972         int ret;
 973         u32 val;
 974         struct device *dev = &hr_dev->pdev->dev;
 975
 976         /* DMAE user config */
 977         val = roce_read(hr_dev, ROCEE_DMAE_USER_CFG1_REG);
 978         roce_set_field(val, ROCEE_DMAE_USER_CFG1_ROCEE_CACHE_TB_CFG_M,
 979                        ROCEE_DMAE_USER_CFG1_ROCEE_CACHE_TB_CFG_S, 0xf);
 980         roce_set_field(val, ROCEE_DMAE_USER_CFG1_ROCEE_STREAM_ID_TB_CFG_M,
 981                        ROCEE_DMAE_USER_CFG1_ROCEE_STREAM_ID_TB_CFG_S,
 982                        1 << PAGES_SHIFT_16);
 983         roce_write(hr_dev, ROCEE_DMAE_USER_CFG1_REG, val);
 984
 985         val = roce_read(hr_dev, ROCEE_DMAE_USER_CFG2_REG);
 986         roce_set_field(val, ROCEE_DMAE_USER_CFG2_ROCEE_CACHE_PKT_CFG_M,
 987                        ROCEE_DMAE_USER_CFG2_ROCEE_CACHE_PKT_CFG_S, 0xf);
 988         roce_set_field(val, ROCEE_DMAE_USER_CFG2_ROCEE_STREAM_ID_PKT_CFG_M,
 989                        ROCEE_DMAE_USER_CFG2_ROCEE_STREAM_ID_PKT_CFG_S,
 990                        1 << PAGES_SHIFT_16);
 991
 992         ret = hns_roce_db_init(hr_dev);
 993         if (ret) {
 994                 dev_err(dev, "doorbell init failed!\n");
 995                 return ret;
 996         }
 997
 998         ret = hns_roce_raq_init(hr_dev);
 999         if (ret) {
1000                 dev_err(dev, "raq init failed!\n");
1001                 goto error_failed_raq_init;
1002         }
1003
1004         hns_roce_port_enable(hr_dev, HNS_ROCE_PORT_UP);
1005
1006         ret = hns_roce_bt_init(hr_dev);
1007         if (ret) {
1008                 dev_err(dev, "bt init failed!\n");
1009                 goto error_failed_bt_init;
1010         }
1011
1012         return 0;
1013
1014 error_failed_bt_init:
1015         hns_roce_port_enable(hr_dev, HNS_ROCE_PORT_DOWN);
1016         hns_roce_raq_free(hr_dev);
1017
1018 error_failed_raq_init:
1019         hns_roce_db_free(hr_dev);
1020         return ret;
1021 }
1022
1023 void hns_roce_v1_exit(struct hns_roce_dev *hr_dev)
1024 {
1025         hns_roce_bt_free(hr_dev);
1026         hns_roce_port_enable(hr_dev, HNS_ROCE_PORT_DOWN);
1027         hns_roce_raq_free(hr_dev);
1028         hns_roce_db_free(hr_dev);
1029 }
1030
1031 void hns_roce_v1_set_gid(struct hns_roce_dev *hr_dev, u8 port, int gid_index,
1032                          union ib_gid *gid)
1033 {
1034         u32 *p = NULL;
1035         u8 gid_idx = 0;
1036
1037         gid_idx = hns_get_gid_index(hr_dev, port, gid_index);
1038
1039         p = (u32 *)&gid->raw[0];
1040         roce_raw_write(*p, hr_dev->reg_base + ROCEE_PORT_GID_L_0_REG +
1041                        (HNS_ROCE_V1_GID_NUM * gid_idx));
1042
1043         p = (u32 *)&gid->raw[4];
1044         roce_raw_write(*p, hr_dev->reg_base + ROCEE_PORT_GID_ML_0_REG +
1045                        (HNS_ROCE_V1_GID_NUM * gid_idx));
1046
1047         p = (u32 *)&gid->raw[8];
1048         roce_raw_write(*p, hr_dev->reg_base + ROCEE_PORT_GID_MH_0_REG +
1049                        (HNS_ROCE_V1_GID_NUM * gid_idx));
1050
1051         p = (u32 *)&gid->raw[0xc];
1052         roce_raw_write(*p, hr_dev->reg_base + ROCEE_PORT_GID_H_0_REG +
1053                        (HNS_ROCE_V1_GID_NUM * gid_idx));
1054 }
1055
1056 void hns_roce_v1_set_mac(struct hns_roce_dev *hr_dev, u8 phy_port, u8 *addr)
1057 {
1058         u32 reg_smac_l;
1059         u16 reg_smac_h;
1060         u16 *p_h;
1061         u32 *p;
1062         u32 val;
1063
1064         p = (u32 *)(&addr[0]);
1065         reg_smac_l = *p;
1066         roce_raw_write(reg_smac_l, hr_dev->reg_base + ROCEE_SMAC_L_0_REG +
1067                        PHY_PORT_OFFSET * phy_port);
1068
1069         val = roce_read(hr_dev,
1070                         ROCEE_SMAC_H_0_REG + phy_port * PHY_PORT_OFFSET);
1071         p_h = (u16 *)(&addr[4]);
1072         reg_smac_h  = *p_h;
1073         roce_set_field(val, ROCEE_SMAC_H_ROCEE_SMAC_H_M,
1074                        ROCEE_SMAC_H_ROCEE_SMAC_H_S, reg_smac_h);
1075         roce_write(hr_dev, ROCEE_SMAC_H_0_REG + phy_port * PHY_PORT_OFFSET,
1076                    val);
1077 }
1078
1079 void hns_roce_v1_set_mtu(struct hns_roce_dev *hr_dev, u8 phy_port,
1080                          enum ib_mtu mtu)
1081 {
1082         u32 val;
1083
1084         val = roce_read(hr_dev,
1085                         ROCEE_SMAC_H_0_REG + phy_port * PHY_PORT_OFFSET);
1086         roce_set_field(val, ROCEE_SMAC_H_ROCEE_PORT_MTU_M,
1087                        ROCEE_SMAC_H_ROCEE_PORT_MTU_S, mtu);
1088         roce_write(hr_dev, ROCEE_SMAC_H_0_REG + phy_port * PHY_PORT_OFFSET,
1089                    val);
1090 }
1091
1092 int hns_roce_v1_write_mtpt(void *mb_buf, struct hns_roce_mr *mr,
1093                            unsigned long mtpt_idx)
1094 {
1095         struct hns_roce_v1_mpt_entry *mpt_entry;
1096         struct scatterlist *sg;
1097         u64 *pages;
1098         int entry;
1099         int i;
1100
1101         /* MPT filled into mailbox buf */
1102         mpt_entry = (struct hns_roce_v1_mpt_entry *)mb_buf;
1103         memset(mpt_entry, 0, sizeof(*mpt_entry));
1104
1105         roce_set_field(mpt_entry->mpt_byte_4, MPT_BYTE_4_KEY_STATE_M,
1106                        MPT_BYTE_4_KEY_STATE_S, KEY_VALID);
1107         roce_set_field(mpt_entry->mpt_byte_4, MPT_BYTE_4_KEY_M,
1108                        MPT_BYTE_4_KEY_S, mr->key);
1109         roce_set_field(mpt_entry->mpt_byte_4, MPT_BYTE_4_PAGE_SIZE_M,
1110                        MPT_BYTE_4_PAGE_SIZE_S, MR_SIZE_4K);
1111         roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_MW_TYPE_S, 0);
1112         roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_MW_BIND_ENABLE_S,
1113                      (mr->access & IB_ACCESS_MW_BIND ? 1 : 0));
1114         roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_OWN_S, 0);
1115         roce_set_field(mpt_entry->mpt_byte_4, MPT_BYTE_4_MEMORY_LOCATION_TYPE_M,
1116                        MPT_BYTE_4_MEMORY_LOCATION_TYPE_S, mr->type);
1117         roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_REMOTE_ATOMIC_S, 0);
1118         roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_LOCAL_WRITE_S,
1119                      (mr->access & IB_ACCESS_LOCAL_WRITE ? 1 : 0));
1120         roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_REMOTE_WRITE_S,
1121                      (mr->access & IB_ACCESS_REMOTE_WRITE ? 1 : 0));
1122         roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_REMOTE_READ_S,
1123                      (mr->access & IB_ACCESS_REMOTE_READ ? 1 : 0));
1124         roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_REMOTE_INVAL_ENABLE_S,
1125                      0);
1126         roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_ADDRESS_TYPE_S, 0);
1127
1128         roce_set_field(mpt_entry->mpt_byte_12, MPT_BYTE_12_PBL_ADDR_H_M,
1129                        MPT_BYTE_12_PBL_ADDR_H_S, 0);
1130         roce_set_field(mpt_entry->mpt_byte_12, MPT_BYTE_12_MW_BIND_COUNTER_M,
1131                        MPT_BYTE_12_MW_BIND_COUNTER_S, 0);
1132
1133         mpt_entry->virt_addr_l = (u32)mr->iova;
1134         mpt_entry->virt_addr_h = (u32)(mr->iova >> 32);
1135         mpt_entry->length = (u32)mr->size;
1136
1137         roce_set_field(mpt_entry->mpt_byte_28, MPT_BYTE_28_PD_M,
1138                        MPT_BYTE_28_PD_S, mr->pd);
1139         roce_set_field(mpt_entry->mpt_byte_28, MPT_BYTE_28_L_KEY_IDX_L_M,
1140                        MPT_BYTE_28_L_KEY_IDX_L_S, mtpt_idx);
1141         roce_set_field(mpt_entry->mpt_byte_64, MPT_BYTE_64_L_KEY_IDX_H_M,
1142                        MPT_BYTE_64_L_KEY_IDX_H_S, mtpt_idx >> MTPT_IDX_SHIFT);
1143
1144         /* DMA momery regsiter */
1145         if (mr->type == MR_TYPE_DMA)
1146                 return 0;
1147
1148         pages = (u64 *) __get_free_page(GFP_KERNEL);
1149         if (!pages)
1150                 return -ENOMEM;
1151
1152         i = 0;
1153         for_each_sg(mr->umem->sg_head.sgl, sg, mr->umem->nmap, entry) {
1154                 pages[i] = ((u64)sg_dma_address(sg)) >> 12;
1155
1156                 /* Directly record to MTPT table firstly 7 entry */
1157                 if (i >= HNS_ROCE_MAX_INNER_MTPT_NUM)
1158                         break;
1159                 i++;
1160         }
1161
1162         /* Register user mr */
1163         for (i = 0; i < HNS_ROCE_MAX_INNER_MTPT_NUM; i++) {
1164                 switch (i) {
1165                 case 0:
1166                         mpt_entry->pa0_l = cpu_to_le32((u32)(pages[i]));
1167                         roce_set_field(mpt_entry->mpt_byte_36,
1168                                 MPT_BYTE_36_PA0_H_M,
1169                                 MPT_BYTE_36_PA0_H_S,
1170                                 cpu_to_le32((u32)(pages[i] >> PAGES_SHIFT_32)));
1171                         break;
1172                 case 1:
1173                         roce_set_field(mpt_entry->mpt_byte_36,
1174                                        MPT_BYTE_36_PA1_L_M,
1175                                        MPT_BYTE_36_PA1_L_S,
1176                                        cpu_to_le32((u32)(pages[i])));
1177                         roce_set_field(mpt_entry->mpt_byte_40,
1178                                 MPT_BYTE_40_PA1_H_M,
1179                                 MPT_BYTE_40_PA1_H_S,
1180                                 cpu_to_le32((u32)(pages[i] >> PAGES_SHIFT_24)));
1181                         break;
1182                 case 2:
1183                         roce_set_field(mpt_entry->mpt_byte_40,
1184                                        MPT_BYTE_40_PA2_L_M,
1185                                        MPT_BYTE_40_PA2_L_S,
1186                                        cpu_to_le32((u32)(pages[i])));
1187                         roce_set_field(mpt_entry->mpt_byte_44,
1188                                 MPT_BYTE_44_PA2_H_M,
1189                                 MPT_BYTE_44_PA2_H_S,
1190                                 cpu_to_le32((u32)(pages[i] >> PAGES_SHIFT_16)));
1191                         break;
1192                 case 3:
1193                         roce_set_field(mpt_entry->mpt_byte_44,
1194                                        MPT_BYTE_44_PA3_L_M,
1195                                        MPT_BYTE_44_PA3_L_S,
1196                                        cpu_to_le32((u32)(pages[i])));
1197                         roce_set_field(mpt_entry->mpt_byte_48,
1198                                 MPT_BYTE_48_PA3_H_M,
1199                                 MPT_BYTE_48_PA3_H_S,
1200                                 cpu_to_le32((u32)(pages[i] >> PAGES_SHIFT_8)));
1201                         break;
1202                 case 4:
1203                         mpt_entry->pa4_l = cpu_to_le32((u32)(pages[i]));
1204                         roce_set_field(mpt_entry->mpt_byte_56,
1205                                 MPT_BYTE_56_PA4_H_M,
1206                                 MPT_BYTE_56_PA4_H_S,
1207                                 cpu_to_le32((u32)(pages[i] >> PAGES_SHIFT_32)));
1208                         break;
1209                 case 5:
1210                         roce_set_field(mpt_entry->mpt_byte_56,
1211                                        MPT_BYTE_56_PA5_L_M,
1212                                        MPT_BYTE_56_PA5_L_S,
1213                                        cpu_to_le32((u32)(pages[i])));
1214                         roce_set_field(mpt_entry->mpt_byte_60,
1215                                 MPT_BYTE_60_PA5_H_M,
1216                                 MPT_BYTE_60_PA5_H_S,
1217                                 cpu_to_le32((u32)(pages[i] >> PAGES_SHIFT_24)));
1218                         break;
1219                 case 6:
1220                         roce_set_field(mpt_entry->mpt_byte_60,
1221                                        MPT_BYTE_60_PA6_L_M,
1222                                        MPT_BYTE_60_PA6_L_S,
1223                                        cpu_to_le32((u32)(pages[i])));
1224                         roce_set_field(mpt_entry->mpt_byte_64,
1225                                 MPT_BYTE_64_PA6_H_M,
1226                                 MPT_BYTE_64_PA6_H_S,
1227                                 cpu_to_le32((u32)(pages[i] >> PAGES_SHIFT_16)));
1228                         break;
1229                 default:
1230                         break;
1231                 }
1232         }
1233
1234         free_page((unsigned long) pages);
1235
1236         mpt_entry->pbl_addr_l = (u32)(mr->pbl_dma_addr);
1237
1238         roce_set_field(mpt_entry->mpt_byte_12, MPT_BYTE_12_PBL_ADDR_H_M,
1239                        MPT_BYTE_12_PBL_ADDR_H_S,
1240                        ((u32)(mr->pbl_dma_addr >> 32)));
1241
1242         return 0;
1243 }
1244
1245 static void *get_cqe(struct hns_roce_cq *hr_cq, int n)
1246 {
1247         return hns_roce_buf_offset(&hr_cq->hr_buf.hr_buf,
1248                                    n * HNS_ROCE_V1_CQE_ENTRY_SIZE);
1249 }
1250
1251 static void *get_sw_cqe(struct hns_roce_cq *hr_cq, int n)
1252 {
1253         struct hns_roce_cqe *hr_cqe = get_cqe(hr_cq, n & hr_cq->ib_cq.cqe);
1254
1255         /* Get cqe when Owner bit is Conversely with the MSB of cons_idx */
1256         return (roce_get_bit(hr_cqe->cqe_byte_4, CQE_BYTE_4_OWNER_S) ^
1257                 !!(n & (hr_cq->ib_cq.cqe + 1))) ? hr_cqe : NULL;
1258 }
1259
1260 static struct hns_roce_cqe *next_cqe_sw(struct hns_roce_cq *hr_cq)
1261 {
1262         return get_sw_cqe(hr_cq, hr_cq->cons_index);
1263 }
1264
1265 void hns_roce_v1_cq_set_ci(struct hns_roce_cq *hr_cq, u32 cons_index)
1266 {
1267         u32 doorbell[2];
1268
1269         doorbell[0] = cons_index & ((hr_cq->cq_depth << 1) - 1);
1270         roce_set_bit(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_HW_SYNS_S, 1);
1271         roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_M,
1272                        ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_S, 3);
1273         roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_M,
1274                        ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_S, 0);
1275         roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_M,
1276                        ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_S, hr_cq->cqn);
1277
1278         hns_roce_write64_k(doorbell, hr_cq->cq_db_l);
1279 }
1280
1281 static void __hns_roce_v1_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn,
1282                                    struct hns_roce_srq *srq)
1283 {
1284         struct hns_roce_cqe *cqe, *dest;
1285         u32 prod_index;
1286         int nfreed = 0;
1287         u8 owner_bit;
1288
1289         for (prod_index = hr_cq->cons_index; get_sw_cqe(hr_cq, prod_index);
1290              ++prod_index) {
1291                 if (prod_index == hr_cq->cons_index + hr_cq->ib_cq.cqe)
1292                         break;
1293         }
1294
1295         /*
1296         * Now backwards through the CQ, removing CQ entries
1297         * that match our QP by overwriting them with next entries.
1298         */
1299         while ((int) --prod_index - (int) hr_cq->cons_index >= 0) {
1300                 cqe = get_cqe(hr_cq, prod_index & hr_cq->ib_cq.cqe);
1301                 if ((roce_get_field(cqe->cqe_byte_16, CQE_BYTE_16_LOCAL_QPN_M,
1302                                      CQE_BYTE_16_LOCAL_QPN_S) &
1303                                      HNS_ROCE_CQE_QPN_MASK) == qpn) {
1304                         /* In v1 engine, not support SRQ */
1305                         ++nfreed;
1306                 } else if (nfreed) {
1307                         dest = get_cqe(hr_cq, (prod_index + nfreed) &
1308                                        hr_cq->ib_cq.cqe);
1309                         owner_bit = roce_get_bit(dest->cqe_byte_4,
1310                                                  CQE_BYTE_4_OWNER_S);
1311                         memcpy(dest, cqe, sizeof(*cqe));
1312                         roce_set_bit(dest->cqe_byte_4, CQE_BYTE_4_OWNER_S,
1313                                      owner_bit);
1314                 }
1315         }
1316
1317         if (nfreed) {
1318                 hr_cq->cons_index += nfreed;
1319                 /*
1320                 * Make sure update of buffer contents is done before
1321                 * updating consumer index.
1322                 */
1323                 wmb();
1324
1325                 hns_roce_v1_cq_set_ci(hr_cq, hr_cq->cons_index);
1326         }
1327 }
1328
1329 static void hns_roce_v1_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn,
1330                                  struct hns_roce_srq *srq)
1331 {
1332         spin_lock_irq(&hr_cq->lock);
1333         __hns_roce_v1_cq_clean(hr_cq, qpn, srq);
1334         spin_unlock_irq(&hr_cq->lock);
1335 }
1336
1337 void hns_roce_v1_write_cqc(struct hns_roce_dev *hr_dev,
1338                            struct hns_roce_cq *hr_cq, void *mb_buf, u64 *mtts,
1339                            dma_addr_t dma_handle, int nent, u32 vector)
1340 {
1341         struct hns_roce_cq_context *cq_context = NULL;
1342         void __iomem *tptr_addr;
1343
1344         cq_context = mb_buf;
1345         memset(cq_context, 0, sizeof(*cq_context));
1346
1347         tptr_addr = 0;
1348         hr_dev->priv_addr = tptr_addr;
1349         hr_cq->tptr_addr = tptr_addr;
1350
1351         /* Register cq_context members */
1352         roce_set_field(cq_context->cqc_byte_4,
1353                        CQ_CONTEXT_CQC_BYTE_4_CQC_STATE_M,
1354                        CQ_CONTEXT_CQC_BYTE_4_CQC_STATE_S, CQ_STATE_VALID);
1355         roce_set_field(cq_context->cqc_byte_4, CQ_CONTEXT_CQC_BYTE_4_CQN_M,
1356                        CQ_CONTEXT_CQC_BYTE_4_CQN_S, hr_cq->cqn);
1357         cq_context->cqc_byte_4 = cpu_to_le32(cq_context->cqc_byte_4);
1358
1359         cq_context->cq_bt_l = (u32)dma_handle;
1360         cq_context->cq_bt_l = cpu_to_le32(cq_context->cq_bt_l);
1361
1362         roce_set_field(cq_context->cqc_byte_12,
1363                        CQ_CONTEXT_CQC_BYTE_12_CQ_BT_H_M,
1364                        CQ_CONTEXT_CQC_BYTE_12_CQ_BT_H_S,
1365                        ((u64)dma_handle >> 32));
1366         roce_set_field(cq_context->cqc_byte_12,
1367                        CQ_CONTEXT_CQC_BYTE_12_CQ_CQE_SHIFT_M,
1368                        CQ_CONTEXT_CQC_BYTE_12_CQ_CQE_SHIFT_S,
1369                        ilog2((unsigned int)nent));
1370         roce_set_field(cq_context->cqc_byte_12, CQ_CONTEXT_CQC_BYTE_12_CEQN_M,
1371                        CQ_CONTEXT_CQC_BYTE_12_CEQN_S, vector);
1372         cq_context->cqc_byte_12 = cpu_to_le32(cq_context->cqc_byte_12);
1373
1374         cq_context->cur_cqe_ba0_l = (u32)(mtts[0]);
1375         cq_context->cur_cqe_ba0_l = cpu_to_le32(cq_context->cur_cqe_ba0_l);
1376
1377         roce_set_field(cq_context->cqc_byte_20,
1378                        CQ_CONTEXT_CQC_BYTE_20_CUR_CQE_BA0_H_M,
1379                        CQ_CONTEXT_CQC_BYTE_20_CUR_CQE_BA0_H_S,
1380                        cpu_to_le32((mtts[0]) >> 32));
1381         /* Dedicated hardware, directly set 0 */
1382         roce_set_field(cq_context->cqc_byte_20,
1383                        CQ_CONTEXT_CQC_BYTE_20_CQ_CUR_INDEX_M,
1384                        CQ_CONTEXT_CQC_BYTE_20_CQ_CUR_INDEX_S, 0);
1385         /**
1386          * 44 = 32 + 12, When evaluating addr to hardware, shift 12 because of
1387          * using 4K page, and shift more 32 because of
1388          * caculating the high 32 bit value evaluated to hardware.
1389          */
1390         roce_set_field(cq_context->cqc_byte_20,
1391                        CQ_CONTEXT_CQC_BYTE_20_CQE_TPTR_ADDR_H_M,
1392                        CQ_CONTEXT_CQC_BYTE_20_CQE_TPTR_ADDR_H_S,
1393                        (u64)tptr_addr >> 44);
1394         cq_context->cqc_byte_20 = cpu_to_le32(cq_context->cqc_byte_20);
1395
1396         cq_context->cqe_tptr_addr_l = (u32)((u64)tptr_addr >> 12);
1397
1398         roce_set_field(cq_context->cqc_byte_32,
1399                        CQ_CONTEXT_CQC_BYTE_32_CUR_CQE_BA1_H_M,
1400                        CQ_CONTEXT_CQC_BYTE_32_CUR_CQE_BA1_H_S, 0);
1401         roce_set_bit(cq_context->cqc_byte_32,
1402                      CQ_CONTEXT_CQC_BYTE_32_SE_FLAG_S, 0);
1403         roce_set_bit(cq_context->cqc_byte_32,
1404                      CQ_CONTEXT_CQC_BYTE_32_CE_FLAG_S, 0);
1405         roce_set_bit(cq_context->cqc_byte_32,
1406                      CQ_CONTEXT_CQC_BYTE_32_NOTIFICATION_FLAG_S, 0);
1407         roce_set_bit(cq_context->cqc_byte_32,
1408                      CQ_CQNTEXT_CQC_BYTE_32_TYPE_OF_COMPLETION_NOTIFICATION_S,
1409                      0);
1410         /*The initial value of cq's ci is 0 */
1411         roce_set_field(cq_context->cqc_byte_32,
1412                        CQ_CONTEXT_CQC_BYTE_32_CQ_CONS_IDX_M,
1413                        CQ_CONTEXT_CQC_BYTE_32_CQ_CONS_IDX_S, 0);
1414         cq_context->cqc_byte_32 = cpu_to_le32(cq_context->cqc_byte_32);
1415 }
1416
1417 int hns_roce_v1_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
1418 {
1419         struct hns_roce_cq *hr_cq = to_hr_cq(ibcq);
1420         u32 notification_flag;
1421         u32 doorbell[2];
1422         int ret = 0;
1423
1424         notification_flag = (flags & IB_CQ_SOLICITED_MASK) ==
1425                             IB_CQ_SOLICITED ? CQ_DB_REQ_NOT : CQ_DB_REQ_NOT_SOL;
1426         /*
1427         * flags = 0; Notification Flag = 1, next
1428         * flags = 1; Notification Flag = 0, solocited
1429         */
1430         doorbell[0] = hr_cq->cons_index & ((hr_cq->cq_depth << 1) - 1);
1431         roce_set_bit(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_HW_SYNS_S, 1);
1432         roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_M,
1433                        ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_S, 3);
1434         roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_M,
1435                        ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_S, 1);
1436         roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_M,
1437                        ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_S,
1438                        hr_cq->cqn | notification_flag);
1439
1440         hns_roce_write64_k(doorbell, hr_cq->cq_db_l);
1441
1442         return ret;
1443 }
1444
1445 static int hns_roce_v1_poll_one(struct hns_roce_cq *hr_cq,
1446                                 struct hns_roce_qp **cur_qp, struct ib_wc *wc)
1447 {
1448         int qpn;
1449         int is_send;
1450         u16 wqe_ctr;
1451         u32 status;
1452         u32 opcode;
1453         struct hns_roce_cqe *cqe;
1454         struct hns_roce_qp *hr_qp;
1455         struct hns_roce_wq *wq;
1456         struct hns_roce_wqe_ctrl_seg *sq_wqe;
1457         struct hns_roce_dev *hr_dev = to_hr_dev(hr_cq->ib_cq.device);
1458         struct device *dev = &hr_dev->pdev->dev;
1459
1460         /* Find cqe according consumer index */
1461         cqe = next_cqe_sw(hr_cq);
1462         if (!cqe)
1463                 return -EAGAIN;
1464
1465         ++hr_cq->cons_index;
1466         /* Memory barrier */
1467         rmb();
1468         /* 0->SQ, 1->RQ */
1469         is_send  = !(roce_get_bit(cqe->cqe_byte_4, CQE_BYTE_4_SQ_RQ_FLAG_S));
1470
1471         /* Local_qpn in UD cqe is always 1, so it needs to compute new qpn */
1472         if (roce_get_field(cqe->cqe_byte_16, CQE_BYTE_16_LOCAL_QPN_M,
1473                            CQE_BYTE_16_LOCAL_QPN_S) <= 1) {
1474                 qpn = roce_get_field(cqe->cqe_byte_20, CQE_BYTE_20_PORT_NUM_M,
1475                                      CQE_BYTE_20_PORT_NUM_S) +
1476                       roce_get_field(cqe->cqe_byte_16, CQE_BYTE_16_LOCAL_QPN_M,
1477                                      CQE_BYTE_16_LOCAL_QPN_S) *
1478                                      HNS_ROCE_MAX_PORTS;
1479         } else {
1480                 qpn = roce_get_field(cqe->cqe_byte_16, CQE_BYTE_16_LOCAL_QPN_M,
1481                                      CQE_BYTE_16_LOCAL_QPN_S);
1482         }
1483
1484         if (!*cur_qp || (qpn & HNS_ROCE_CQE_QPN_MASK) != (*cur_qp)->qpn) {
1485                 hr_qp = __hns_roce_qp_lookup(hr_dev, qpn);
1486                 if (unlikely(!hr_qp)) {
1487                         dev_err(dev, "CQ %06lx with entry for unknown QPN %06x\n",
1488                                 hr_cq->cqn, (qpn & HNS_ROCE_CQE_QPN_MASK));
1489                         return -EINVAL;
1490                 }
1491
1492                 *cur_qp = hr_qp;
1493         }
1494
1495         wc->qp = &(*cur_qp)->ibqp;
1496         wc->vendor_err = 0;
1497
1498         status = roce_get_field(cqe->cqe_byte_4,
1499                                 CQE_BYTE_4_STATUS_OF_THE_OPERATION_M,
1500                                 CQE_BYTE_4_STATUS_OF_THE_OPERATION_S) &
1501                                 HNS_ROCE_CQE_STATUS_MASK;
1502         switch (status) {
1503         case HNS_ROCE_CQE_SUCCESS:
1504                 wc->status = IB_WC_SUCCESS;
1505                 break;
1506         case HNS_ROCE_CQE_SYNDROME_LOCAL_LENGTH_ERR:
1507                 wc->status = IB_WC_LOC_LEN_ERR;
1508                 break;
1509         case HNS_ROCE_CQE_SYNDROME_LOCAL_QP_OP_ERR:
1510                 wc->status = IB_WC_LOC_QP_OP_ERR;
1511                 break;
1512         case HNS_ROCE_CQE_SYNDROME_LOCAL_PROT_ERR:
1513                 wc->status = IB_WC_LOC_PROT_ERR;
1514                 break;
1515         case HNS_ROCE_CQE_SYNDROME_WR_FLUSH_ERR:
1516                 wc->status = IB_WC_WR_FLUSH_ERR;
1517                 break;
1518         case HNS_ROCE_CQE_SYNDROME_MEM_MANAGE_OPERATE_ERR:
1519                 wc->status = IB_WC_MW_BIND_ERR;
1520                 break;
1521         case HNS_ROCE_CQE_SYNDROME_BAD_RESP_ERR:
1522                 wc->status = IB_WC_BAD_RESP_ERR;
1523                 break;
1524         case HNS_ROCE_CQE_SYNDROME_LOCAL_ACCESS_ERR:
1525                 wc->status = IB_WC_LOC_ACCESS_ERR;
1526                 break;
1527         case HNS_ROCE_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR:
1528                 wc->status = IB_WC_REM_INV_REQ_ERR;
1529                 break;
1530         case HNS_ROCE_CQE_SYNDROME_REMOTE_ACCESS_ERR:
1531                 wc->status = IB_WC_REM_ACCESS_ERR;
1532                 break;
1533         case HNS_ROCE_CQE_SYNDROME_REMOTE_OP_ERR:
1534                 wc->status = IB_WC_REM_OP_ERR;
1535                 break;
1536         case HNS_ROCE_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR:
1537                 wc->status = IB_WC_RETRY_EXC_ERR;
1538                 break;
1539         case HNS_ROCE_CQE_SYNDROME_RNR_RETRY_EXC_ERR:
1540                 wc->status = IB_WC_RNR_RETRY_EXC_ERR;
1541                 break;
1542         default:
1543                 wc->status = IB_WC_GENERAL_ERR;
1544                 break;
1545         }
1546
1547         /* CQE status error, directly return */
1548         if (wc->status != IB_WC_SUCCESS)
1549                 return 0;
1550
1551         if (is_send) {
1552                 /* SQ conrespond to CQE */
1553                 sq_wqe = get_send_wqe(*cur_qp, roce_get_field(cqe->cqe_byte_4,
1554                                                 CQE_BYTE_4_WQE_INDEX_M,
1555                                                 CQE_BYTE_4_WQE_INDEX_S)&
1556                                                 ((*cur_qp)->sq.wqe_cnt-1));
1557                 switch (sq_wqe->flag & HNS_ROCE_WQE_OPCODE_MASK) {
1558                 case HNS_ROCE_WQE_OPCODE_SEND:
1559                         wc->opcode = IB_WC_SEND;
1560                         break;
1561                 case HNS_ROCE_WQE_OPCODE_RDMA_READ:
1562                         wc->opcode = IB_WC_RDMA_READ;
1563                         wc->byte_len = le32_to_cpu(cqe->byte_cnt);
1564                         break;
1565                 case HNS_ROCE_WQE_OPCODE_RDMA_WRITE:
1566                         wc->opcode = IB_WC_RDMA_WRITE;
1567                         break;
1568                 case HNS_ROCE_WQE_OPCODE_LOCAL_INV:
1569                         wc->opcode = IB_WC_LOCAL_INV;
1570                         break;
1571                 case HNS_ROCE_WQE_OPCODE_UD_SEND:
1572                         wc->opcode = IB_WC_SEND;
1573                         break;
1574                 default:
1575                         wc->status = IB_WC_GENERAL_ERR;
1576                         break;
1577                 }
1578                 wc->wc_flags = (sq_wqe->flag & HNS_ROCE_WQE_IMM ?
1579                                 IB_WC_WITH_IMM : 0);
1580
1581                 wq = &(*cur_qp)->sq;
1582                 if ((*cur_qp)->sq_signal_bits) {
1583                         /*
1584                         * If sg_signal_bit is 1,
1585                         * firstly tail pointer updated to wqe
1586                         * which current cqe correspond to
1587                         */
1588                         wqe_ctr = (u16)roce_get_field(cqe->cqe_byte_4,
1589                                                       CQE_BYTE_4_WQE_INDEX_M,
1590                                                       CQE_BYTE_4_WQE_INDEX_S);
1591                         wq->tail += (wqe_ctr - (u16)wq->tail) &
1592                                     (wq->wqe_cnt - 1);
1593                 }
1594                 wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
1595                 ++wq->tail;
1596                 } else {
1597                 /* RQ conrespond to CQE */
1598                 wc->byte_len = le32_to_cpu(cqe->byte_cnt);
1599                 opcode = roce_get_field(cqe->cqe_byte_4,
1600                                         CQE_BYTE_4_OPERATION_TYPE_M,
1601                                         CQE_BYTE_4_OPERATION_TYPE_S) &
1602                                         HNS_ROCE_CQE_OPCODE_MASK;
1603                 switch (opcode) {
1604                 case HNS_ROCE_OPCODE_RDMA_WITH_IMM_RECEIVE:
1605                         wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
1606                         wc->wc_flags = IB_WC_WITH_IMM;
1607                         wc->ex.imm_data = le32_to_cpu(cqe->immediate_data);
1608                         break;
1609                 case HNS_ROCE_OPCODE_SEND_DATA_RECEIVE:
1610                         if (roce_get_bit(cqe->cqe_byte_4,
1611                                          CQE_BYTE_4_IMM_INDICATOR_S)) {
1612                                 wc->opcode = IB_WC_RECV;
1613                                 wc->wc_flags = IB_WC_WITH_IMM;
1614                                 wc->ex.imm_data = le32_to_cpu(
1615                                                   cqe->immediate_data);
1616                         } else {
1617                                 wc->opcode = IB_WC_RECV;
1618                                 wc->wc_flags = 0;
1619                         }
1620                         break;
1621                 default:
1622                         wc->status = IB_WC_GENERAL_ERR;
1623                         break;
1624                 }
1625
1626                 /* Update tail pointer, record wr_id */
1627                 wq = &(*cur_qp)->rq;
1628                 wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
1629                 ++wq->tail;
1630                 wc->sl = (u8)roce_get_field(cqe->cqe_byte_20, CQE_BYTE_20_SL_M,
1631                                             CQE_BYTE_20_SL_S);
1632                 wc->src_qp = (u8)roce_get_field(cqe->cqe_byte_20,
1633                                                 CQE_BYTE_20_REMOTE_QPN_M,
1634                                                 CQE_BYTE_20_REMOTE_QPN_S);
1635                 wc->wc_flags |= (roce_get_bit(cqe->cqe_byte_20,
1636                                               CQE_BYTE_20_GRH_PRESENT_S) ?
1637                                               IB_WC_GRH : 0);
1638                 wc->pkey_index = (u16)roce_get_field(cqe->cqe_byte_28,
1639                                                      CQE_BYTE_28_P_KEY_IDX_M,
1640                                                      CQE_BYTE_28_P_KEY_IDX_S);
1641         }
1642
1643         return 0;
1644 }
1645
1646 int hns_roce_v1_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
1647 {
1648         struct hns_roce_cq *hr_cq = to_hr_cq(ibcq);
1649         struct hns_roce_qp *cur_qp = NULL;
1650         unsigned long flags;
1651         int npolled;
1652         int ret = 0;
1653
1654         spin_lock_irqsave(&hr_cq->lock, flags);
1655
1656         for (npolled = 0; npolled < num_entries; ++npolled) {
1657                 ret = hns_roce_v1_poll_one(hr_cq, &cur_qp, wc + npolled);
1658                 if (ret)
1659                         break;
1660         }
1661
1662         if (npolled)
1663                 hns_roce_v1_cq_set_ci(hr_cq, hr_cq->cons_index);
1664
1665         spin_unlock_irqrestore(&hr_cq->lock, flags);
1666
1667         if (ret == 0 || ret == -EAGAIN)
1668                 return npolled;
1669         else
1670                 return ret;
1671 }
1672
1673 int hns_roce_v1_clear_hem(struct hns_roce_dev *hr_dev,
1674                 struct hns_roce_hem_table *table, int obj)
1675 {
1676         struct device *dev = &hr_dev->pdev->dev;
1677         struct hns_roce_v1_priv *priv;
1678         unsigned long end = 0, flags = 0;
1679         uint32_t bt_cmd_val[2] = {0};
1680         void __iomem *bt_cmd;
1681         u64 bt_ba = 0;
1682
1683         priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
1684
1685         switch (table->type) {
1686         case HEM_TYPE_QPC:
1687                 roce_set_field(bt_cmd_val[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
1688                         ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, HEM_TYPE_QPC);
1689                 bt_ba = priv->bt_table.qpc_buf.map >> 12;
1690                 break;
1691         case HEM_TYPE_MTPT:
1692                 roce_set_field(bt_cmd_val[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
1693                         ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, HEM_TYPE_MTPT);
1694                 bt_ba = priv->bt_table.mtpt_buf.map >> 12;
1695                 break;
1696         case HEM_TYPE_CQC:
1697                 roce_set_field(bt_cmd_val[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
1698                         ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, HEM_TYPE_CQC);
1699                 bt_ba = priv->bt_table.cqc_buf.map >> 12;
1700                 break;
1701         case HEM_TYPE_SRQC:
1702                 dev_dbg(dev, "HEM_TYPE_SRQC not support.\n");
1703                 return -EINVAL;
1704         default:
1705                 return 0;
1706         }
1707         roce_set_field(bt_cmd_val[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_M,
1708                 ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_S, obj);
1709         roce_set_bit(bt_cmd_val[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_S, 0);
1710         roce_set_bit(bt_cmd_val[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_HW_SYNS_S, 1);
1711
1712         spin_lock_irqsave(&hr_dev->bt_cmd_lock, flags);
1713
1714         bt_cmd = hr_dev->reg_base + ROCEE_BT_CMD_H_REG;
1715
1716         end = msecs_to_jiffies(HW_SYNC_TIMEOUT_MSECS) + jiffies;
1717         while (1) {
1718                 if (readl(bt_cmd) >> BT_CMD_SYNC_SHIFT) {
1719                         if (!(time_before(jiffies, end))) {
1720                                 dev_err(dev, "Write bt_cmd err,hw_sync is not zero.\n");
1721                                 spin_unlock_irqrestore(&hr_dev->bt_cmd_lock,
1722                                         flags);
1723                                 return -EBUSY;
1724                         }
1725                 } else {
1726                         break;
1727                 }
1728                 msleep(HW_SYNC_SLEEP_TIME_INTERVAL);
1729         }
1730
1731         bt_cmd_val[0] = (uint32_t)bt_ba;
1732         roce_set_field(bt_cmd_val[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_M,
1733                 ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_S, bt_ba >> 32);
1734         hns_roce_write64_k(bt_cmd_val, hr_dev->reg_base + ROCEE_BT_CMD_L_REG);
1735
1736         spin_unlock_irqrestore(&hr_dev->bt_cmd_lock, flags);
1737
1738         return 0;
1739 }
1740
1741 static int hns_roce_v1_qp_modify(struct hns_roce_dev *hr_dev,
1742                                  struct hns_roce_mtt *mtt,
1743                                  enum hns_roce_qp_state cur_state,
1744                                  enum hns_roce_qp_state new_state,
1745                                  struct hns_roce_qp_context *context,
1746                                  struct hns_roce_qp *hr_qp)
1747 {
1748         static const u16
1749         op[HNS_ROCE_QP_NUM_STATE][HNS_ROCE_QP_NUM_STATE] = {
1750                 [HNS_ROCE_QP_STATE_RST] = {
1751                 [HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP,
1752                 [HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP,
1753                 [HNS_ROCE_QP_STATE_INIT] = HNS_ROCE_CMD_RST2INIT_QP,
1754                 },
1755                 [HNS_ROCE_QP_STATE_INIT] = {
1756                 [HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP,
1757                 [HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP,
1758                 /* Note: In v1 engine, HW doesn't support RST2INIT.
1759                  * We use RST2INIT cmd instead of INIT2INIT.
1760                  */
1761                 [HNS_ROCE_QP_STATE_INIT] = HNS_ROCE_CMD_RST2INIT_QP,
1762                 [HNS_ROCE_QP_STATE_RTR] = HNS_ROCE_CMD_INIT2RTR_QP,
1763                 },
1764                 [HNS_ROCE_QP_STATE_RTR] = {
1765                 [HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP,
1766                 [HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP,
1767                 [HNS_ROCE_QP_STATE_RTS] = HNS_ROCE_CMD_RTR2RTS_QP,
1768                 },
1769                 [HNS_ROCE_QP_STATE_RTS] = {
1770                 [HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP,
1771                 [HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP,
1772                 [HNS_ROCE_QP_STATE_RTS] = HNS_ROCE_CMD_RTS2RTS_QP,
1773                 [HNS_ROCE_QP_STATE_SQD] = HNS_ROCE_CMD_RTS2SQD_QP,
1774                 },
1775                 [HNS_ROCE_QP_STATE_SQD] = {
1776                 [HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP,
1777                 [HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP,
1778                 [HNS_ROCE_QP_STATE_RTS] = HNS_ROCE_CMD_SQD2RTS_QP,
1779                 [HNS_ROCE_QP_STATE_SQD] = HNS_ROCE_CMD_SQD2SQD_QP,
1780                 },
1781                 [HNS_ROCE_QP_STATE_ERR] = {
1782                 [HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP,
1783                 [HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP,
1784                 }
1785         };
1786
1787         struct hns_roce_cmd_mailbox *mailbox;
1788         struct device *dev = &hr_dev->pdev->dev;
1789         int ret = 0;
1790
1791         if (cur_state >= HNS_ROCE_QP_NUM_STATE ||
1792             new_state >= HNS_ROCE_QP_NUM_STATE ||
1793             !op[cur_state][new_state]) {
1794                 dev_err(dev, "[modify_qp]not support state %d to %d\n",
1795                         cur_state, new_state);
1796                 return -EINVAL;
1797         }
1798
1799         if (op[cur_state][new_state] == HNS_ROCE_CMD_2RST_QP)
1800                 return hns_roce_cmd_mbox(hr_dev, 0, 0, hr_qp->qpn, 2,
1801                                          HNS_ROCE_CMD_2RST_QP,
1802                                          HNS_ROCE_CMD_TIME_CLASS_A);
1803
1804         if (op[cur_state][new_state] == HNS_ROCE_CMD_2ERR_QP)
1805                 return hns_roce_cmd_mbox(hr_dev, 0, 0, hr_qp->qpn, 2,
1806                                          HNS_ROCE_CMD_2ERR_QP,
1807                                          HNS_ROCE_CMD_TIME_CLASS_A);
1808
1809         mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
1810         if (IS_ERR(mailbox))
1811                 return PTR_ERR(mailbox);
1812
1813         memcpy(mailbox->buf, context, sizeof(*context));
1814
1815         ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, hr_qp->qpn, 0,
1816                                 op[cur_state][new_state],
1817                                 HNS_ROCE_CMD_TIME_CLASS_C);
1818
1819         hns_roce_free_cmd_mailbox(hr_dev, mailbox);
1820         return ret;
1821 }
1822
1823 static int hns_roce_v1_m_sqp(struct ib_qp *ibqp, const struct ib_qp_attr *attr,
1824                              int attr_mask, enum ib_qp_state cur_state,
1825                              enum ib_qp_state new_state)
1826 {
1827         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
1828         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
1829         struct hns_roce_sqp_context *context;
1830         struct device *dev = &hr_dev->pdev->dev;
1831         dma_addr_t dma_handle = 0;
1832         int rq_pa_start;
1833         u32 reg_val;
1834         u64 *mtts;
1835         u32 *addr;
1836
1837         context = kzalloc(sizeof(*context), GFP_KERNEL);
1838         if (!context)
1839                 return -ENOMEM;
1840
1841         /* Search QP buf's MTTs */
1842         mtts = hns_roce_table_find(&hr_dev->mr_table.mtt_table,
1843                                    hr_qp->mtt.first_seg, &dma_handle);
1844         if (!mtts) {
1845                 dev_err(dev, "qp buf pa find failed\n");
1846                 goto out;
1847         }
1848
1849         if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
1850                 roce_set_field(context->qp1c_bytes_4,
1851                                QP1C_BYTES_4_SQ_WQE_SHIFT_M,
1852                                QP1C_BYTES_4_SQ_WQE_SHIFT_S,
1853                                ilog2((unsigned int)hr_qp->sq.wqe_cnt));
1854                 roce_set_field(context->qp1c_bytes_4,
1855                                QP1C_BYTES_4_RQ_WQE_SHIFT_M,
1856                                QP1C_BYTES_4_RQ_WQE_SHIFT_S,
1857                                ilog2((unsigned int)hr_qp->rq.wqe_cnt));
1858                 roce_set_field(context->qp1c_bytes_4, QP1C_BYTES_4_PD_M,
1859                                QP1C_BYTES_4_PD_S, to_hr_pd(ibqp->pd)->pdn);
1860
1861                 context->sq_rq_bt_l = (u32)(dma_handle);
1862                 roce_set_field(context->qp1c_bytes_12,
1863                                QP1C_BYTES_12_SQ_RQ_BT_H_M,
1864                                QP1C_BYTES_12_SQ_RQ_BT_H_S,
1865                                ((u32)(dma_handle >> 32)));
1866
1867                 roce_set_field(context->qp1c_bytes_16, QP1C_BYTES_16_RQ_HEAD_M,
1868                                QP1C_BYTES_16_RQ_HEAD_S, hr_qp->rq.head);
1869                 roce_set_field(context->qp1c_bytes_16, QP1C_BYTES_16_PORT_NUM_M,
1870                                QP1C_BYTES_16_PORT_NUM_S, hr_qp->phy_port);
1871                 roce_set_bit(context->qp1c_bytes_16,
1872                              QP1C_BYTES_16_SIGNALING_TYPE_S,
1873                              hr_qp->sq_signal_bits);
1874                 roce_set_bit(context->qp1c_bytes_16, QP1C_BYTES_16_RQ_BA_FLG_S,
1875                              1);
1876                 roce_set_bit(context->qp1c_bytes_16, QP1C_BYTES_16_SQ_BA_FLG_S,
1877                              1);
1878                 roce_set_bit(context->qp1c_bytes_16, QP1C_BYTES_16_QP1_ERR_S,
1879                              0);
1880
1881                 roce_set_field(context->qp1c_bytes_20, QP1C_BYTES_20_SQ_HEAD_M,
1882                                QP1C_BYTES_20_SQ_HEAD_S, hr_qp->sq.head);
1883                 roce_set_field(context->qp1c_bytes_20, QP1C_BYTES_20_PKEY_IDX_M,
1884                                QP1C_BYTES_20_PKEY_IDX_S, attr->pkey_index);
1885
1886                 rq_pa_start = (u32)hr_qp->rq.offset / PAGE_SIZE;
1887                 context->cur_rq_wqe_ba_l = (u32)(mtts[rq_pa_start]);
1888
1889                 roce_set_field(context->qp1c_bytes_28,
1890                                QP1C_BYTES_28_CUR_RQ_WQE_BA_H_M,
1891                                QP1C_BYTES_28_CUR_RQ_WQE_BA_H_S,
1892                                (mtts[rq_pa_start]) >> 32);
1893                 roce_set_field(context->qp1c_bytes_28,
1894                                QP1C_BYTES_28_RQ_CUR_IDX_M,
1895                                QP1C_BYTES_28_RQ_CUR_IDX_S, 0);
1896
1897                 roce_set_field(context->qp1c_bytes_32,
1898                                QP1C_BYTES_32_RX_CQ_NUM_M,
1899                                QP1C_BYTES_32_RX_CQ_NUM_S,
1900                                to_hr_cq(ibqp->recv_cq)->cqn);
1901                 roce_set_field(context->qp1c_bytes_32,
1902                                QP1C_BYTES_32_TX_CQ_NUM_M,
1903                                QP1C_BYTES_32_TX_CQ_NUM_S,
1904                                to_hr_cq(ibqp->send_cq)->cqn);
1905
1906                 context->cur_sq_wqe_ba_l  = (u32)mtts[0];
1907
1908                 roce_set_field(context->qp1c_bytes_40,
1909                                QP1C_BYTES_40_CUR_SQ_WQE_BA_H_M,
1910                                QP1C_BYTES_40_CUR_SQ_WQE_BA_H_S,
1911                                (mtts[0]) >> 32);
1912                 roce_set_field(context->qp1c_bytes_40,
1913                                QP1C_BYTES_40_SQ_CUR_IDX_M,
1914                                QP1C_BYTES_40_SQ_CUR_IDX_S, 0);
1915
1916                 /* Copy context to QP1C register */
1917                 addr = (u32 *)(hr_dev->reg_base + ROCEE_QP1C_CFG0_0_REG +
1918                         hr_qp->phy_port * sizeof(*context));
1919
1920                 writel(context->qp1c_bytes_4, addr);
1921                 writel(context->sq_rq_bt_l, addr + 1);
1922                 writel(context->qp1c_bytes_12, addr + 2);
1923                 writel(context->qp1c_bytes_16, addr + 3);
1924                 writel(context->qp1c_bytes_20, addr + 4);
1925                 writel(context->cur_rq_wqe_ba_l, addr + 5);
1926                 writel(context->qp1c_bytes_28, addr + 6);
1927                 writel(context->qp1c_bytes_32, addr + 7);
1928                 writel(context->cur_sq_wqe_ba_l, addr + 8);
1929                 writel(context->qp1c_bytes_40, addr + 9);
1930         }
1931
1932         /* Modify QP1C status */
1933         reg_val = roce_read(hr_dev, ROCEE_QP1C_CFG0_0_REG +
1934                             hr_qp->phy_port * sizeof(*context));
1935         roce_set_field(reg_val, ROCEE_QP1C_CFG0_0_ROCEE_QP1C_QP_ST_M,
1936                        ROCEE_QP1C_CFG0_0_ROCEE_QP1C_QP_ST_S, new_state);
1937         roce_write(hr_dev, ROCEE_QP1C_CFG0_0_REG +
1938                     hr_qp->phy_port * sizeof(*context), reg_val);
1939
1940         hr_qp->state = new_state;
1941         if (new_state == IB_QPS_RESET) {
1942                 hns_roce_v1_cq_clean(to_hr_cq(ibqp->recv_cq), hr_qp->qpn,
1943                                      ibqp->srq ? to_hr_srq(ibqp->srq) : NULL);
1944                 if (ibqp->send_cq != ibqp->recv_cq)
1945                         hns_roce_v1_cq_clean(to_hr_cq(ibqp->send_cq),
1946                                              hr_qp->qpn, NULL);
1947
1948                 hr_qp->rq.head = 0;
1949                 hr_qp->rq.tail = 0;
1950                 hr_qp->sq.head = 0;
1951                 hr_qp->sq.tail = 0;
1952                 hr_qp->sq_next_wqe = 0;
1953         }
1954
1955         kfree(context);
1956         return 0;
1957
1958 out:
1959         kfree(context);
1960         return -EINVAL;
1961 }
1962
1963 static int hns_roce_v1_m_qp(struct ib_qp *ibqp, const struct ib_qp_attr *attr,
1964                             int attr_mask, enum ib_qp_state cur_state,
1965                             enum ib_qp_state new_state)
1966 {
1967         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
1968         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
1969         struct device *dev = &hr_dev->pdev->dev;
1970         struct hns_roce_qp_context *context;
1971         dma_addr_t dma_handle_2 = 0;
1972         dma_addr_t dma_handle = 0;
1973         uint32_t doorbell[2] = {0};
1974         int rq_pa_start = 0;
1975         u64 *mtts_2 = NULL;
1976         int ret = -EINVAL;
1977         u64 *mtts = NULL;
1978         int port;
1979         u8 *dmac;
1980         u8 *smac;
1981
1982         context = kzalloc(sizeof(*context), GFP_KERNEL);
1983         if (!context)
1984                 return -ENOMEM;
1985
1986         /* Search qp buf's mtts */
1987         mtts = hns_roce_table_find(&hr_dev->mr_table.mtt_table,
1988                                    hr_qp->mtt.first_seg, &dma_handle);
1989         if (mtts == NULL) {
1990                 dev_err(dev, "qp buf pa find failed\n");
1991                 goto out;
1992         }
1993
1994         /* Search IRRL's mtts */
1995         mtts_2 = hns_roce_table_find(&hr_dev->qp_table.irrl_table, hr_qp->qpn,
1996                                      &dma_handle_2);
1997         if (mtts_2 == NULL) {
1998                 dev_err(dev, "qp irrl_table find failed\n");
1999                 goto out;
2000         }
2001
2002         /*
2003         *Reset to init
2004         *       Mandatory param:
2005         *       IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT | IB_QP_ACCESS_FLAGS
2006         *       Optional param: NA
2007         */
2008         if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
2009                 roce_set_field(context->qpc_bytes_4,
2010                                QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_M,
2011                                QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_S,
2012                                to_hr_qp_type(hr_qp->ibqp.qp_type));
2013
2014                 roce_set_bit(context->qpc_bytes_4,
2015                              QP_CONTEXT_QPC_BYTE_4_ENABLE_FPMR_S, 0);
2016                 roce_set_bit(context->qpc_bytes_4,
2017                              QP_CONTEXT_QPC_BYTE_4_RDMA_READ_ENABLE_S,
2018                              !!(attr->qp_access_flags & IB_ACCESS_REMOTE_READ));
2019                 roce_set_bit(context->qpc_bytes_4,
2020                              QP_CONTEXT_QPC_BYTE_4_RDMA_WRITE_ENABLE_S,
2021                              !!(attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE)
2022                              );
2023                 roce_set_bit(context->qpc_bytes_4,
2024                              QP_CONTEXT_QPC_BYTE_4_ATOMIC_OPERATION_ENABLE_S,
2025                              !!(attr->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC)
2026                              );
2027                 roce_set_bit(context->qpc_bytes_4,
2028                              QP_CONTEXT_QPC_BYTE_4_RDMAR_USE_S, 1);
2029                 roce_set_field(context->qpc_bytes_4,
2030                                QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_M,
2031                                QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_S,
2032                                ilog2((unsigned int)hr_qp->sq.wqe_cnt));
2033                 roce_set_field(context->qpc_bytes_4,
2034                                QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_M,
2035                                QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_S,
2036                                ilog2((unsigned int)hr_qp->rq.wqe_cnt));
2037                 roce_set_field(context->qpc_bytes_4,
2038                                QP_CONTEXT_QPC_BYTES_4_PD_M,
2039                                QP_CONTEXT_QPC_BYTES_4_PD_S,
2040                                to_hr_pd(ibqp->pd)->pdn);
2041                 hr_qp->access_flags = attr->qp_access_flags;
2042                 roce_set_field(context->qpc_bytes_8,
2043                                QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_M,
2044                                QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_S,
2045                                to_hr_cq(ibqp->send_cq)->cqn);
2046                 roce_set_field(context->qpc_bytes_8,
2047                                QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_M,
2048                                QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_S,
2049                                to_hr_cq(ibqp->recv_cq)->cqn);
2050
2051                 if (ibqp->srq)
2052                         roce_set_field(context->qpc_bytes_12,
2053                                        QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_M,
2054                                        QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_S,
2055                                        to_hr_srq(ibqp->srq)->srqn);
2056
2057                 roce_set_field(context->qpc_bytes_12,
2058                                QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_M,
2059                                QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S,
2060                                attr->pkey_index);
2061                 hr_qp->pkey_index = attr->pkey_index;
2062                 roce_set_field(context->qpc_bytes_16,
2063                                QP_CONTEXT_QPC_BYTES_16_QP_NUM_M,
2064                                QP_CONTEXT_QPC_BYTES_16_QP_NUM_S, hr_qp->qpn);
2065
2066         } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_INIT) {
2067                 roce_set_field(context->qpc_bytes_4,
2068                                QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_M,
2069                                QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_S,
2070                                to_hr_qp_type(hr_qp->ibqp.qp_type));
2071                 roce_set_bit(context->qpc_bytes_4,
2072                              QP_CONTEXT_QPC_BYTE_4_ENABLE_FPMR_S, 0);
2073                 if (attr_mask & IB_QP_ACCESS_FLAGS) {
2074                         roce_set_bit(context->qpc_bytes_4,
2075                                      QP_CONTEXT_QPC_BYTE_4_RDMA_READ_ENABLE_S,
2076                                      !!(attr->qp_access_flags &
2077                                      IB_ACCESS_REMOTE_READ));
2078                         roce_set_bit(context->qpc_bytes_4,
2079                                      QP_CONTEXT_QPC_BYTE_4_RDMA_WRITE_ENABLE_S,
2080                                      !!(attr->qp_access_flags &
2081                                      IB_ACCESS_REMOTE_WRITE));
2082                 } else {
2083                         roce_set_bit(context->qpc_bytes_4,
2084                                      QP_CONTEXT_QPC_BYTE_4_RDMA_READ_ENABLE_S,
2085                                      !!(hr_qp->access_flags &
2086                                      IB_ACCESS_REMOTE_READ));
2087                         roce_set_bit(context->qpc_bytes_4,
2088                                      QP_CONTEXT_QPC_BYTE_4_RDMA_WRITE_ENABLE_S,
2089                                      !!(hr_qp->access_flags &
2090                                      IB_ACCESS_REMOTE_WRITE));
2091                 }
2092
2093                 roce_set_bit(context->qpc_bytes_4,
2094                              QP_CONTEXT_QPC_BYTE_4_RDMAR_USE_S, 1);
2095                 roce_set_field(context->qpc_bytes_4,
2096                                QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_M,
2097                                QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_S,
2098                                ilog2((unsigned int)hr_qp->sq.wqe_cnt));
2099                 roce_set_field(context->qpc_bytes_4,
2100                                QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_M,
2101                                QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_S,
2102                                ilog2((unsigned int)hr_qp->rq.wqe_cnt));
2103                 roce_set_field(context->qpc_bytes_4,
2104                                QP_CONTEXT_QPC_BYTES_4_PD_M,
2105                                QP_CONTEXT_QPC_BYTES_4_PD_S,
2106                                to_hr_pd(ibqp->pd)->pdn);
2107
2108                 roce_set_field(context->qpc_bytes_8,
2109                                QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_M,
2110                                QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_S,
2111                                to_hr_cq(ibqp->send_cq)->cqn);
2112                 roce_set_field(context->qpc_bytes_8,
2113                                QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_M,
2114                                QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_S,
2115                                to_hr_cq(ibqp->recv_cq)->cqn);
2116
2117                 if (ibqp->srq)
2118                         roce_set_field(context->qpc_bytes_12,
2119                                        QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_M,
2120                                        QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_S,
2121                                        to_hr_srq(ibqp->srq)->srqn);
2122                 if (attr_mask & IB_QP_PKEY_INDEX)
2123                         roce_set_field(context->qpc_bytes_12,
2124                                        QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_M,
2125                                        QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S,
2126                                        attr->pkey_index);
2127                 else
2128                         roce_set_field(context->qpc_bytes_12,
2129                                        QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_M,
2130                                        QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S,
2131                                        hr_qp->pkey_index);
2132
2133                 roce_set_field(context->qpc_bytes_16,
2134                                QP_CONTEXT_QPC_BYTES_16_QP_NUM_M,
2135                                QP_CONTEXT_QPC_BYTES_16_QP_NUM_S, hr_qp->qpn);
2136         } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) {
2137                 if ((attr_mask & IB_QP_ALT_PATH) ||
2138                     (attr_mask & IB_QP_ACCESS_FLAGS) ||
2139                     (attr_mask & IB_QP_PKEY_INDEX) ||
2140                     (attr_mask & IB_QP_QKEY)) {
2141                         dev_err(dev, "INIT2RTR attr_mask error\n");
2142                         goto out;
2143                 }
2144
2145                 dmac = (u8 *)attr->ah_attr.dmac;
2146
2147                 context->sq_rq_bt_l = (u32)(dma_handle);
2148                 roce_set_field(context->qpc_bytes_24,
2149                                QP_CONTEXT_QPC_BYTES_24_SQ_RQ_BT_H_M,
2150                                QP_CONTEXT_QPC_BYTES_24_SQ_RQ_BT_H_S,
2151                                ((u32)(dma_handle >> 32)));
2152                 roce_set_bit(context->qpc_bytes_24,
2153                              QP_CONTEXT_QPC_BYTE_24_REMOTE_ENABLE_E2E_CREDITS_S,
2154                              1);
2155                 roce_set_field(context->qpc_bytes_24,
2156                                QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_M,
2157                                QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_S,
2158                                attr->min_rnr_timer);
2159                 context->irrl_ba_l = (u32)(dma_handle_2);
2160                 roce_set_field(context->qpc_bytes_32,
2161                                QP_CONTEXT_QPC_BYTES_32_IRRL_BA_H_M,
2162                                QP_CONTEXT_QPC_BYTES_32_IRRL_BA_H_S,
2163                                ((u32)(dma_handle_2 >> 32)) &
2164                                 QP_CONTEXT_QPC_BYTES_32_IRRL_BA_H_M);
2165                 roce_set_field(context->qpc_bytes_32,
2166                                QP_CONTEXT_QPC_BYTES_32_MIG_STATE_M,
2167                                QP_CONTEXT_QPC_BYTES_32_MIG_STATE_S, 0);
2168                 roce_set_bit(context->qpc_bytes_32,
2169                              QP_CONTEXT_QPC_BYTE_32_LOCAL_ENABLE_E2E_CREDITS_S,
2170                              1);
2171                 roce_set_bit(context->qpc_bytes_32,
2172                              QP_CONTEXT_QPC_BYTE_32_SIGNALING_TYPE_S,
2173                              hr_qp->sq_signal_bits);
2174
2175                 for (port = 0; port < hr_dev->caps.num_ports; port++) {
2176                         smac = (u8 *)hr_dev->dev_addr[port];
2177                         dev_dbg(dev, "smac: %2x: %2x: %2x: %2x: %2x: %2x\n",
2178                                 smac[0], smac[1], smac[2], smac[3], smac[4],
2179                                 smac[5]);
2180                         if ((dmac[0] == smac[0]) && (dmac[1] == smac[1]) &&
2181                             (dmac[2] == smac[2]) && (dmac[3] == smac[3]) &&
2182                             (dmac[4] == smac[4]) && (dmac[5] == smac[5])) {
2183                                 roce_set_bit(context->qpc_bytes_32,
2184                                     QP_CONTEXT_QPC_BYTE_32_LOOPBACK_INDICATOR_S,
2185                                     1);
2186                                 break;
2187                         }
2188                 }
2189
2190                 if (hr_dev->loop_idc == 0x1)
2191                         roce_set_bit(context->qpc_bytes_32,
2192                                 QP_CONTEXT_QPC_BYTE_32_LOOPBACK_INDICATOR_S, 1);
2193
2194                 roce_set_bit(context->qpc_bytes_32,
2195                              QP_CONTEXT_QPC_BYTE_32_GLOBAL_HEADER_S,
2196                              attr->ah_attr.ah_flags);
2197                 roce_set_field(context->qpc_bytes_32,
2198                                QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_M,
2199                                QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_S,
2200                                ilog2((unsigned int)attr->max_dest_rd_atomic));
2201
2202                 roce_set_field(context->qpc_bytes_36,
2203                                QP_CONTEXT_QPC_BYTES_36_DEST_QP_M,
2204                                QP_CONTEXT_QPC_BYTES_36_DEST_QP_S,
2205                                attr->dest_qp_num);
2206
2207                 /* Configure GID index */
2208                 roce_set_field(context->qpc_bytes_36,
2209                                QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_M,
2210                                QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_S,
2211                                hns_get_gid_index(hr_dev,
2212                                                  attr->ah_attr.port_num - 1,
2213                                                  attr->ah_attr.grh.sgid_index));
2214
2215                 memcpy(&(context->dmac_l), dmac, 4);
2216
2217                 roce_set_field(context->qpc_bytes_44,
2218                                QP_CONTEXT_QPC_BYTES_44_DMAC_H_M,
2219                                QP_CONTEXT_QPC_BYTES_44_DMAC_H_S,
2220                                *((u16 *)(&dmac[4])));
2221                 roce_set_field(context->qpc_bytes_44,
2222                                QP_CONTEXT_QPC_BYTES_44_MAXIMUM_STATIC_RATE_M,
2223                                QP_CONTEXT_QPC_BYTES_44_MAXIMUM_STATIC_RATE_S,
2224                                attr->ah_attr.static_rate);
2225                 roce_set_field(context->qpc_bytes_44,
2226                                QP_CONTEXT_QPC_BYTES_44_HOPLMT_M,
2227                                QP_CONTEXT_QPC_BYTES_44_HOPLMT_S,
2228                                attr->ah_attr.grh.hop_limit);
2229
2230                 roce_set_field(context->qpc_bytes_48,
2231                                QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_M,
2232                                QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_S,
2233                                attr->ah_attr.grh.flow_label);
2234                 roce_set_field(context->qpc_bytes_48,
2235                                QP_CONTEXT_QPC_BYTES_48_TCLASS_M,
2236                                QP_CONTEXT_QPC_BYTES_48_TCLASS_S,
2237                                attr->ah_attr.grh.traffic_class);
2238                 roce_set_field(context->qpc_bytes_48,
2239                                QP_CONTEXT_QPC_BYTES_48_MTU_M,
2240                                QP_CONTEXT_QPC_BYTES_48_MTU_S, attr->path_mtu);
2241
2242                 memcpy(context->dgid, attr->ah_attr.grh.dgid.raw,
2243                        sizeof(attr->ah_attr.grh.dgid.raw));
2244
2245                 dev_dbg(dev, "dmac:%x :%lx\n", context->dmac_l,
2246                         roce_get_field(context->qpc_bytes_44,
2247                                        QP_CONTEXT_QPC_BYTES_44_DMAC_H_M,
2248                                        QP_CONTEXT_QPC_BYTES_44_DMAC_H_S));
2249
2250                 roce_set_field(context->qpc_bytes_68,
2251                                QP_CONTEXT_QPC_BYTES_68_RQ_HEAD_M,
2252                                QP_CONTEXT_QPC_BYTES_68_RQ_HEAD_S,
2253                                hr_qp->rq.head);
2254                 roce_set_field(context->qpc_bytes_68,
2255                                QP_CONTEXT_QPC_BYTES_68_RQ_CUR_INDEX_M,
2256                                QP_CONTEXT_QPC_BYTES_68_RQ_CUR_INDEX_S, 0);
2257
2258                 rq_pa_start = (u32)hr_qp->rq.offset / PAGE_SIZE;
2259                 context->cur_rq_wqe_ba_l = (u32)(mtts[rq_pa_start]);
2260
2261                 roce_set_field(context->qpc_bytes_76,
2262                         QP_CONTEXT_QPC_BYTES_76_CUR_RQ_WQE_BA_H_M,
2263                         QP_CONTEXT_QPC_BYTES_76_CUR_RQ_WQE_BA_H_S,
2264                         mtts[rq_pa_start] >> 32);
2265                 roce_set_field(context->qpc_bytes_76,
2266                                QP_CONTEXT_QPC_BYTES_76_RX_REQ_MSN_M,
2267                                QP_CONTEXT_QPC_BYTES_76_RX_REQ_MSN_S, 0);
2268
2269                 context->rx_rnr_time = 0;
2270
2271                 roce_set_field(context->qpc_bytes_84,
2272                                QP_CONTEXT_QPC_BYTES_84_LAST_ACK_PSN_M,
2273                                QP_CONTEXT_QPC_BYTES_84_LAST_ACK_PSN_S,
2274                                attr->rq_psn - 1);
2275                 roce_set_field(context->qpc_bytes_84,
2276                                QP_CONTEXT_QPC_BYTES_84_TRRL_HEAD_M,
2277                                QP_CONTEXT_QPC_BYTES_84_TRRL_HEAD_S, 0);
2278
2279                 roce_set_field(context->qpc_bytes_88,
2280                                QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_M,
2281                                QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_S,
2282                                attr->rq_psn);
2283                 roce_set_bit(context->qpc_bytes_88,
2284                              QP_CONTEXT_QPC_BYTES_88_RX_REQ_PSN_ERR_FLAG_S, 0);
2285                 roce_set_bit(context->qpc_bytes_88,
2286                              QP_CONTEXT_QPC_BYTES_88_RX_LAST_OPCODE_FLG_S, 0);
2287                 roce_set_field(context->qpc_bytes_88,
2288                         QP_CONTEXT_QPC_BYTES_88_RQ_REQ_LAST_OPERATION_TYPE_M,
2289                         QP_CONTEXT_QPC_BYTES_88_RQ_REQ_LAST_OPERATION_TYPE_S,
2290                         0);
2291                 roce_set_field(context->qpc_bytes_88,
2292                                QP_CONTEXT_QPC_BYTES_88_RQ_REQ_RDMA_WR_FLAG_M,
2293                                QP_CONTEXT_QPC_BYTES_88_RQ_REQ_RDMA_WR_FLAG_S,
2294                                0);
2295
2296                 context->dma_length = 0;
2297                 context->r_key = 0;
2298                 context->va_l = 0;
2299                 context->va_h = 0;
2300
2301                 roce_set_field(context->qpc_bytes_108,
2302                                QP_CONTEXT_QPC_BYTES_108_TRRL_SDB_PSN_M,
2303                                QP_CONTEXT_QPC_BYTES_108_TRRL_SDB_PSN_S, 0);
2304                 roce_set_bit(context->qpc_bytes_108,
2305                              QP_CONTEXT_QPC_BYTES_108_TRRL_SDB_PSN_FLG_S, 0);
2306                 roce_set_bit(context->qpc_bytes_108,
2307                              QP_CONTEXT_QPC_BYTES_108_TRRL_TDB_PSN_FLG_S, 0);
2308
2309                 roce_set_field(context->qpc_bytes_112,
2310                                QP_CONTEXT_QPC_BYTES_112_TRRL_TDB_PSN_M,
2311                                QP_CONTEXT_QPC_BYTES_112_TRRL_TDB_PSN_S, 0);
2312                 roce_set_field(context->qpc_bytes_112,
2313                                QP_CONTEXT_QPC_BYTES_112_TRRL_TAIL_M,
2314                                QP_CONTEXT_QPC_BYTES_112_TRRL_TAIL_S, 0);
2315
2316                 /* For chip resp ack */
2317                 roce_set_field(context->qpc_bytes_156,
2318                                QP_CONTEXT_QPC_BYTES_156_PORT_NUM_M,
2319                                QP_CONTEXT_QPC_BYTES_156_PORT_NUM_S,
2320                                hr_qp->phy_port);
2321                 roce_set_field(context->qpc_bytes_156,
2322                                QP_CONTEXT_QPC_BYTES_156_SL_M,
2323                                QP_CONTEXT_QPC_BYTES_156_SL_S, attr->ah_attr.sl);
2324                 hr_qp->sl = attr->ah_attr.sl;
2325         } else if (cur_state == IB_QPS_RTR &&
2326                 new_state == IB_QPS_RTS) {
2327                 /* If exist optional param, return error */
2328                 if ((attr_mask & IB_QP_ALT_PATH) ||
2329                     (attr_mask & IB_QP_ACCESS_FLAGS) ||
2330                     (attr_mask & IB_QP_QKEY) ||
2331                     (attr_mask & IB_QP_PATH_MIG_STATE) ||
2332                     (attr_mask & IB_QP_CUR_STATE) ||
2333                     (attr_mask & IB_QP_MIN_RNR_TIMER)) {
2334                         dev_err(dev, "RTR2RTS attr_mask error\n");
2335                         goto out;
2336                 }
2337
2338                 context->rx_cur_sq_wqe_ba_l = (u32)(mtts[0]);
2339
2340                 roce_set_field(context->qpc_bytes_120,
2341                                QP_CONTEXT_QPC_BYTES_120_RX_CUR_SQ_WQE_BA_H_M,
2342                                QP_CONTEXT_QPC_BYTES_120_RX_CUR_SQ_WQE_BA_H_S,
2343                                (mtts[0]) >> 32);
2344
2345                 roce_set_field(context->qpc_bytes_124,
2346                                QP_CONTEXT_QPC_BYTES_124_RX_ACK_MSN_M,
2347                                QP_CONTEXT_QPC_BYTES_124_RX_ACK_MSN_S, 0);
2348                 roce_set_field(context->qpc_bytes_124,
2349                                QP_CONTEXT_QPC_BYTES_124_IRRL_MSG_IDX_M,
2350                                QP_CONTEXT_QPC_BYTES_124_IRRL_MSG_IDX_S, 0);
2351
2352                 roce_set_field(context->qpc_bytes_128,
2353                                QP_CONTEXT_QPC_BYTES_128_RX_ACK_EPSN_M,
2354                                QP_CONTEXT_QPC_BYTES_128_RX_ACK_EPSN_S,
2355                                attr->sq_psn);
2356                 roce_set_bit(context->qpc_bytes_128,
2357                              QP_CONTEXT_QPC_BYTES_128_RX_ACK_PSN_ERR_FLG_S, 0);
2358                 roce_set_field(context->qpc_bytes_128,
2359                              QP_CONTEXT_QPC_BYTES_128_ACK_LAST_OPERATION_TYPE_M,
2360                              QP_CONTEXT_QPC_BYTES_128_ACK_LAST_OPERATION_TYPE_S,
2361                              0);
2362                 roce_set_bit(context->qpc_bytes_128,
2363                              QP_CONTEXT_QPC_BYTES_128_IRRL_PSN_VLD_FLG_S, 0);
2364
2365                 roce_set_field(context->qpc_bytes_132,
2366                                QP_CONTEXT_QPC_BYTES_132_IRRL_PSN_M,
2367                                QP_CONTEXT_QPC_BYTES_132_IRRL_PSN_S, 0);
2368                 roce_set_field(context->qpc_bytes_132,
2369                                QP_CONTEXT_QPC_BYTES_132_IRRL_TAIL_M,
2370                                QP_CONTEXT_QPC_BYTES_132_IRRL_TAIL_S, 0);
2371
2372                 roce_set_field(context->qpc_bytes_136,
2373                                QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_PSN_M,
2374                                QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_PSN_S,
2375                                attr->sq_psn);
2376                 roce_set_field(context->qpc_bytes_136,
2377                                QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_FPKT_PSN_L_M,
2378                                QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_FPKT_PSN_L_S,
2379                                attr->sq_psn);
2380
2381                 roce_set_field(context->qpc_bytes_140,
2382                                QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_FPKT_PSN_H_M,
2383                                QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_FPKT_PSN_H_S,
2384                                (attr->sq_psn >> SQ_PSN_SHIFT));
2385                 roce_set_field(context->qpc_bytes_140,
2386                                QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_MSN_M,
2387                                QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_MSN_S, 0);
2388                 roce_set_bit(context->qpc_bytes_140,
2389                              QP_CONTEXT_QPC_BYTES_140_RNR_RETRY_FLG_S, 0);
2390
2391                 roce_set_field(context->qpc_bytes_148,
2392                                QP_CONTEXT_QPC_BYTES_148_CHECK_FLAG_M,
2393                                QP_CONTEXT_QPC_BYTES_148_CHECK_FLAG_S, 0);
2394                 roce_set_field(context->qpc_bytes_148,
2395                                QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_M,
2396                                QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_S,
2397                                attr->retry_cnt);
2398                 roce_set_field(context->qpc_bytes_148,
2399                                QP_CONTEXT_QPC_BYTES_148_RNR_RETRY_COUNT_M,
2400                                QP_CONTEXT_QPC_BYTES_148_RNR_RETRY_COUNT_S,
2401                                attr->rnr_retry);
2402                 roce_set_field(context->qpc_bytes_148,
2403                                QP_CONTEXT_QPC_BYTES_148_LSN_M,
2404                                QP_CONTEXT_QPC_BYTES_148_LSN_S, 0x100);
2405
2406                 context->rnr_retry = 0;
2407
2408                 roce_set_field(context->qpc_bytes_156,
2409                                QP_CONTEXT_QPC_BYTES_156_RETRY_COUNT_INIT_M,
2410                                QP_CONTEXT_QPC_BYTES_156_RETRY_COUNT_INIT_S,
2411                                attr->retry_cnt);
2412                 if (attr->timeout < 0x12) {
2413                         dev_info(dev, "ack timeout value(0x%x) must bigger than 0x12.\n",
2414                                  attr->timeout);
2415                         roce_set_field(context->qpc_bytes_156,
2416                                        QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_M,
2417                                        QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_S,
2418                                        0x12);
2419                 } else {
2420                         roce_set_field(context->qpc_bytes_156,
2421                                        QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_M,
2422                                        QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_S,
2423                                        attr->timeout);
2424                 }
2425                 roce_set_field(context->qpc_bytes_156,
2426                                QP_CONTEXT_QPC_BYTES_156_RNR_RETRY_COUNT_INIT_M,
2427                                QP_CONTEXT_QPC_BYTES_156_RNR_RETRY_COUNT_INIT_S,
2428                                attr->rnr_retry);
2429                 roce_set_field(context->qpc_bytes_156,
2430                                QP_CONTEXT_QPC_BYTES_156_PORT_NUM_M,
2431                                QP_CONTEXT_QPC_BYTES_156_PORT_NUM_S,
2432                                hr_qp->phy_port);
2433                 roce_set_field(context->qpc_bytes_156,
2434                                QP_CONTEXT_QPC_BYTES_156_SL_M,
2435                                QP_CONTEXT_QPC_BYTES_156_SL_S, attr->ah_attr.sl);
2436                 hr_qp->sl = attr->ah_attr.sl;
2437                 roce_set_field(context->qpc_bytes_156,
2438                                QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_M,
2439                                QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_S,
2440                                ilog2((unsigned int)attr->max_rd_atomic));
2441                 roce_set_field(context->qpc_bytes_156,
2442                                QP_CONTEXT_QPC_BYTES_156_ACK_REQ_IND_M,
2443                                QP_CONTEXT_QPC_BYTES_156_ACK_REQ_IND_S, 0);
2444                 context->pkt_use_len = 0;
2445
2446                 roce_set_field(context->qpc_bytes_164,
2447                                QP_CONTEXT_QPC_BYTES_164_SQ_PSN_M,
2448                                QP_CONTEXT_QPC_BYTES_164_SQ_PSN_S, attr->sq_psn);
2449                 roce_set_field(context->qpc_bytes_164,
2450                                QP_CONTEXT_QPC_BYTES_164_IRRL_HEAD_M,
2451                                QP_CONTEXT_QPC_BYTES_164_IRRL_HEAD_S, 0);
2452
2453                 roce_set_field(context->qpc_bytes_168,
2454                                QP_CONTEXT_QPC_BYTES_168_RETRY_SQ_PSN_M,
2455                                QP_CONTEXT_QPC_BYTES_168_RETRY_SQ_PSN_S,
2456                                attr->sq_psn);
2457                 roce_set_field(context->qpc_bytes_168,
2458                                QP_CONTEXT_QPC_BYTES_168_SGE_USE_FLA_M,
2459                                QP_CONTEXT_QPC_BYTES_168_SGE_USE_FLA_S, 0);
2460                 roce_set_field(context->qpc_bytes_168,
2461                                QP_CONTEXT_QPC_BYTES_168_DB_TYPE_M,
2462                                QP_CONTEXT_QPC_BYTES_168_DB_TYPE_S, 0);
2463                 roce_set_bit(context->qpc_bytes_168,
2464                              QP_CONTEXT_QPC_BYTES_168_MSG_LP_IND_S, 0);
2465                 roce_set_bit(context->qpc_bytes_168,
2466                              QP_CONTEXT_QPC_BYTES_168_CSDB_LP_IND_S, 0);
2467                 roce_set_bit(context->qpc_bytes_168,
2468                              QP_CONTEXT_QPC_BYTES_168_QP_ERR_FLG_S, 0);
2469                 context->sge_use_len = 0;
2470
2471                 roce_set_field(context->qpc_bytes_176,
2472                                QP_CONTEXT_QPC_BYTES_176_DB_CUR_INDEX_M,
2473                                QP_CONTEXT_QPC_BYTES_176_DB_CUR_INDEX_S, 0);
2474                 roce_set_field(context->qpc_bytes_176,
2475                                QP_CONTEXT_QPC_BYTES_176_RETRY_DB_CUR_INDEX_M,
2476                                QP_CONTEXT_QPC_BYTES_176_RETRY_DB_CUR_INDEX_S,
2477                                0);
2478                 roce_set_field(context->qpc_bytes_180,
2479                                QP_CONTEXT_QPC_BYTES_180_SQ_CUR_INDEX_M,
2480                                QP_CONTEXT_QPC_BYTES_180_SQ_CUR_INDEX_S, 0);
2481                 roce_set_field(context->qpc_bytes_180,
2482                                QP_CONTEXT_QPC_BYTES_180_SQ_HEAD_M,
2483                                QP_CONTEXT_QPC_BYTES_180_SQ_HEAD_S, 0);
2484
2485                 context->tx_cur_sq_wqe_ba_l = (u32)(mtts[0]);
2486
2487                 roce_set_field(context->qpc_bytes_188,
2488                                QP_CONTEXT_QPC_BYTES_188_TX_CUR_SQ_WQE_BA_H_M,
2489                                QP_CONTEXT_QPC_BYTES_188_TX_CUR_SQ_WQE_BA_H_S,
2490                                (mtts[0]) >> 32);
2491                 roce_set_bit(context->qpc_bytes_188,
2492                              QP_CONTEXT_QPC_BYTES_188_PKT_RETRY_FLG_S, 0);
2493                 roce_set_field(context->qpc_bytes_188,
2494                                QP_CONTEXT_QPC_BYTES_188_TX_RETRY_CUR_INDEX_M,
2495                                QP_CONTEXT_QPC_BYTES_188_TX_RETRY_CUR_INDEX_S,
2496                                0);
2497         } else if (!((cur_state == IB_QPS_INIT && new_state == IB_QPS_RESET) ||
2498                    (cur_state == IB_QPS_INIT && new_state == IB_QPS_ERR) ||
2499                    (cur_state == IB_QPS_RTR && new_state == IB_QPS_RESET) ||
2500                    (cur_state == IB_QPS_RTR && new_state == IB_QPS_ERR) ||
2501                    (cur_state == IB_QPS_RTS && new_state == IB_QPS_RESET) ||
2502                    (cur_state == IB_QPS_RTS && new_state == IB_QPS_ERR) ||
2503                    (cur_state == IB_QPS_ERR && new_state == IB_QPS_RESET) ||
2504                    (cur_state == IB_QPS_ERR && new_state == IB_QPS_ERR))) {
2505                 dev_err(dev, "not support this status migration\n");
2506                 goto out;
2507         }
2508
2509         /* Every status migrate must change state */
2510         roce_set_field(context->qpc_bytes_144,
2511                        QP_CONTEXT_QPC_BYTES_144_QP_STATE_M,
2512                        QP_CONTEXT_QPC_BYTES_144_QP_STATE_S, attr->qp_state);
2513
2514         /* SW pass context to HW */
2515         ret = hns_roce_v1_qp_modify(hr_dev, &hr_qp->mtt,
2516                                     to_hns_roce_state(cur_state),
2517                                     to_hns_roce_state(new_state), context,
2518                                     hr_qp);
2519         if (ret) {
2520                 dev_err(dev, "hns_roce_qp_modify failed\n");
2521                 goto out;
2522         }
2523
2524         /*
2525         * Use rst2init to instead of init2init with drv,
2526         * need to hw to flash RQ HEAD by DB again
2527         */
2528         if (cur_state == IB_QPS_INIT && new_state == IB_QPS_INIT) {
2529                 /* Memory barrier */
2530                 wmb();
2531
2532                 roce_set_field(doorbell[0], RQ_DOORBELL_U32_4_RQ_HEAD_M,
2533                                RQ_DOORBELL_U32_4_RQ_HEAD_S, hr_qp->rq.head);
2534                 roce_set_field(doorbell[1], RQ_DOORBELL_U32_8_QPN_M,
2535                                RQ_DOORBELL_U32_8_QPN_S, hr_qp->qpn);
2536                 roce_set_field(doorbell[1], RQ_DOORBELL_U32_8_CMD_M,
2537                                RQ_DOORBELL_U32_8_CMD_S, 1);
2538                 roce_set_bit(doorbell[1], RQ_DOORBELL_U32_8_HW_SYNC_S, 1);
2539
2540                 if (ibqp->uobject) {
2541                         hr_qp->rq.db_reg_l = hr_dev->reg_base +
2542                                      ROCEE_DB_OTHERS_L_0_REG +
2543                                      DB_REG_OFFSET * hr_dev->priv_uar.index;
2544                 }
2545
2546                 hns_roce_write64_k(doorbell, hr_qp->rq.db_reg_l);
2547         }
2548
2549         hr_qp->state = new_state;
2550
2551         if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
2552                 hr_qp->resp_depth = attr->max_dest_rd_atomic;
2553         if (attr_mask & IB_QP_PORT) {
2554                 hr_qp->port = attr->port_num - 1;
2555                 hr_qp->phy_port = hr_dev->iboe.phy_port[hr_qp->port];
2556         }
2557
2558         if (new_state == IB_QPS_RESET && !ibqp->uobject) {
2559                 hns_roce_v1_cq_clean(to_hr_cq(ibqp->recv_cq), hr_qp->qpn,
2560                                      ibqp->srq ? to_hr_srq(ibqp->srq) : NULL);
2561                 if (ibqp->send_cq != ibqp->recv_cq)
2562                         hns_roce_v1_cq_clean(to_hr_cq(ibqp->send_cq),
2563                                              hr_qp->qpn, NULL);
2564
2565                 hr_qp->rq.head = 0;
2566                 hr_qp->rq.tail = 0;
2567                 hr_qp->sq.head = 0;
2568                 hr_qp->sq.tail = 0;
2569                 hr_qp->sq_next_wqe = 0;
2570         }
2571 out:
2572         kfree(context);
2573         return ret;
2574 }
2575
2576 int hns_roce_v1_modify_qp(struct ib_qp *ibqp, const struct ib_qp_attr *attr,
2577                           int attr_mask, enum ib_qp_state cur_state,
2578                           enum ib_qp_state new_state)
2579 {
2580
2581         if (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_SMI)
2582                 return hns_roce_v1_m_sqp(ibqp, attr, attr_mask, cur_state,
2583                                          new_state);
2584         else
2585                 return hns_roce_v1_m_qp(ibqp, attr, attr_mask, cur_state,
2586                                         new_state);
2587 }
2588
2589 static enum ib_qp_state to_ib_qp_state(enum hns_roce_qp_state state)
2590 {
2591         switch (state) {
2592         case HNS_ROCE_QP_STATE_RST:
2593                 return IB_QPS_RESET;
2594         case HNS_ROCE_QP_STATE_INIT:
2595                 return IB_QPS_INIT;
2596         case HNS_ROCE_QP_STATE_RTR:
2597                 return IB_QPS_RTR;
2598         case HNS_ROCE_QP_STATE_RTS:
2599                 return IB_QPS_RTS;
2600         case HNS_ROCE_QP_STATE_SQD:
2601                 return IB_QPS_SQD;
2602         case HNS_ROCE_QP_STATE_ERR:
2603                 return IB_QPS_ERR;
2604         default:
2605                 return IB_QPS_ERR;
2606         }
2607 }
2608
2609 static int hns_roce_v1_query_qpc(struct hns_roce_dev *hr_dev,
2610                                  struct hns_roce_qp *hr_qp,
2611                                  struct hns_roce_qp_context *hr_context)
2612 {
2613         struct hns_roce_cmd_mailbox *mailbox;
2614         int ret;
2615
2616         mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
2617         if (IS_ERR(mailbox))
2618                 return PTR_ERR(mailbox);
2619
2620         ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, hr_qp->qpn, 0,
2621                                 HNS_ROCE_CMD_QUERY_QP,
2622                                 HNS_ROCE_CMD_TIME_CLASS_A);
2623         if (!ret)
2624                 memcpy(hr_context, mailbox->buf, sizeof(*hr_context));
2625         else
2626                 dev_err(&hr_dev->pdev->dev, "QUERY QP cmd process error\n");
2627
2628         hns_roce_free_cmd_mailbox(hr_dev, mailbox);
2629
2630         return ret;
2631 }
2632
2633 int hns_roce_v1_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
2634                          int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
2635 {
2636         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
2637         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
2638         struct device *dev = &hr_dev->pdev->dev;
2639         struct hns_roce_qp_context *context;
2640         int tmp_qp_state = 0;
2641         int ret = 0;
2642         int state;
2643
2644         context = kzalloc(sizeof(*context), GFP_KERNEL);
2645         if (!context)
2646                 return -ENOMEM;
2647
2648         memset(qp_attr, 0, sizeof(*qp_attr));
2649         memset(qp_init_attr, 0, sizeof(*qp_init_attr));
2650
2651         mutex_lock(&hr_qp->mutex);
2652
2653         if (hr_qp->state == IB_QPS_RESET) {
2654                 qp_attr->qp_state = IB_QPS_RESET;
2655                 goto done;
2656         }
2657
2658         ret = hns_roce_v1_query_qpc(hr_dev, hr_qp, context);
2659         if (ret) {
2660                 dev_err(dev, "query qpc error\n");
2661                 ret = -EINVAL;
2662                 goto out;
2663         }
2664
2665         state = roce_get_field(context->qpc_bytes_144,
2666                                QP_CONTEXT_QPC_BYTES_144_QP_STATE_M,
2667                                QP_CONTEXT_QPC_BYTES_144_QP_STATE_S);
2668         tmp_qp_state = (int)to_ib_qp_state((enum hns_roce_qp_state)state);
2669         if (tmp_qp_state == -1) {
2670                 dev_err(dev, "to_ib_qp_state error\n");
2671                 ret = -EINVAL;
2672                 goto out;
2673         }
2674         hr_qp->state = (u8)tmp_qp_state;
2675         qp_attr->qp_state = (enum ib_qp_state)hr_qp->state;
2676         qp_attr->path_mtu = (enum ib_mtu)roce_get_field(context->qpc_bytes_48,
2677                                                QP_CONTEXT_QPC_BYTES_48_MTU_M,
2678                                                QP_CONTEXT_QPC_BYTES_48_MTU_S);
2679         qp_attr->path_mig_state = IB_MIG_ARMED;
2680         if (hr_qp->ibqp.qp_type == IB_QPT_UD)
2681                 qp_attr->qkey = QKEY_VAL;
2682
2683         qp_attr->rq_psn = roce_get_field(context->qpc_bytes_88,
2684                                          QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_M,
2685                                          QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_S);
2686         qp_attr->sq_psn = (u32)roce_get_field(context->qpc_bytes_164,
2687                                              QP_CONTEXT_QPC_BYTES_164_SQ_PSN_M,
2688                                              QP_CONTEXT_QPC_BYTES_164_SQ_PSN_S);
2689         qp_attr->dest_qp_num = (u8)roce_get_field(context->qpc_bytes_36,
2690                                         QP_CONTEXT_QPC_BYTES_36_DEST_QP_M,
2691                                         QP_CONTEXT_QPC_BYTES_36_DEST_QP_S);
2692         qp_attr->qp_access_flags = ((roce_get_bit(context->qpc_bytes_4,
2693                         QP_CONTEXT_QPC_BYTE_4_RDMA_READ_ENABLE_S)) << 2) |
2694                                    ((roce_get_bit(context->qpc_bytes_4,
2695                         QP_CONTEXT_QPC_BYTE_4_RDMA_WRITE_ENABLE_S)) << 1) |
2696                                    ((roce_get_bit(context->qpc_bytes_4,
2697                         QP_CONTEXT_QPC_BYTE_4_ATOMIC_OPERATION_ENABLE_S)) << 3);
2698
2699         if (hr_qp->ibqp.qp_type == IB_QPT_RC ||
2700             hr_qp->ibqp.qp_type == IB_QPT_UC) {
2701                 qp_attr->ah_attr.sl = roce_get_field(context->qpc_bytes_156,
2702                                                 QP_CONTEXT_QPC_BYTES_156_SL_M,
2703                                                 QP_CONTEXT_QPC_BYTES_156_SL_S);
2704                 qp_attr->ah_attr.grh.flow_label = roce_get_field(
2705                                         context->qpc_bytes_48,
2706                                         QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_M,
2707                                         QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_S);
2708                 qp_attr->ah_attr.grh.sgid_index = roce_get_field(
2709                                         context->qpc_bytes_36,
2710                                         QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_M,
2711                                         QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_S);
2712                 qp_attr->ah_attr.grh.hop_limit = roce_get_field(
2713                                         context->qpc_bytes_44,
2714                                         QP_CONTEXT_QPC_BYTES_44_HOPLMT_M,
2715                                         QP_CONTEXT_QPC_BYTES_44_HOPLMT_S);
2716                 qp_attr->ah_attr.grh.traffic_class = roce_get_field(
2717                                         context->qpc_bytes_48,
2718                                         QP_CONTEXT_QPC_BYTES_48_TCLASS_M,
2719                                         QP_CONTEXT_QPC_BYTES_48_TCLASS_S);
2720
2721                 memcpy(qp_attr->ah_attr.grh.dgid.raw, context->dgid,
2722                        sizeof(qp_attr->ah_attr.grh.dgid.raw));
2723         }
2724
2725         qp_attr->pkey_index = roce_get_field(context->qpc_bytes_12,
2726                               QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_M,
2727                               QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S);
2728         qp_attr->port_num = (u8)roce_get_field(context->qpc_bytes_156,
2729                              QP_CONTEXT_QPC_BYTES_156_PORT_NUM_M,
2730                              QP_CONTEXT_QPC_BYTES_156_PORT_NUM_S) + 1;
2731         qp_attr->sq_draining = 0;
2732         qp_attr->max_rd_atomic = roce_get_field(context->qpc_bytes_156,
2733                                  QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_M,
2734                                  QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_S);
2735         qp_attr->max_dest_rd_atomic = roce_get_field(context->qpc_bytes_32,
2736                                  QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_M,
2737                                  QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_S);
2738         qp_attr->min_rnr_timer = (u8)(roce_get_field(context->qpc_bytes_24,
2739                         QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_M,
2740                         QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_S));
2741         qp_attr->timeout = (u8)(roce_get_field(context->qpc_bytes_156,
2742                             QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_M,
2743                             QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_S));
2744         qp_attr->retry_cnt = roce_get_field(context->qpc_bytes_148,
2745                              QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_M,
2746                              QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_S);
2747         qp_attr->rnr_retry = context->rnr_retry;
2748
2749 done:
2750         qp_attr->cur_qp_state = qp_attr->qp_state;
2751         qp_attr->cap.max_recv_wr = hr_qp->rq.wqe_cnt;
2752         qp_attr->cap.max_recv_sge = hr_qp->rq.max_gs;
2753
2754         if (!ibqp->uobject) {
2755                 qp_attr->cap.max_send_wr = hr_qp->sq.wqe_cnt;
2756                 qp_attr->cap.max_send_sge = hr_qp->sq.max_gs;
2757         } else {
2758                 qp_attr->cap.max_send_wr = 0;
2759                 qp_attr->cap.max_send_sge = 0;
2760         }
2761
2762         qp_init_attr->cap = qp_attr->cap;
2763
2764 out:
2765         mutex_unlock(&hr_qp->mutex);
2766         kfree(context);
2767         return ret;
2768 }
2769
2770 static void hns_roce_v1_destroy_qp_common(struct hns_roce_dev *hr_dev,
2771                                           struct hns_roce_qp *hr_qp,
2772                                           int is_user)
2773 {
2774         u32 sdbinvcnt;
2775         unsigned long end = 0;
2776         u32 sdbinvcnt_val;
2777         u32 sdbsendptr_val;
2778         u32 sdbisusepr_val;
2779         struct hns_roce_cq *send_cq, *recv_cq;
2780         struct device *dev = &hr_dev->pdev->dev;
2781
2782         if (hr_qp->ibqp.qp_type == IB_QPT_RC) {
2783                 if (hr_qp->state != IB_QPS_RESET) {
2784                         /*
2785                         * Set qp to ERR,
2786                         * waiting for hw complete processing all dbs
2787                         */
2788                         if (hns_roce_v1_qp_modify(hr_dev, NULL,
2789                                         to_hns_roce_state(
2790                                                 (enum ib_qp_state)hr_qp->state),
2791                                                 HNS_ROCE_QP_STATE_ERR, NULL,
2792                                                 hr_qp))
2793                                 dev_err(dev, "modify QP %06lx to ERR failed.\n",
2794                                         hr_qp->qpn);
2795
2796                         /* Record issued doorbell */
2797                         sdbisusepr_val = roce_read(hr_dev,
2798                                          ROCEE_SDB_ISSUE_PTR_REG);
2799                         /*
2800                         * Query db process status,
2801                         * until hw process completely
2802                         */
2803                         end = msecs_to_jiffies(
2804                               HNS_ROCE_QP_DESTROY_TIMEOUT_MSECS) + jiffies;
2805                         do {
2806                                 sdbsendptr_val = roce_read(hr_dev,
2807                                                  ROCEE_SDB_SEND_PTR_REG);
2808                                 if (!time_before(jiffies, end)) {
2809                                         dev_err(dev, "destroy qp(0x%lx) timeout!!!",
2810                                                 hr_qp->qpn);
2811                                         break;
2812                                 }
2813                         } while ((short)(roce_get_field(sdbsendptr_val,
2814                                         ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_M,
2815                                         ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_S) -
2816                                 roce_get_field(sdbisusepr_val,
2817                                         ROCEE_SDB_ISSUE_PTR_SDB_ISSUE_PTR_M,
2818                                         ROCEE_SDB_ISSUE_PTR_SDB_ISSUE_PTR_S)
2819                                 ) < 0);
2820
2821                         /* Get list pointer */
2822                         sdbinvcnt = roce_read(hr_dev, ROCEE_SDB_INV_CNT_REG);
2823
2824                         /* Query db's list status, until hw reversal */
2825                         do {
2826                                 sdbinvcnt_val = roce_read(hr_dev,
2827                                                 ROCEE_SDB_INV_CNT_REG);
2828                                 if (!time_before(jiffies, end)) {
2829                                         dev_err(dev, "destroy qp(0x%lx) timeout!!!",
2830                                                 hr_qp->qpn);
2831                                         dev_err(dev, "SdbInvCnt = 0x%x\n",
2832                                                 sdbinvcnt_val);
2833                                         break;
2834                                 }
2835                         } while ((short)(roce_get_field(sdbinvcnt_val,
2836                                   ROCEE_SDB_INV_CNT_SDB_INV_CNT_M,
2837                                   ROCEE_SDB_INV_CNT_SDB_INV_CNT_S) -
2838                                   (sdbinvcnt + SDB_INV_CNT_OFFSET)) < 0);
2839
2840                         /* Modify qp to reset before destroying qp */
2841                         if (hns_roce_v1_qp_modify(hr_dev, NULL,
2842                                         to_hns_roce_state(
2843                                         (enum ib_qp_state)hr_qp->state),
2844                                         HNS_ROCE_QP_STATE_RST, NULL, hr_qp))
2845                                 dev_err(dev, "modify QP %06lx to RESET failed.\n",
2846                                         hr_qp->qpn);
2847                 }
2848         }
2849
2850         send_cq = to_hr_cq(hr_qp->ibqp.send_cq);
2851         recv_cq = to_hr_cq(hr_qp->ibqp.recv_cq);
2852
2853         hns_roce_lock_cqs(send_cq, recv_cq);
2854
2855         if (!is_user) {
2856                 __hns_roce_v1_cq_clean(recv_cq, hr_qp->qpn, hr_qp->ibqp.srq ?
2857                                        to_hr_srq(hr_qp->ibqp.srq) : NULL);
2858                 if (send_cq != recv_cq)
2859                         __hns_roce_v1_cq_clean(send_cq, hr_qp->qpn, NULL);
2860         }
2861
2862         hns_roce_qp_remove(hr_dev, hr_qp);
2863
2864         hns_roce_unlock_cqs(send_cq, recv_cq);
2865
2866         hns_roce_qp_free(hr_dev, hr_qp);
2867
2868         /* Not special_QP, free their QPN */
2869         if ((hr_qp->ibqp.qp_type == IB_QPT_RC) ||
2870             (hr_qp->ibqp.qp_type == IB_QPT_UC) ||
2871             (hr_qp->ibqp.qp_type == IB_QPT_UD))
2872                 hns_roce_release_range_qp(hr_dev, hr_qp->qpn, 1);
2873
2874         hns_roce_mtt_cleanup(hr_dev, &hr_qp->mtt);
2875
2876         if (is_user) {
2877                 ib_umem_release(hr_qp->umem);
2878         } else {
2879                 kfree(hr_qp->sq.wrid);
2880                 kfree(hr_qp->rq.wrid);
2881                 hns_roce_buf_free(hr_dev, hr_qp->buff_size, &hr_qp->hr_buf);
2882         }
2883 }
2884
2885 int hns_roce_v1_destroy_qp(struct ib_qp *ibqp)
2886 {
2887         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
2888         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
2889
2890         hns_roce_v1_destroy_qp_common(hr_dev, hr_qp, !!ibqp->pd->uobject);
2891
2892         if (hr_qp->ibqp.qp_type == IB_QPT_GSI)
2893                 kfree(hr_to_hr_sqp(hr_qp));
2894         else
2895                 kfree(hr_qp);
2896
2897         return 0;
2898 }
2899
2900 struct hns_roce_v1_priv hr_v1_priv;
2901
2902 struct hns_roce_hw hns_roce_hw_v1 = {
2903         .reset = hns_roce_v1_reset,
2904         .hw_profile = hns_roce_v1_profile,
2905         .hw_init = hns_roce_v1_init,
2906         .hw_exit = hns_roce_v1_exit,
2907         .set_gid = hns_roce_v1_set_gid,
2908         .set_mac = hns_roce_v1_set_mac,
2909         .set_mtu = hns_roce_v1_set_mtu,
2910         .write_mtpt = hns_roce_v1_write_mtpt,
2911         .write_cqc = hns_roce_v1_write_cqc,
2912         .clear_hem = hns_roce_v1_clear_hem,
2913         .modify_qp = hns_roce_v1_modify_qp,
2914         .query_qp = hns_roce_v1_query_qp,
2915         .destroy_qp = hns_roce_v1_destroy_qp,
2916         .post_send = hns_roce_v1_post_send,
2917         .post_recv = hns_roce_v1_post_recv,
2918         .req_notify_cq = hns_roce_v1_req_notify_cq,
2919         .poll_cq = hns_roce_v1_poll_cq,
2920         .priv = &hr_v1_priv,
2921 };