drivers/scsi/cxgbi/cxgb4i/cxgb4i.c

   1 /*
   2  * cxgb4i.c: Chelsio T4 iSCSI driver.
   3  *
   4  * Copyright (c) 2010-2015 Chelsio Communications, Inc.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation.
   9  *
  10  * Written by:  Karen Xie (kxie@chelsio.com)
  11  *              Rakesh Ranjan (rranjan@chelsio.com)
  12  */
  13
  14 #define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__
  15
  16 #include <linux/kernel.h>
  17 #include <linux/module.h>
  18 #include <linux/moduleparam.h>
  19 #include <scsi/scsi_host.h>
  20 #include <net/tcp.h>
  21 #include <net/dst.h>
  22 #include <linux/netdevice.h>
  23 #include <net/addrconf.h>
  24
  25 #include "t4_regs.h"
  26 #include "t4_msg.h"
  27 #include "cxgb4.h"
  28 #include "cxgb4_uld.h"
  29 #include "t4fw_api.h"
  30 #include "l2t.h"
  31 #include "cxgb4i.h"
  32 #include "clip_tbl.h"
  33
  34 static unsigned int dbg_level;
  35
  36 #include "../libcxgbi.h"
  37
  38 #define DRV_MODULE_NAME         "cxgb4i"
  39 #define DRV_MODULE_DESC         "Chelsio T4/T5 iSCSI Driver"
  40 #define DRV_MODULE_VERSION      "0.9.5-ko"
  41 #define DRV_MODULE_RELDATE      "Apr. 2015"
  42
  43 static char version[] =
  44         DRV_MODULE_DESC " " DRV_MODULE_NAME
  45         " v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";
  46
  47 MODULE_AUTHOR("Chelsio Communications, Inc.");
  48 MODULE_DESCRIPTION(DRV_MODULE_DESC);
  49 MODULE_VERSION(DRV_MODULE_VERSION);
  50 MODULE_LICENSE("GPL");
  51
  52 module_param(dbg_level, uint, 0644);
  53 MODULE_PARM_DESC(dbg_level, "Debug flag (default=0)");
  54
  55 #define CXGB4I_DEFAULT_10G_RCV_WIN (256 * 1024)
  56 static int cxgb4i_rcv_win = -1;
  57 module_param(cxgb4i_rcv_win, int, 0644);
  58 MODULE_PARM_DESC(cxgb4i_rcv_win, "TCP reveive window in bytes");
  59
  60 #define CXGB4I_DEFAULT_10G_SND_WIN (128 * 1024)
  61 static int cxgb4i_snd_win = -1;
  62 module_param(cxgb4i_snd_win, int, 0644);
  63 MODULE_PARM_DESC(cxgb4i_snd_win, "TCP send window in bytes");
  64
  65 static int cxgb4i_rx_credit_thres = 10 * 1024;
  66 module_param(cxgb4i_rx_credit_thres, int, 0644);
  67 MODULE_PARM_DESC(cxgb4i_rx_credit_thres,
  68                 "RX credits return threshold in bytes (default=10KB)");
  69
  70 static unsigned int cxgb4i_max_connect = (8 * 1024);
  71 module_param(cxgb4i_max_connect, uint, 0644);
  72 MODULE_PARM_DESC(cxgb4i_max_connect, "Maximum number of connections");
  73
  74 static unsigned short cxgb4i_sport_base = 20000;
  75 module_param(cxgb4i_sport_base, ushort, 0644);
  76 MODULE_PARM_DESC(cxgb4i_sport_base, "Starting port number (default 20000)");
  77
  78 typedef void (*cxgb4i_cplhandler_func)(struct cxgbi_device *, struct sk_buff *);
  79
  80 static void *t4_uld_add(const struct cxgb4_lld_info *);
  81 static int t4_uld_rx_handler(void *, const __be64 *, const struct pkt_gl *);
  82 static int t4_uld_state_change(void *, enum cxgb4_state state);
  83 static inline int send_tx_flowc_wr(struct cxgbi_sock *);
  84
  85 static const struct cxgb4_uld_info cxgb4i_uld_info = {
  86         .name = DRV_MODULE_NAME,
  87         .nrxq = MAX_ULD_QSETS,
  88         .rxq_size = 1024,
  89         .lro = false,
  90         .add = t4_uld_add,
  91         .rx_handler = t4_uld_rx_handler,
  92         .state_change = t4_uld_state_change,
  93 };
  94
  95 static struct scsi_host_template cxgb4i_host_template = {
  96         .module         = THIS_MODULE,
  97         .name           = DRV_MODULE_NAME,
  98         .proc_name      = DRV_MODULE_NAME,
  99         .can_queue      = CXGB4I_SCSI_HOST_QDEPTH,
 100         .queuecommand   = iscsi_queuecommand,
 101         .change_queue_depth = scsi_change_queue_depth,
 102         .sg_tablesize   = SG_ALL,
 103         .max_sectors    = 0xFFFF,
 104         .cmd_per_lun    = ISCSI_DEF_CMD_PER_LUN,
 105         .eh_abort_handler = iscsi_eh_abort,
 106         .eh_device_reset_handler = iscsi_eh_device_reset,
 107         .eh_target_reset_handler = iscsi_eh_recover_target,
 108         .target_alloc   = iscsi_target_alloc,
 109         .use_clustering = DISABLE_CLUSTERING,
 110         .this_id        = -1,
 111         .track_queue_depth = 1,
 112 };
 113
 114 static struct iscsi_transport cxgb4i_iscsi_transport = {
 115         .owner          = THIS_MODULE,
 116         .name           = DRV_MODULE_NAME,
 117         .caps           = CAP_RECOVERY_L0 | CAP_MULTI_R2T | CAP_HDRDGST |
 118                                 CAP_DATADGST | CAP_DIGEST_OFFLOAD |
 119                                 CAP_PADDING_OFFLOAD | CAP_TEXT_NEGO,
 120         .attr_is_visible        = cxgbi_attr_is_visible,
 121         .get_host_param = cxgbi_get_host_param,
 122         .set_host_param = cxgbi_set_host_param,
 123         /* session management */
 124         .create_session = cxgbi_create_session,
 125         .destroy_session        = cxgbi_destroy_session,
 126         .get_session_param = iscsi_session_get_param,
 127         /* connection management */
 128         .create_conn    = cxgbi_create_conn,
 129         .bind_conn              = cxgbi_bind_conn,
 130         .destroy_conn   = iscsi_tcp_conn_teardown,
 131         .start_conn             = iscsi_conn_start,
 132         .stop_conn              = iscsi_conn_stop,
 133         .get_conn_param = iscsi_conn_get_param,
 134         .set_param      = cxgbi_set_conn_param,
 135         .get_stats      = cxgbi_get_conn_stats,
 136         /* pdu xmit req from user space */
 137         .send_pdu       = iscsi_conn_send_pdu,
 138         /* task */
 139         .init_task      = iscsi_tcp_task_init,
 140         .xmit_task      = iscsi_tcp_task_xmit,
 141         .cleanup_task   = cxgbi_cleanup_task,
 142         /* pdu */
 143         .alloc_pdu      = cxgbi_conn_alloc_pdu,
 144         .init_pdu       = cxgbi_conn_init_pdu,
 145         .xmit_pdu       = cxgbi_conn_xmit_pdu,
 146         .parse_pdu_itt  = cxgbi_parse_pdu_itt,
 147         /* TCP connect/disconnect */
 148         .get_ep_param   = cxgbi_get_ep_param,
 149         .ep_connect     = cxgbi_ep_connect,
 150         .ep_poll        = cxgbi_ep_poll,
 151         .ep_disconnect  = cxgbi_ep_disconnect,
 152         /* Error recovery timeout call */
 153         .session_recovery_timedout = iscsi_session_recovery_timedout,
 154 };
 155
 156 static struct scsi_transport_template *cxgb4i_stt;
 157
 158 /*
 159  * CPL (Chelsio Protocol Language) defines a message passing interface between
 160  * the host driver and Chelsio asic.
 161  * The section below implments CPLs that related to iscsi tcp connection
 162  * open/close/abort and data send/receive.
 163  */
 164
 165 #define RCV_BUFSIZ_MASK         0x3FFU
 166 #define MAX_IMM_TX_PKT_LEN      256
 167
 168 static int push_tx_frames(struct cxgbi_sock *, int);
 169
 170 /*
 171  * is_ofld_imm - check whether a packet can be sent as immediate data
 172  * @skb: the packet
 173  *
 174  * Returns true if a packet can be sent as an offload WR with immediate
 175  * data.  We currently use the same limit as for Ethernet packets.
 176  */
 177 static inline bool is_ofld_imm(const struct sk_buff *skb)
 178 {
 179         int len = skb->len;
 180
 181         if (likely(cxgbi_skcb_test_flag(skb, SKCBF_TX_NEED_HDR)))
 182                 len += sizeof(struct fw_ofld_tx_data_wr);
 183
 184         return len <= MAX_IMM_TX_PKT_LEN;
 185 }
 186
 187 static void send_act_open_req(struct cxgbi_sock *csk, struct sk_buff *skb,
 188                                 struct l2t_entry *e)
 189 {
 190         struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(csk->cdev);
 191         int t4 = is_t4(lldi->adapter_type);
 192         int wscale = cxgbi_sock_compute_wscale(csk->mss_idx);
 193         unsigned long long opt0;
 194         unsigned int opt2;
 195         unsigned int qid_atid = ((unsigned int)csk->atid) |
 196                                  (((unsigned int)csk->rss_qid) << 14);
 197
 198         opt0 = KEEP_ALIVE_F |
 199                 WND_SCALE_V(wscale) |
 200                 MSS_IDX_V(csk->mss_idx) |
 201                 L2T_IDX_V(((struct l2t_entry *)csk->l2t)->idx) |
 202                 TX_CHAN_V(csk->tx_chan) |
 203                 SMAC_SEL_V(csk->smac_idx) |
 204                 ULP_MODE_V(ULP_MODE_ISCSI) |
 205                 RCV_BUFSIZ_V(csk->rcv_win >> 10);
 206
 207         opt2 = RX_CHANNEL_V(0) |
 208                 RSS_QUEUE_VALID_F |
 209                 RSS_QUEUE_V(csk->rss_qid);
 210
 211         if (is_t4(lldi->adapter_type)) {
 212                 struct cpl_act_open_req *req =
 213                                 (struct cpl_act_open_req *)skb->head;
 214
 215                 INIT_TP_WR(req, 0);
 216                 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ,
 217                                         qid_atid));
 218                 req->local_port = csk->saddr.sin_port;
 219                 req->peer_port = csk->daddr.sin_port;
 220                 req->local_ip = csk->saddr.sin_addr.s_addr;
 221                 req->peer_ip = csk->daddr.sin_addr.s_addr;
 222                 req->opt0 = cpu_to_be64(opt0);
 223                 req->params = cpu_to_be32(cxgb4_select_ntuple(
 224                                         csk->cdev->ports[csk->port_id],
 225                                         csk->l2t));
 226                 opt2 |= RX_FC_VALID_F;
 227                 req->opt2 = cpu_to_be32(opt2);
 228
 229                 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
 230                         "csk t4 0x%p, %pI4:%u-%pI4:%u, atid %d, qid %u.\n",
 231                         csk, &req->local_ip, ntohs(req->local_port),
 232                         &req->peer_ip, ntohs(req->peer_port),
 233                         csk->atid, csk->rss_qid);
 234         } else {
 235                 struct cpl_t5_act_open_req *req =
 236                                 (struct cpl_t5_act_open_req *)skb->head;
 237                 u32 isn = (prandom_u32() & ~7UL) - 1;
 238
 239                 INIT_TP_WR(req, 0);
 240                 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ,
 241                                         qid_atid));
 242                 req->local_port = csk->saddr.sin_port;
 243                 req->peer_port = csk->daddr.sin_port;
 244                 req->local_ip = csk->saddr.sin_addr.s_addr;
 245                 req->peer_ip = csk->daddr.sin_addr.s_addr;
 246                 req->opt0 = cpu_to_be64(opt0);
 247                 req->params = cpu_to_be64(FILTER_TUPLE_V(
 248                                 cxgb4_select_ntuple(
 249                                         csk->cdev->ports[csk->port_id],
 250                                         csk->l2t)));
 251                 req->rsvd = cpu_to_be32(isn);
 252                 opt2 |= T5_ISS_VALID;
 253                 opt2 |= T5_OPT_2_VALID_F;
 254
 255                 req->opt2 = cpu_to_be32(opt2);
 256
 257                 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
 258                         "csk t5 0x%p, %pI4:%u-%pI4:%u, atid %d, qid %u.\n",
 259                         csk, &req->local_ip, ntohs(req->local_port),
 260                         &req->peer_ip, ntohs(req->peer_port),
 261                         csk->atid, csk->rss_qid);
 262         }
 263
 264         set_wr_txq(skb, CPL_PRIORITY_SETUP, csk->port_id);
 265
 266         pr_info_ipaddr("t%d csk 0x%p,%u,0x%lx,%u, rss_qid %u.\n",
 267                        (&csk->saddr), (&csk->daddr), t4 ? 4 : 5, csk,
 268                        csk->state, csk->flags, csk->atid, csk->rss_qid);
 269
 270         cxgb4_l2t_send(csk->cdev->ports[csk->port_id], skb, csk->l2t);
 271 }
 272
 273 #if IS_ENABLED(CONFIG_IPV6)
 274 static void send_act_open_req6(struct cxgbi_sock *csk, struct sk_buff *skb,
 275                                struct l2t_entry *e)
 276 {
 277         struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(csk->cdev);
 278         int t4 = is_t4(lldi->adapter_type);
 279         int wscale = cxgbi_sock_compute_wscale(csk->mss_idx);
 280         unsigned long long opt0;
 281         unsigned int opt2;
 282         unsigned int qid_atid = ((unsigned int)csk->atid) |
 283                                  (((unsigned int)csk->rss_qid) << 14);
 284
 285         opt0 = KEEP_ALIVE_F |
 286                 WND_SCALE_V(wscale) |
 287                 MSS_IDX_V(csk->mss_idx) |
 288                 L2T_IDX_V(((struct l2t_entry *)csk->l2t)->idx) |
 289                 TX_CHAN_V(csk->tx_chan) |
 290                 SMAC_SEL_V(csk->smac_idx) |
 291                 ULP_MODE_V(ULP_MODE_ISCSI) |
 292                 RCV_BUFSIZ_V(csk->rcv_win >> 10);
 293
 294         opt2 = RX_CHANNEL_V(0) |
 295                 RSS_QUEUE_VALID_F |
 296                 RX_FC_DISABLE_F |
 297                 RSS_QUEUE_V(csk->rss_qid);
 298
 299         if (t4) {
 300                 struct cpl_act_open_req6 *req =
 301                             (struct cpl_act_open_req6 *)skb->head;
 302
 303                 INIT_TP_WR(req, 0);
 304                 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6,
 305                                                             qid_atid));
 306                 req->local_port = csk->saddr6.sin6_port;
 307                 req->peer_port = csk->daddr6.sin6_port;
 308
 309                 req->local_ip_hi = *(__be64 *)(csk->saddr6.sin6_addr.s6_addr);
 310                 req->local_ip_lo = *(__be64 *)(csk->saddr6.sin6_addr.s6_addr +
 311                                                                     8);
 312                 req->peer_ip_hi = *(__be64 *)(csk->daddr6.sin6_addr.s6_addr);
 313                 req->peer_ip_lo = *(__be64 *)(csk->daddr6.sin6_addr.s6_addr +
 314                                                                     8);
 315
 316                 req->opt0 = cpu_to_be64(opt0);
 317
 318                 opt2 |= RX_FC_VALID_F;
 319                 req->opt2 = cpu_to_be32(opt2);
 320
 321                 req->params = cpu_to_be32(cxgb4_select_ntuple(
 322                                           csk->cdev->ports[csk->port_id],
 323                                           csk->l2t));
 324         } else {
 325                 struct cpl_t5_act_open_req6 *req =
 326                                 (struct cpl_t5_act_open_req6 *)skb->head;
 327
 328                 INIT_TP_WR(req, 0);
 329                 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6,
 330                                                             qid_atid));
 331                 req->local_port = csk->saddr6.sin6_port;
 332                 req->peer_port = csk->daddr6.sin6_port;
 333                 req->local_ip_hi = *(__be64 *)(csk->saddr6.sin6_addr.s6_addr);
 334                 req->local_ip_lo = *(__be64 *)(csk->saddr6.sin6_addr.s6_addr +
 335                                                                         8);
 336                 req->peer_ip_hi = *(__be64 *)(csk->daddr6.sin6_addr.s6_addr);
 337                 req->peer_ip_lo = *(__be64 *)(csk->daddr6.sin6_addr.s6_addr +
 338                                                                         8);
 339                 req->opt0 = cpu_to_be64(opt0);
 340
 341                 opt2 |= T5_OPT_2_VALID_F;
 342                 req->opt2 = cpu_to_be32(opt2);
 343
 344                 req->params = cpu_to_be64(FILTER_TUPLE_V(cxgb4_select_ntuple(
 345                                           csk->cdev->ports[csk->port_id],
 346                                           csk->l2t)));
 347         }
 348
 349         set_wr_txq(skb, CPL_PRIORITY_SETUP, csk->port_id);
 350
 351         pr_info("t%d csk 0x%p,%u,0x%lx,%u, [%pI6]:%u-[%pI6]:%u, rss_qid %u.\n",
 352                 t4 ? 4 : 5, csk, csk->state, csk->flags, csk->atid,
 353                 &csk->saddr6.sin6_addr, ntohs(csk->saddr.sin_port),
 354                 &csk->daddr6.sin6_addr, ntohs(csk->daddr.sin_port),
 355                 csk->rss_qid);
 356
 357         cxgb4_l2t_send(csk->cdev->ports[csk->port_id], skb, csk->l2t);
 358 }
 359 #endif
 360
 361 static void send_close_req(struct cxgbi_sock *csk)
 362 {
 363         struct sk_buff *skb = csk->cpl_close;
 364         struct cpl_close_con_req *req = (struct cpl_close_con_req *)skb->head;
 365         unsigned int tid = csk->tid;
 366
 367         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
 368                 "csk 0x%p,%u,0x%lx, tid %u.\n",
 369                 csk, csk->state, csk->flags, csk->tid);
 370         csk->cpl_close = NULL;
 371         set_wr_txq(skb, CPL_PRIORITY_DATA, csk->port_id);
 372         INIT_TP_WR(req, tid);
 373         OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid));
 374         req->rsvd = 0;
 375
 376         cxgbi_sock_skb_entail(csk, skb);
 377         if (csk->state >= CTP_ESTABLISHED)
 378                 push_tx_frames(csk, 1);
 379 }
 380
 381 static void abort_arp_failure(void *handle, struct sk_buff *skb)
 382 {
 383         struct cxgbi_sock *csk = (struct cxgbi_sock *)handle;
 384         struct cpl_abort_req *req;
 385
 386         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
 387                 "csk 0x%p,%u,0x%lx, tid %u, abort.\n",
 388                 csk, csk->state, csk->flags, csk->tid);
 389         req = (struct cpl_abort_req *)skb->data;
 390         req->cmd = CPL_ABORT_NO_RST;
 391         cxgb4_ofld_send(csk->cdev->ports[csk->port_id], skb);
 392 }
 393
 394 static void send_abort_req(struct cxgbi_sock *csk)
 395 {
 396         struct cpl_abort_req *req;
 397         struct sk_buff *skb = csk->cpl_abort_req;
 398
 399         if (unlikely(csk->state == CTP_ABORTING) || !skb || !csk->cdev)
 400                 return;
 401
 402         if (!cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT)) {
 403                 send_tx_flowc_wr(csk);
 404                 cxgbi_sock_set_flag(csk, CTPF_TX_DATA_SENT);
 405         }
 406
 407         cxgbi_sock_set_state(csk, CTP_ABORTING);
 408         cxgbi_sock_set_flag(csk, CTPF_ABORT_RPL_PENDING);
 409         cxgbi_sock_purge_write_queue(csk);
 410
 411         csk->cpl_abort_req = NULL;
 412         req = (struct cpl_abort_req *)skb->head;
 413         set_wr_txq(skb, CPL_PRIORITY_DATA, csk->port_id);
 414         req->cmd = CPL_ABORT_SEND_RST;
 415         t4_set_arp_err_handler(skb, csk, abort_arp_failure);
 416         INIT_TP_WR(req, csk->tid);
 417         OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_REQ, csk->tid));
 418         req->rsvd0 = htonl(csk->snd_nxt);
 419         req->rsvd1 = !cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT);
 420
 421         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
 422                 "csk 0x%p,%u,0x%lx,%u, snd_nxt %u, 0x%x.\n",
 423                 csk, csk->state, csk->flags, csk->tid, csk->snd_nxt,
 424                 req->rsvd1);
 425
 426         cxgb4_l2t_send(csk->cdev->ports[csk->port_id], skb, csk->l2t);
 427 }
 428
 429 static void send_abort_rpl(struct cxgbi_sock *csk, int rst_status)
 430 {
 431         struct sk_buff *skb = csk->cpl_abort_rpl;
 432         struct cpl_abort_rpl *rpl = (struct cpl_abort_rpl *)skb->head;
 433
 434         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
 435                 "csk 0x%p,%u,0x%lx,%u, status %d.\n",
 436                 csk, csk->state, csk->flags, csk->tid, rst_status);
 437
 438         csk->cpl_abort_rpl = NULL;
 439         set_wr_txq(skb, CPL_PRIORITY_DATA, csk->port_id);
 440         INIT_TP_WR(rpl, csk->tid);
 441         OPCODE_TID(rpl) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_RPL, csk->tid));
 442         rpl->cmd = rst_status;
 443         cxgb4_ofld_send(csk->cdev->ports[csk->port_id], skb);
 444 }
 445
 446 /*
 447  * CPL connection rx data ack: host ->
 448  * Send RX credits through an RX_DATA_ACK CPL message. Returns the number of
 449  * credits sent.
 450  */
 451 static u32 send_rx_credits(struct cxgbi_sock *csk, u32 credits)
 452 {
 453         struct sk_buff *skb;
 454         struct cpl_rx_data_ack *req;
 455
 456         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
 457                 "csk 0x%p,%u,0x%lx,%u, credit %u.\n",
 458                 csk, csk->state, csk->flags, csk->tid, credits);
 459
 460         skb = alloc_wr(sizeof(*req), 0, GFP_ATOMIC);
 461         if (!skb) {
 462                 pr_info("csk 0x%p, credit %u, OOM.\n", csk, credits);
 463                 return 0;
 464         }
 465         req = (struct cpl_rx_data_ack *)skb->head;
 466
 467         set_wr_txq(skb, CPL_PRIORITY_ACK, csk->port_id);
 468         INIT_TP_WR(req, csk->tid);
 469         OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_RX_DATA_ACK,
 470                                       csk->tid));
 471         req->credit_dack = cpu_to_be32(RX_CREDITS_V(credits)
 472                                        | RX_FORCE_ACK_F);
 473         cxgb4_ofld_send(csk->cdev->ports[csk->port_id], skb);
 474         return credits;
 475 }
 476
 477 /*
 478  * sgl_len - calculates the size of an SGL of the given capacity
 479  * @n: the number of SGL entries
 480  * Calculates the number of flits needed for a scatter/gather list that
 481  * can hold the given number of entries.
 482  */
 483 static inline unsigned int sgl_len(unsigned int n)
 484 {
 485         n--;
 486         return (3 * n) / 2 + (n & 1) + 2;
 487 }
 488
 489 /*
 490  * calc_tx_flits_ofld - calculate # of flits for an offload packet
 491  * @skb: the packet
 492  *
 493  * Returns the number of flits needed for the given offload packet.
 494  * These packets are already fully constructed and no additional headers
 495  * will be added.
 496  */
 497 static inline unsigned int calc_tx_flits_ofld(const struct sk_buff *skb)
 498 {
 499         unsigned int flits, cnt;
 500
 501         if (is_ofld_imm(skb))
 502                 return DIV_ROUND_UP(skb->len, 8);
 503         flits = skb_transport_offset(skb) / 8;
 504         cnt = skb_shinfo(skb)->nr_frags;
 505         if (skb_tail_pointer(skb) != skb_transport_header(skb))
 506                 cnt++;
 507         return flits + sgl_len(cnt);
 508 }
 509
 510 #define FLOWC_WR_NPARAMS_MIN    9
 511 static inline int tx_flowc_wr_credits(int *nparamsp, int *flowclenp)
 512 {
 513         int nparams, flowclen16, flowclen;
 514
 515         nparams = FLOWC_WR_NPARAMS_MIN;
 516         flowclen = offsetof(struct fw_flowc_wr, mnemval[nparams]);
 517         flowclen16 = DIV_ROUND_UP(flowclen, 16);
 518         flowclen = flowclen16 * 16;
 519         /*
 520          * Return the number of 16-byte credits used by the FlowC request.
 521          * Pass back the nparams and actual FlowC length if requested.
 522          */
 523         if (nparamsp)
 524                 *nparamsp = nparams;
 525         if (flowclenp)
 526                 *flowclenp = flowclen;
 527
 528         return flowclen16;
 529 }
 530
 531 static inline int send_tx_flowc_wr(struct cxgbi_sock *csk)
 532 {
 533         struct sk_buff *skb;
 534         struct fw_flowc_wr *flowc;
 535         int nparams, flowclen16, flowclen;
 536
 537         flowclen16 = tx_flowc_wr_credits(&nparams, &flowclen);
 538         skb = alloc_wr(flowclen, 0, GFP_ATOMIC);
 539         flowc = (struct fw_flowc_wr *)skb->head;
 540         flowc->op_to_nparams =
 541                 htonl(FW_WR_OP_V(FW_FLOWC_WR) | FW_FLOWC_WR_NPARAMS_V(nparams));
 542         flowc->flowid_len16 =
 543                 htonl(FW_WR_LEN16_V(flowclen16) | FW_WR_FLOWID_V(csk->tid));
 544         flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_PFNVFN;
 545         flowc->mnemval[0].val = htonl(csk->cdev->pfvf);
 546         flowc->mnemval[1].mnemonic = FW_FLOWC_MNEM_CH;
 547         flowc->mnemval[1].val = htonl(csk->tx_chan);
 548         flowc->mnemval[2].mnemonic = FW_FLOWC_MNEM_PORT;
 549         flowc->mnemval[2].val = htonl(csk->tx_chan);
 550         flowc->mnemval[3].mnemonic = FW_FLOWC_MNEM_IQID;
 551         flowc->mnemval[3].val = htonl(csk->rss_qid);
 552         flowc->mnemval[4].mnemonic = FW_FLOWC_MNEM_SNDNXT;
 553         flowc->mnemval[4].val = htonl(csk->snd_nxt);
 554         flowc->mnemval[5].mnemonic = FW_FLOWC_MNEM_RCVNXT;
 555         flowc->mnemval[5].val = htonl(csk->rcv_nxt);
 556         flowc->mnemval[6].mnemonic = FW_FLOWC_MNEM_SNDBUF;
 557         flowc->mnemval[6].val = htonl(csk->snd_win);
 558         flowc->mnemval[7].mnemonic = FW_FLOWC_MNEM_MSS;
 559         flowc->mnemval[7].val = htonl(csk->advmss);
 560         flowc->mnemval[8].mnemonic = 0;
 561         flowc->mnemval[8].val = 0;
 562         flowc->mnemval[8].mnemonic = FW_FLOWC_MNEM_TXDATAPLEN_MAX;
 563         flowc->mnemval[8].val = 16384;
 564
 565         set_wr_txq(skb, CPL_PRIORITY_DATA, csk->port_id);
 566
 567         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
 568                 "csk 0x%p, tid 0x%x, %u,%u,%u,%u,%u,%u,%u.\n",
 569                 csk, csk->tid, 0, csk->tx_chan, csk->rss_qid,
 570                 csk->snd_nxt, csk->rcv_nxt, csk->snd_win,
 571                 csk->advmss);
 572
 573         cxgb4_ofld_send(csk->cdev->ports[csk->port_id], skb);
 574
 575         return flowclen16;
 576 }
 577
 578 static inline void make_tx_data_wr(struct cxgbi_sock *csk, struct sk_buff *skb,
 579                                    int dlen, int len, u32 credits, int compl)
 580 {
 581         struct fw_ofld_tx_data_wr *req;
 582         unsigned int submode = cxgbi_skcb_ulp_mode(skb) & 3;
 583         unsigned int wr_ulp_mode = 0, val;
 584         bool imm = is_ofld_imm(skb);
 585
 586         req = (struct fw_ofld_tx_data_wr *)__skb_push(skb, sizeof(*req));
 587
 588         if (imm) {
 589                 req->op_to_immdlen = htonl(FW_WR_OP_V(FW_OFLD_TX_DATA_WR) |
 590                                         FW_WR_COMPL_F |
 591                                         FW_WR_IMMDLEN_V(dlen));
 592                 req->flowid_len16 = htonl(FW_WR_FLOWID_V(csk->tid) |
 593                                                 FW_WR_LEN16_V(credits));
 594         } else {
 595                 req->op_to_immdlen =
 596                         cpu_to_be32(FW_WR_OP_V(FW_OFLD_TX_DATA_WR) |
 597                                         FW_WR_COMPL_F |
 598                                         FW_WR_IMMDLEN_V(0));
 599                 req->flowid_len16 =
 600                         cpu_to_be32(FW_WR_FLOWID_V(csk->tid) |
 601                                         FW_WR_LEN16_V(credits));
 602         }
 603         if (submode)
 604                 wr_ulp_mode = FW_OFLD_TX_DATA_WR_ULPMODE_V(ULP2_MODE_ISCSI) |
 605                                 FW_OFLD_TX_DATA_WR_ULPSUBMODE_V(submode);
 606         val = skb_peek(&csk->write_queue) ? 0 : 1;
 607         req->tunnel_to_proxy = htonl(wr_ulp_mode |
 608                                      FW_OFLD_TX_DATA_WR_SHOVE_V(val));
 609         req->plen = htonl(len);
 610         if (!cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT))
 611                 cxgbi_sock_set_flag(csk, CTPF_TX_DATA_SENT);
 612 }
 613
 614 static void arp_failure_skb_discard(void *handle, struct sk_buff *skb)
 615 {
 616         kfree_skb(skb);
 617 }
 618
 619 static int push_tx_frames(struct cxgbi_sock *csk, int req_completion)
 620 {
 621         int total_size = 0;
 622         struct sk_buff *skb;
 623
 624         if (unlikely(csk->state < CTP_ESTABLISHED ||
 625                 csk->state == CTP_CLOSE_WAIT_1 || csk->state >= CTP_ABORTING)) {
 626                 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK |
 627                          1 << CXGBI_DBG_PDU_TX,
 628                         "csk 0x%p,%u,0x%lx,%u, in closing state.\n",
 629                         csk, csk->state, csk->flags, csk->tid);
 630                 return 0;
 631         }
 632
 633         while (csk->wr_cred && (skb = skb_peek(&csk->write_queue)) != NULL) {
 634                 int dlen = skb->len;
 635                 int len = skb->len;
 636                 unsigned int credits_needed;
 637                 int flowclen16 = 0;
 638
 639                 skb_reset_transport_header(skb);
 640                 if (is_ofld_imm(skb))
 641                         credits_needed = DIV_ROUND_UP(dlen, 16);
 642                 else
 643                         credits_needed = DIV_ROUND_UP(
 644                                                 8 * calc_tx_flits_ofld(skb),
 645                                                 16);
 646
 647                 if (likely(cxgbi_skcb_test_flag(skb, SKCBF_TX_NEED_HDR)))
 648                         credits_needed += DIV_ROUND_UP(
 649                                         sizeof(struct fw_ofld_tx_data_wr),
 650                                         16);
 651
 652                 /*
 653                  * Assumes the initial credits is large enough to support
 654                  * fw_flowc_wr plus largest possible first payload
 655                  */
 656                 if (!cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT)) {
 657                         flowclen16 = send_tx_flowc_wr(csk);
 658                         csk->wr_cred -= flowclen16;
 659                         csk->wr_una_cred += flowclen16;
 660                         cxgbi_sock_set_flag(csk, CTPF_TX_DATA_SENT);
 661                 }
 662
 663                 if (csk->wr_cred < credits_needed) {
 664                         log_debug(1 << CXGBI_DBG_PDU_TX,
 665                                 "csk 0x%p, skb %u/%u, wr %d < %u.\n",
 666                                 csk, skb->len, skb->data_len,
 667                                 credits_needed, csk->wr_cred);
 668                         break;
 669                 }
 670                 __skb_unlink(skb, &csk->write_queue);
 671                 set_wr_txq(skb, CPL_PRIORITY_DATA, csk->port_id);
 672                 skb->csum = credits_needed + flowclen16;
 673                 csk->wr_cred -= credits_needed;
 674                 csk->wr_una_cred += credits_needed;
 675                 cxgbi_sock_enqueue_wr(csk, skb);
 676
 677                 log_debug(1 << CXGBI_DBG_PDU_TX,
 678                         "csk 0x%p, skb %u/%u, wr %d, left %u, unack %u.\n",
 679                         csk, skb->len, skb->data_len, credits_needed,
 680                         csk->wr_cred, csk->wr_una_cred);
 681
 682                 if (likely(cxgbi_skcb_test_flag(skb, SKCBF_TX_NEED_HDR))) {
 683                         len += cxgbi_ulp_extra_len(cxgbi_skcb_ulp_mode(skb));
 684                         make_tx_data_wr(csk, skb, dlen, len, credits_needed,
 685                                         req_completion);
 686                         csk->snd_nxt += len;
 687                         cxgbi_skcb_clear_flag(skb, SKCBF_TX_NEED_HDR);
 688                 }
 689                 total_size += skb->truesize;
 690                 t4_set_arp_err_handler(skb, csk, arp_failure_skb_discard);
 691
 692                 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_TX,
 693                         "csk 0x%p,%u,0x%lx,%u, skb 0x%p, %u.\n",
 694                         csk, csk->state, csk->flags, csk->tid, skb, len);
 695
 696                 cxgb4_l2t_send(csk->cdev->ports[csk->port_id], skb, csk->l2t);
 697         }
 698         return total_size;
 699 }
 700
 701 static inline void free_atid(struct cxgbi_sock *csk)
 702 {
 703         struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(csk->cdev);
 704
 705         if (cxgbi_sock_flag(csk, CTPF_HAS_ATID)) {
 706                 cxgb4_free_atid(lldi->tids, csk->atid);
 707                 cxgbi_sock_clear_flag(csk, CTPF_HAS_ATID);
 708                 cxgbi_sock_put(csk);
 709         }
 710 }
 711
 712 static void do_act_establish(struct cxgbi_device *cdev, struct sk_buff *skb)
 713 {
 714         struct cxgbi_sock *csk;
 715         struct cpl_act_establish *req = (struct cpl_act_establish *)skb->data;
 716         unsigned short tcp_opt = ntohs(req->tcp_opt);
 717         unsigned int tid = GET_TID(req);
 718         unsigned int atid = TID_TID_G(ntohl(req->tos_atid));
 719         struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
 720         struct tid_info *t = lldi->tids;
 721         u32 rcv_isn = be32_to_cpu(req->rcv_isn);
 722
 723         csk = lookup_atid(t, atid);
 724         if (unlikely(!csk)) {
 725                 pr_err("NO conn. for atid %u, cdev 0x%p.\n", atid, cdev);
 726                 goto rel_skb;
 727         }
 728
 729         if (csk->atid != atid) {
 730                 pr_err("bad conn atid %u, csk 0x%p,%u,0x%lx,tid %u, atid %u.\n",
 731                         atid, csk, csk->state, csk->flags, csk->tid, csk->atid);
 732                 goto rel_skb;
 733         }
 734
 735         pr_info_ipaddr("atid 0x%x, tid 0x%x, csk 0x%p,%u,0x%lx, isn %u.\n",
 736                        (&csk->saddr), (&csk->daddr),
 737                        atid, tid, csk, csk->state, csk->flags, rcv_isn);
 738
 739         module_put(THIS_MODULE);
 740
 741         cxgbi_sock_get(csk);
 742         csk->tid = tid;
 743         cxgb4_insert_tid(lldi->tids, csk, tid);
 744         cxgbi_sock_set_flag(csk, CTPF_HAS_TID);
 745
 746         free_atid(csk);
 747
 748         spin_lock_bh(&csk->lock);
 749         if (unlikely(csk->state != CTP_ACTIVE_OPEN))
 750                 pr_info("csk 0x%p,%u,0x%lx,%u, got EST.\n",
 751                         csk, csk->state, csk->flags, csk->tid);
 752
 753         if (csk->retry_timer.function) {
 754                 del_timer(&csk->retry_timer);
 755                 csk->retry_timer.function = NULL;
 756         }
 757
 758         csk->copied_seq = csk->rcv_wup = csk->rcv_nxt = rcv_isn;
 759         /*
 760          * Causes the first RX_DATA_ACK to supply any Rx credits we couldn't
 761          * pass through opt0.
 762          */
 763         if (csk->rcv_win > (RCV_BUFSIZ_MASK << 10))
 764                 csk->rcv_wup -= csk->rcv_win - (RCV_BUFSIZ_MASK << 10);
 765
 766         csk->advmss = lldi->mtus[TCPOPT_MSS_G(tcp_opt)] - 40;
 767         if (TCPOPT_TSTAMP_G(tcp_opt))
 768                 csk->advmss -= 12;
 769         if (csk->advmss < 128)
 770                 csk->advmss = 128;
 771
 772         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
 773                 "csk 0x%p, mss_idx %u, advmss %u.\n",
 774                         csk, TCPOPT_MSS_G(tcp_opt), csk->advmss);
 775
 776         cxgbi_sock_established(csk, ntohl(req->snd_isn), ntohs(req->tcp_opt));
 777
 778         if (unlikely(cxgbi_sock_flag(csk, CTPF_ACTIVE_CLOSE_NEEDED)))
 779                 send_abort_req(csk);
 780         else {
 781                 if (skb_queue_len(&csk->write_queue))
 782                         push_tx_frames(csk, 0);
 783                 cxgbi_conn_tx_open(csk);
 784         }
 785         spin_unlock_bh(&csk->lock);
 786
 787 rel_skb:
 788         __kfree_skb(skb);
 789 }
 790
 791 static int act_open_rpl_status_to_errno(int status)
 792 {
 793         switch (status) {
 794         case CPL_ERR_CONN_RESET:
 795                 return -ECONNREFUSED;
 796         case CPL_ERR_ARP_MISS:
 797                 return -EHOSTUNREACH;
 798         case CPL_ERR_CONN_TIMEDOUT:
 799                 return -ETIMEDOUT;
 800         case CPL_ERR_TCAM_FULL:
 801                 return -ENOMEM;
 802         case CPL_ERR_CONN_EXIST:
 803                 return -EADDRINUSE;
 804         default:
 805                 return -EIO;
 806         }
 807 }
 808
 809 static void csk_act_open_retry_timer(unsigned long data)
 810 {
 811         struct sk_buff *skb = NULL;
 812         struct cxgbi_sock *csk = (struct cxgbi_sock *)data;
 813         struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(csk->cdev);
 814         void (*send_act_open_func)(struct cxgbi_sock *, struct sk_buff *,
 815                                    struct l2t_entry *);
 816         int t4 = is_t4(lldi->adapter_type), size, size6;
 817
 818         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
 819                 "csk 0x%p,%u,0x%lx,%u.\n",
 820                 csk, csk->state, csk->flags, csk->tid);
 821
 822         cxgbi_sock_get(csk);
 823         spin_lock_bh(&csk->lock);
 824
 825         if (t4) {
 826                 size = sizeof(struct cpl_act_open_req);
 827                 size6 = sizeof(struct cpl_act_open_req6);
 828         } else {
 829                 size = sizeof(struct cpl_t5_act_open_req);
 830                 size6 = sizeof(struct cpl_t5_act_open_req6);
 831         }
 832
 833         if (csk->csk_family == AF_INET) {
 834                 send_act_open_func = send_act_open_req;
 835                 skb = alloc_wr(size, 0, GFP_ATOMIC);
 836 #if IS_ENABLED(CONFIG_IPV6)
 837         } else {
 838                 send_act_open_func = send_act_open_req6;
 839                 skb = alloc_wr(size6, 0, GFP_ATOMIC);
 840 #endif
 841         }
 842
 843         if (!skb)
 844                 cxgbi_sock_fail_act_open(csk, -ENOMEM);
 845         else {
 846                 skb->sk = (struct sock *)csk;
 847                 t4_set_arp_err_handler(skb, csk,
 848                                        cxgbi_sock_act_open_req_arp_failure);
 849                 send_act_open_func(csk, skb, csk->l2t);
 850         }
 851
 852         spin_unlock_bh(&csk->lock);
 853         cxgbi_sock_put(csk);
 854
 855 }
 856
 857 static inline bool is_neg_adv(unsigned int status)
 858 {
 859         return status == CPL_ERR_RTX_NEG_ADVICE ||
 860                 status == CPL_ERR_KEEPALV_NEG_ADVICE ||
 861                 status == CPL_ERR_PERSIST_NEG_ADVICE;
 862 }
 863
 864 static void do_act_open_rpl(struct cxgbi_device *cdev, struct sk_buff *skb)
 865 {
 866         struct cxgbi_sock *csk;
 867         struct cpl_act_open_rpl *rpl = (struct cpl_act_open_rpl *)skb->data;
 868         unsigned int tid = GET_TID(rpl);
 869         unsigned int atid =
 870                 TID_TID_G(AOPEN_ATID_G(be32_to_cpu(rpl->atid_status)));
 871         unsigned int status = AOPEN_STATUS_G(be32_to_cpu(rpl->atid_status));
 872         struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
 873         struct tid_info *t = lldi->tids;
 874
 875         csk = lookup_atid(t, atid);
 876         if (unlikely(!csk)) {
 877                 pr_err("NO matching conn. atid %u, tid %u.\n", atid, tid);
 878                 goto rel_skb;
 879         }
 880
 881         pr_info_ipaddr("tid %u/%u, status %u.\n"
 882                        "csk 0x%p,%u,0x%lx. ", (&csk->saddr), (&csk->daddr),
 883                        atid, tid, status, csk, csk->state, csk->flags);
 884
 885         if (is_neg_adv(status))
 886                 goto rel_skb;
 887
 888         module_put(THIS_MODULE);
 889
 890         if (status && status != CPL_ERR_TCAM_FULL &&
 891             status != CPL_ERR_CONN_EXIST &&
 892             status != CPL_ERR_ARP_MISS)
 893                 cxgb4_remove_tid(lldi->tids, csk->port_id, GET_TID(rpl));
 894
 895         cxgbi_sock_get(csk);
 896         spin_lock_bh(&csk->lock);
 897
 898         if (status == CPL_ERR_CONN_EXIST &&
 899             csk->retry_timer.function != csk_act_open_retry_timer) {
 900                 csk->retry_timer.function = csk_act_open_retry_timer;
 901                 mod_timer(&csk->retry_timer, jiffies + HZ / 2);
 902         } else
 903                 cxgbi_sock_fail_act_open(csk,
 904                                         act_open_rpl_status_to_errno(status));
 905
 906         spin_unlock_bh(&csk->lock);
 907         cxgbi_sock_put(csk);
 908 rel_skb:
 909         __kfree_skb(skb);
 910 }
 911
 912 static void do_peer_close(struct cxgbi_device *cdev, struct sk_buff *skb)
 913 {
 914         struct cxgbi_sock *csk;
 915         struct cpl_peer_close *req = (struct cpl_peer_close *)skb->data;
 916         unsigned int tid = GET_TID(req);
 917         struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
 918         struct tid_info *t = lldi->tids;
 919
 920         csk = lookup_tid(t, tid);
 921         if (unlikely(!csk)) {
 922                 pr_err("can't find connection for tid %u.\n", tid);
 923                 goto rel_skb;
 924         }
 925         pr_info_ipaddr("csk 0x%p,%u,0x%lx,%u.\n",
 926                        (&csk->saddr), (&csk->daddr),
 927                        csk, csk->state, csk->flags, csk->tid);
 928         cxgbi_sock_rcv_peer_close(csk);
 929 rel_skb:
 930         __kfree_skb(skb);
 931 }
 932
 933 static void do_close_con_rpl(struct cxgbi_device *cdev, struct sk_buff *skb)
 934 {
 935         struct cxgbi_sock *csk;
 936         struct cpl_close_con_rpl *rpl = (struct cpl_close_con_rpl *)skb->data;
 937         unsigned int tid = GET_TID(rpl);
 938         struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
 939         struct tid_info *t = lldi->tids;
 940
 941         csk = lookup_tid(t, tid);
 942         if (unlikely(!csk)) {
 943                 pr_err("can't find connection for tid %u.\n", tid);
 944                 goto rel_skb;
 945         }
 946         pr_info_ipaddr("csk 0x%p,%u,0x%lx,%u.\n",
 947                        (&csk->saddr), (&csk->daddr),
 948                        csk, csk->state, csk->flags, csk->tid);
 949         cxgbi_sock_rcv_close_conn_rpl(csk, ntohl(rpl->snd_nxt));
 950 rel_skb:
 951         __kfree_skb(skb);
 952 }
 953
 954 static int abort_status_to_errno(struct cxgbi_sock *csk, int abort_reason,
 955                                                                 int *need_rst)
 956 {
 957         switch (abort_reason) {
 958         case CPL_ERR_BAD_SYN: /* fall through */
 959         case CPL_ERR_CONN_RESET:
 960                 return csk->state > CTP_ESTABLISHED ?
 961                         -EPIPE : -ECONNRESET;
 962         case CPL_ERR_XMIT_TIMEDOUT:
 963         case CPL_ERR_PERSIST_TIMEDOUT:
 964         case CPL_ERR_FINWAIT2_TIMEDOUT:
 965         case CPL_ERR_KEEPALIVE_TIMEDOUT:
 966                 return -ETIMEDOUT;
 967         default:
 968                 return -EIO;
 969         }
 970 }
 971
 972 static void do_abort_req_rss(struct cxgbi_device *cdev, struct sk_buff *skb)
 973 {
 974         struct cxgbi_sock *csk;
 975         struct cpl_abort_req_rss *req = (struct cpl_abort_req_rss *)skb->data;
 976         unsigned int tid = GET_TID(req);
 977         struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
 978         struct tid_info *t = lldi->tids;
 979         int rst_status = CPL_ABORT_NO_RST;
 980
 981         csk = lookup_tid(t, tid);
 982         if (unlikely(!csk)) {
 983                 pr_err("can't find connection for tid %u.\n", tid);
 984                 goto rel_skb;
 985         }
 986
 987         pr_info_ipaddr("csk 0x%p,%u,0x%lx,%u, status %u.\n",
 988                        (&csk->saddr), (&csk->daddr),
 989                        csk, csk->state, csk->flags, csk->tid, req->status);
 990
 991         if (is_neg_adv(req->status))
 992                 goto rel_skb;
 993
 994         cxgbi_sock_get(csk);
 995         spin_lock_bh(&csk->lock);
 996
 997         cxgbi_sock_clear_flag(csk, CTPF_ABORT_REQ_RCVD);
 998
 999         if (!cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT)) {
1000                 send_tx_flowc_wr(csk);
1001                 cxgbi_sock_set_flag(csk, CTPF_TX_DATA_SENT);
1002         }
1003
1004         cxgbi_sock_set_flag(csk, CTPF_ABORT_REQ_RCVD);
1005         cxgbi_sock_set_state(csk, CTP_ABORTING);
1006
1007         send_abort_rpl(csk, rst_status);
1008
1009         if (!cxgbi_sock_flag(csk, CTPF_ABORT_RPL_PENDING)) {
1010                 csk->err = abort_status_to_errno(csk, req->status, &rst_status);
1011                 cxgbi_sock_closed(csk);
1012         }
1013
1014         spin_unlock_bh(&csk->lock);
1015         cxgbi_sock_put(csk);
1016 rel_skb:
1017         __kfree_skb(skb);
1018 }
1019
1020 static void do_abort_rpl_rss(struct cxgbi_device *cdev, struct sk_buff *skb)
1021 {
1022         struct cxgbi_sock *csk;
1023         struct cpl_abort_rpl_rss *rpl = (struct cpl_abort_rpl_rss *)skb->data;
1024         unsigned int tid = GET_TID(rpl);
1025         struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
1026         struct tid_info *t = lldi->tids;
1027
1028         csk = lookup_tid(t, tid);
1029         if (!csk)
1030                 goto rel_skb;
1031
1032         if (csk)
1033                 pr_info_ipaddr("csk 0x%p,%u,0x%lx,%u, status %u.\n",
1034                                (&csk->saddr), (&csk->daddr), csk,
1035                                csk->state, csk->flags, csk->tid, rpl->status);
1036
1037         if (rpl->status == CPL_ERR_ABORT_FAILED)
1038                 goto rel_skb;
1039
1040         cxgbi_sock_rcv_abort_rpl(csk);
1041 rel_skb:
1042         __kfree_skb(skb);
1043 }
1044
1045 static void do_rx_data(struct cxgbi_device *cdev, struct sk_buff *skb)
1046 {
1047         struct cxgbi_sock *csk;
1048         struct cpl_rx_data *cpl = (struct cpl_rx_data *)skb->data;
1049         unsigned int tid = GET_TID(cpl);
1050         struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
1051         struct tid_info *t = lldi->tids;
1052
1053         csk = lookup_tid(t, tid);
1054         if (!csk) {
1055                 pr_err("can't find connection for tid %u.\n", tid);
1056         } else {
1057                 /* not expecting this, reset the connection. */
1058                 pr_err("csk 0x%p, tid %u, rcv cpl_rx_data.\n", csk, tid);
1059                 spin_lock_bh(&csk->lock);
1060                 send_abort_req(csk);
1061                 spin_unlock_bh(&csk->lock);
1062         }
1063         __kfree_skb(skb);
1064 }
1065
1066 static void do_rx_iscsi_hdr(struct cxgbi_device *cdev, struct sk_buff *skb)
1067 {
1068         struct cxgbi_sock *csk;
1069         struct cpl_iscsi_hdr *cpl = (struct cpl_iscsi_hdr *)skb->data;
1070         unsigned short pdu_len_ddp = be16_to_cpu(cpl->pdu_len_ddp);
1071         unsigned int tid = GET_TID(cpl);
1072         struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
1073         struct tid_info *t = lldi->tids;
1074
1075         csk = lookup_tid(t, tid);
1076         if (unlikely(!csk)) {
1077                 pr_err("can't find conn. for tid %u.\n", tid);
1078                 goto rel_skb;
1079         }
1080
1081         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
1082                 "csk 0x%p,%u,0x%lx, tid %u, skb 0x%p,%u, 0x%x.\n",
1083                 csk, csk->state, csk->flags, csk->tid, skb, skb->len,
1084                 pdu_len_ddp);
1085
1086         spin_lock_bh(&csk->lock);
1087
1088         if (unlikely(csk->state >= CTP_PASSIVE_CLOSE)) {
1089                 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
1090                         "csk 0x%p,%u,0x%lx,%u, bad state.\n",
1091                         csk, csk->state, csk->flags, csk->tid);
1092                 if (csk->state != CTP_ABORTING)
1093                         goto abort_conn;
1094                 else
1095                         goto discard;
1096         }
1097
1098         cxgbi_skcb_tcp_seq(skb) = ntohl(cpl->seq);
1099         cxgbi_skcb_flags(skb) = 0;
1100
1101         skb_reset_transport_header(skb);
1102         __skb_pull(skb, sizeof(*cpl));
1103         __pskb_trim(skb, ntohs(cpl->len));
1104
1105         if (!csk->skb_ulp_lhdr) {
1106                 unsigned char *bhs;
1107                 unsigned int hlen, dlen, plen;
1108
1109                 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
1110                         "csk 0x%p,%u,0x%lx, tid %u, skb 0x%p header.\n",
1111                         csk, csk->state, csk->flags, csk->tid, skb);
1112                 csk->skb_ulp_lhdr = skb;
1113                 cxgbi_skcb_set_flag(skb, SKCBF_RX_HDR);
1114
1115                 if (cxgbi_skcb_tcp_seq(skb) != csk->rcv_nxt) {
1116                         pr_info("tid %u, CPL_ISCSI_HDR, bad seq, 0x%x/0x%x.\n",
1117                                 csk->tid, cxgbi_skcb_tcp_seq(skb),
1118                                 csk->rcv_nxt);
1119                         goto abort_conn;
1120                 }
1121
1122                 bhs = skb->data;
1123                 hlen = ntohs(cpl->len);
1124                 dlen = ntohl(*(unsigned int *)(bhs + 4)) & 0xFFFFFF;
1125
1126                 plen = ISCSI_PDU_LEN_G(pdu_len_ddp);
1127                 if (is_t4(lldi->adapter_type))
1128                         plen -= 40;
1129
1130                 if ((hlen + dlen) != plen) {
1131                         pr_info("tid 0x%x, CPL_ISCSI_HDR, pdu len "
1132                                 "mismatch %u != %u + %u, seq 0x%x.\n",
1133                                 csk->tid, plen, hlen, dlen,
1134                                 cxgbi_skcb_tcp_seq(skb));
1135                         goto abort_conn;
1136                 }
1137
1138                 cxgbi_skcb_rx_pdulen(skb) = (hlen + dlen + 3) & (~0x3);
1139                 if (dlen)
1140                         cxgbi_skcb_rx_pdulen(skb) += csk->dcrc_len;
1141                 csk->rcv_nxt += cxgbi_skcb_rx_pdulen(skb);
1142
1143                 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
1144                         "csk 0x%p, skb 0x%p, 0x%x,%u+%u,0x%x,0x%x.\n",
1145                         csk, skb, *bhs, hlen, dlen,
1146                         ntohl(*((unsigned int *)(bhs + 16))),
1147                         ntohl(*((unsigned int *)(bhs + 24))));
1148
1149         } else {
1150                 struct sk_buff *lskb = csk->skb_ulp_lhdr;
1151
1152                 cxgbi_skcb_set_flag(lskb, SKCBF_RX_DATA);
1153                 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
1154                         "csk 0x%p,%u,0x%lx, skb 0x%p data, 0x%p.\n",
1155                         csk, csk->state, csk->flags, skb, lskb);
1156         }
1157
1158         __skb_queue_tail(&csk->receive_queue, skb);
1159         spin_unlock_bh(&csk->lock);
1160         return;
1161
1162 abort_conn:
1163         send_abort_req(csk);
1164 discard:
1165         spin_unlock_bh(&csk->lock);
1166 rel_skb:
1167         __kfree_skb(skb);
1168 }
1169
1170 static void do_rx_data_ddp(struct cxgbi_device *cdev,
1171                                   struct sk_buff *skb)
1172 {
1173         struct cxgbi_sock *csk;
1174         struct sk_buff *lskb;
1175         struct cpl_rx_data_ddp *rpl = (struct cpl_rx_data_ddp *)skb->data;
1176         unsigned int tid = GET_TID(rpl);
1177         struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
1178         struct tid_info *t = lldi->tids;
1179         unsigned int status = ntohl(rpl->ddpvld);
1180
1181         csk = lookup_tid(t, tid);
1182         if (unlikely(!csk)) {
1183                 pr_err("can't find connection for tid %u.\n", tid);
1184                 goto rel_skb;
1185         }
1186
1187         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
1188                 "csk 0x%p,%u,0x%lx, skb 0x%p,0x%x, lhdr 0x%p.\n",
1189                 csk, csk->state, csk->flags, skb, status, csk->skb_ulp_lhdr);
1190
1191         spin_lock_bh(&csk->lock);
1192
1193         if (unlikely(csk->state >= CTP_PASSIVE_CLOSE)) {
1194                 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
1195                         "csk 0x%p,%u,0x%lx,%u, bad state.\n",
1196                         csk, csk->state, csk->flags, csk->tid);
1197                 if (csk->state != CTP_ABORTING)
1198                         goto abort_conn;
1199                 else
1200                         goto discard;
1201         }
1202
1203         if (!csk->skb_ulp_lhdr) {
1204                 pr_err("tid 0x%x, rcv RX_DATA_DDP w/o pdu bhs.\n", csk->tid);
1205                 goto abort_conn;
1206         }
1207
1208         lskb = csk->skb_ulp_lhdr;
1209         csk->skb_ulp_lhdr = NULL;
1210
1211         cxgbi_skcb_rx_ddigest(lskb) = ntohl(rpl->ulp_crc);
1212
1213         if (ntohs(rpl->len) != cxgbi_skcb_rx_pdulen(lskb))
1214                 pr_info("tid 0x%x, RX_DATA_DDP pdulen %u != %u.\n",
1215                         csk->tid, ntohs(rpl->len), cxgbi_skcb_rx_pdulen(lskb));
1216
1217         if (status & (1 << CPL_RX_DDP_STATUS_HCRC_SHIFT)) {
1218                 pr_info("csk 0x%p, lhdr 0x%p, status 0x%x, hcrc bad 0x%lx.\n",
1219                         csk, lskb, status, cxgbi_skcb_flags(lskb));
1220                 cxgbi_skcb_set_flag(lskb, SKCBF_RX_HCRC_ERR);
1221         }
1222         if (status & (1 << CPL_RX_DDP_STATUS_DCRC_SHIFT)) {
1223                 pr_info("csk 0x%p, lhdr 0x%p, status 0x%x, dcrc bad 0x%lx.\n",
1224                         csk, lskb, status, cxgbi_skcb_flags(lskb));
1225                 cxgbi_skcb_set_flag(lskb, SKCBF_RX_DCRC_ERR);
1226         }
1227         if (status & (1 << CPL_RX_DDP_STATUS_PAD_SHIFT)) {
1228                 log_debug(1 << CXGBI_DBG_PDU_RX,
1229                         "csk 0x%p, lhdr 0x%p, status 0x%x, pad bad.\n",
1230                         csk, lskb, status);
1231                 cxgbi_skcb_set_flag(lskb, SKCBF_RX_PAD_ERR);
1232         }
1233         if ((status & (1 << CPL_RX_DDP_STATUS_DDP_SHIFT)) &&
1234                 !cxgbi_skcb_test_flag(lskb, SKCBF_RX_DATA)) {
1235                 log_debug(1 << CXGBI_DBG_PDU_RX,
1236                         "csk 0x%p, lhdr 0x%p, 0x%x, data ddp'ed.\n",
1237                         csk, lskb, status);
1238                 cxgbi_skcb_set_flag(lskb, SKCBF_RX_DATA_DDPD);
1239         }
1240         log_debug(1 << CXGBI_DBG_PDU_RX,
1241                 "csk 0x%p, lskb 0x%p, f 0x%lx.\n",
1242                 csk, lskb, cxgbi_skcb_flags(lskb));
1243
1244         cxgbi_skcb_set_flag(lskb, SKCBF_RX_STATUS);
1245         cxgbi_conn_pdu_ready(csk);
1246         spin_unlock_bh(&csk->lock);
1247         goto rel_skb;
1248
1249 abort_conn:
1250         send_abort_req(csk);
1251 discard:
1252         spin_unlock_bh(&csk->lock);
1253 rel_skb:
1254         __kfree_skb(skb);
1255 }
1256
1257 static void do_fw4_ack(struct cxgbi_device *cdev, struct sk_buff *skb)
1258 {
1259         struct cxgbi_sock *csk;
1260         struct cpl_fw4_ack *rpl = (struct cpl_fw4_ack *)skb->data;
1261         unsigned int tid = GET_TID(rpl);
1262         struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
1263         struct tid_info *t = lldi->tids;
1264
1265         csk = lookup_tid(t, tid);
1266         if (unlikely(!csk))
1267                 pr_err("can't find connection for tid %u.\n", tid);
1268         else {
1269                 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
1270                         "csk 0x%p,%u,0x%lx,%u.\n",
1271                         csk, csk->state, csk->flags, csk->tid);
1272                 cxgbi_sock_rcv_wr_ack(csk, rpl->credits, ntohl(rpl->snd_una),
1273                                         rpl->seq_vld);
1274         }
1275         __kfree_skb(skb);
1276 }
1277
1278 static void do_set_tcb_rpl(struct cxgbi_device *cdev, struct sk_buff *skb)
1279 {
1280         struct cpl_set_tcb_rpl *rpl = (struct cpl_set_tcb_rpl *)skb->data;
1281         unsigned int tid = GET_TID(rpl);
1282         struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
1283         struct tid_info *t = lldi->tids;
1284         struct cxgbi_sock *csk;
1285
1286         csk = lookup_tid(t, tid);
1287         if (!csk)
1288                 pr_err("can't find conn. for tid %u.\n", tid);
1289
1290         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
1291                 "csk 0x%p,%u,%lx,%u, status 0x%x.\n",
1292                 csk, csk->state, csk->flags, csk->tid, rpl->status);
1293
1294         if (rpl->status != CPL_ERR_NONE)
1295                 pr_err("csk 0x%p,%u, SET_TCB_RPL status %u.\n",
1296                         csk, tid, rpl->status);
1297
1298         __kfree_skb(skb);
1299 }
1300
1301 static int alloc_cpls(struct cxgbi_sock *csk)
1302 {
1303         csk->cpl_close = alloc_wr(sizeof(struct cpl_close_con_req),
1304                                         0, GFP_KERNEL);
1305         if (!csk->cpl_close)
1306                 return -ENOMEM;
1307
1308         csk->cpl_abort_req = alloc_wr(sizeof(struct cpl_abort_req),
1309                                         0, GFP_KERNEL);
1310         if (!csk->cpl_abort_req)
1311                 goto free_cpls;
1312
1313         csk->cpl_abort_rpl = alloc_wr(sizeof(struct cpl_abort_rpl),
1314                                         0, GFP_KERNEL);
1315         if (!csk->cpl_abort_rpl)
1316                 goto free_cpls;
1317         return 0;
1318
1319 free_cpls:
1320         cxgbi_sock_free_cpl_skbs(csk);
1321         return -ENOMEM;
1322 }
1323
1324 static inline void l2t_put(struct cxgbi_sock *csk)
1325 {
1326         if (csk->l2t) {
1327                 cxgb4_l2t_release(csk->l2t);
1328                 csk->l2t = NULL;
1329                 cxgbi_sock_put(csk);
1330         }
1331 }
1332
1333 static void release_offload_resources(struct cxgbi_sock *csk)
1334 {
1335         struct cxgb4_lld_info *lldi;
1336 #if IS_ENABLED(CONFIG_IPV6)
1337         struct net_device *ndev = csk->cdev->ports[csk->port_id];
1338 #endif
1339
1340         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
1341                 "csk 0x%p,%u,0x%lx,%u.\n",
1342                 csk, csk->state, csk->flags, csk->tid);
1343
1344         cxgbi_sock_free_cpl_skbs(csk);
1345         if (csk->wr_cred != csk->wr_max_cred) {
1346                 cxgbi_sock_purge_wr_queue(csk);
1347                 cxgbi_sock_reset_wr_list(csk);
1348         }
1349
1350         l2t_put(csk);
1351 #if IS_ENABLED(CONFIG_IPV6)
1352         if (csk->csk_family == AF_INET6)
1353                 cxgb4_clip_release(ndev,
1354                                    (const u32 *)&csk->saddr6.sin6_addr, 1);
1355 #endif
1356
1357         if (cxgbi_sock_flag(csk, CTPF_HAS_ATID))
1358                 free_atid(csk);
1359         else if (cxgbi_sock_flag(csk, CTPF_HAS_TID)) {
1360                 lldi = cxgbi_cdev_priv(csk->cdev);
1361                 cxgb4_remove_tid(lldi->tids, 0, csk->tid);
1362                 cxgbi_sock_clear_flag(csk, CTPF_HAS_TID);
1363                 cxgbi_sock_put(csk);
1364         }
1365         csk->dst = NULL;
1366         csk->cdev = NULL;
1367 }
1368
1369 static int init_act_open(struct cxgbi_sock *csk)
1370 {
1371         struct cxgbi_device *cdev = csk->cdev;
1372         struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
1373         struct net_device *ndev = cdev->ports[csk->port_id];
1374         struct sk_buff *skb = NULL;
1375         struct neighbour *n = NULL;
1376         void *daddr;
1377         unsigned int step;
1378         unsigned int size, size6;
1379         int t4 = is_t4(lldi->adapter_type);
1380         unsigned int linkspeed;
1381         unsigned int rcv_winf, snd_winf;
1382
1383         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
1384                 "csk 0x%p,%u,0x%lx,%u.\n",
1385                 csk, csk->state, csk->flags, csk->tid);
1386
1387         if (csk->csk_family == AF_INET)
1388                 daddr = &csk->daddr.sin_addr.s_addr;
1389 #if IS_ENABLED(CONFIG_IPV6)
1390         else if (csk->csk_family == AF_INET6)
1391                 daddr = &csk->daddr6.sin6_addr;
1392 #endif
1393         else {
1394                 pr_err("address family 0x%x not supported\n", csk->csk_family);
1395                 goto rel_resource;
1396         }
1397
1398         n = dst_neigh_lookup(csk->dst, daddr);
1399
1400         if (!n) {
1401                 pr_err("%s, can't get neighbour of csk->dst.\n", ndev->name);
1402                 goto rel_resource;
1403         }
1404
1405         csk->atid = cxgb4_alloc_atid(lldi->tids, csk);
1406         if (csk->atid < 0) {
1407                 pr_err("%s, NO atid available.\n", ndev->name);
1408                 return -EINVAL;
1409         }
1410         cxgbi_sock_set_flag(csk, CTPF_HAS_ATID);
1411         cxgbi_sock_get(csk);
1412
1413         csk->l2t = cxgb4_l2t_get(lldi->l2t, n, ndev, 0);
1414         if (!csk->l2t) {
1415                 pr_err("%s, cannot alloc l2t.\n", ndev->name);
1416                 goto rel_resource_without_clip;
1417         }
1418         cxgbi_sock_get(csk);
1419
1420 #if IS_ENABLED(CONFIG_IPV6)
1421         if (csk->csk_family == AF_INET6)
1422                 cxgb4_clip_get(ndev, (const u32 *)&csk->saddr6.sin6_addr, 1);
1423 #endif
1424
1425         if (t4) {
1426                 size = sizeof(struct cpl_act_open_req);
1427                 size6 = sizeof(struct cpl_act_open_req6);
1428         } else {
1429                 size = sizeof(struct cpl_t5_act_open_req);
1430                 size6 = sizeof(struct cpl_t5_act_open_req6);
1431         }
1432
1433         if (csk->csk_family == AF_INET)
1434                 skb = alloc_wr(size, 0, GFP_NOIO);
1435 #if IS_ENABLED(CONFIG_IPV6)
1436         else
1437                 skb = alloc_wr(size6, 0, GFP_NOIO);
1438 #endif
1439
1440         if (!skb)
1441                 goto rel_resource;
1442         skb->sk = (struct sock *)csk;
1443         t4_set_arp_err_handler(skb, csk, cxgbi_sock_act_open_req_arp_failure);
1444
1445         if (!csk->mtu)
1446                 csk->mtu = dst_mtu(csk->dst);
1447         cxgb4_best_mtu(lldi->mtus, csk->mtu, &csk->mss_idx);
1448         csk->tx_chan = cxgb4_port_chan(ndev);
1449         /* SMT two entries per row */
1450         csk->smac_idx = ((cxgb4_port_viid(ndev) & 0x7F)) << 1;
1451         step = lldi->ntxq / lldi->nchan;
1452         csk->txq_idx = cxgb4_port_idx(ndev) * step;
1453         step = lldi->nrxq / lldi->nchan;
1454         csk->rss_qid = lldi->rxq_ids[cxgb4_port_idx(ndev) * step];
1455         linkspeed = ((struct port_info *)netdev_priv(ndev))->link_cfg.speed;
1456         csk->snd_win = cxgb4i_snd_win;
1457         csk->rcv_win = cxgb4i_rcv_win;
1458         if (cxgb4i_rcv_win <= 0) {
1459                 csk->rcv_win = CXGB4I_DEFAULT_10G_RCV_WIN;
1460                 rcv_winf = linkspeed / SPEED_10000;
1461                 if (rcv_winf)
1462                         csk->rcv_win *= rcv_winf;
1463         }
1464         if (cxgb4i_snd_win <= 0) {
1465                 csk->snd_win = CXGB4I_DEFAULT_10G_SND_WIN;
1466                 snd_winf = linkspeed / SPEED_10000;
1467                 if (snd_winf)
1468                         csk->snd_win *= snd_winf;
1469         }
1470         csk->wr_cred = lldi->wr_cred -
1471                        DIV_ROUND_UP(sizeof(struct cpl_abort_req), 16);
1472         csk->wr_max_cred = csk->wr_cred;
1473         csk->wr_una_cred = 0;
1474         cxgbi_sock_reset_wr_list(csk);
1475         csk->err = 0;
1476
1477         pr_info_ipaddr("csk 0x%p,%u,0x%lx,%u,%u,%u, mtu %u,%u, smac %u.\n",
1478                        (&csk->saddr), (&csk->daddr), csk, csk->state,
1479                        csk->flags, csk->tx_chan, csk->txq_idx, csk->rss_qid,
1480                        csk->mtu, csk->mss_idx, csk->smac_idx);
1481
1482         /* must wait for either a act_open_rpl or act_open_establish */
1483         try_module_get(THIS_MODULE);
1484         cxgbi_sock_set_state(csk, CTP_ACTIVE_OPEN);
1485         if (csk->csk_family == AF_INET)
1486                 send_act_open_req(csk, skb, csk->l2t);
1487 #if IS_ENABLED(CONFIG_IPV6)
1488         else
1489                 send_act_open_req6(csk, skb, csk->l2t);
1490 #endif
1491         neigh_release(n);
1492
1493         return 0;
1494
1495 rel_resource:
1496 #if IS_ENABLED(CONFIG_IPV6)
1497         if (csk->csk_family == AF_INET6)
1498                 cxgb4_clip_release(ndev,
1499                                    (const u32 *)&csk->saddr6.sin6_addr, 1);
1500 #endif
1501 rel_resource_without_clip:
1502         if (n)
1503                 neigh_release(n);
1504         if (skb)
1505                 __kfree_skb(skb);
1506         return -EINVAL;
1507 }
1508
1509 static cxgb4i_cplhandler_func cxgb4i_cplhandlers[NUM_CPL_CMDS] = {
1510         [CPL_ACT_ESTABLISH] = do_act_establish,
1511         [CPL_ACT_OPEN_RPL] = do_act_open_rpl,
1512         [CPL_PEER_CLOSE] = do_peer_close,
1513         [CPL_ABORT_REQ_RSS] = do_abort_req_rss,
1514         [CPL_ABORT_RPL_RSS] = do_abort_rpl_rss,
1515         [CPL_CLOSE_CON_RPL] = do_close_con_rpl,
1516         [CPL_FW4_ACK] = do_fw4_ack,
1517         [CPL_ISCSI_HDR] = do_rx_iscsi_hdr,
1518         [CPL_ISCSI_DATA] = do_rx_iscsi_hdr,
1519         [CPL_SET_TCB_RPL] = do_set_tcb_rpl,
1520         [CPL_RX_DATA_DDP] = do_rx_data_ddp,
1521         [CPL_RX_ISCSI_DDP] = do_rx_data_ddp,
1522         [CPL_RX_DATA] = do_rx_data,
1523 };
1524
1525 static int cxgb4i_ofld_init(struct cxgbi_device *cdev)
1526 {
1527         int rc;
1528
1529         if (cxgb4i_max_connect > CXGB4I_MAX_CONN)
1530                 cxgb4i_max_connect = CXGB4I_MAX_CONN;
1531
1532         rc = cxgbi_device_portmap_create(cdev, cxgb4i_sport_base,
1533                                         cxgb4i_max_connect);
1534         if (rc < 0)
1535                 return rc;
1536
1537         cdev->csk_release_offload_resources = release_offload_resources;
1538         cdev->csk_push_tx_frames = push_tx_frames;
1539         cdev->csk_send_abort_req = send_abort_req;
1540         cdev->csk_send_close_req = send_close_req;
1541         cdev->csk_send_rx_credits = send_rx_credits;
1542         cdev->csk_alloc_cpls = alloc_cpls;
1543         cdev->csk_init_act_open = init_act_open;
1544
1545         pr_info("cdev 0x%p, offload up, added.\n", cdev);
1546         return 0;
1547 }
1548
1549 static inline void
1550 ulp_mem_io_set_hdr(struct cxgbi_device *cdev,
1551                    struct ulp_mem_io *req,
1552                    unsigned int wr_len, unsigned int dlen,
1553                    unsigned int pm_addr,
1554                    int tid)
1555 {
1556         struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
1557         struct ulptx_idata *idata = (struct ulptx_idata *)(req + 1);
1558
1559         INIT_ULPTX_WR(req, wr_len, 0, tid);
1560         req->wr.wr_hi = htonl(FW_WR_OP_V(FW_ULPTX_WR) |
1561                 FW_WR_ATOMIC_V(0));
1562         req->cmd = htonl(ULPTX_CMD_V(ULP_TX_MEM_WRITE) |
1563                 ULP_MEMIO_ORDER_V(is_t4(lldi->adapter_type)) |
1564                 T5_ULP_MEMIO_IMM_V(!is_t4(lldi->adapter_type)));
1565         req->dlen = htonl(ULP_MEMIO_DATA_LEN_V(dlen >> 5));
1566         req->lock_addr = htonl(ULP_MEMIO_ADDR_V(pm_addr >> 5));
1567         req->len16 = htonl(DIV_ROUND_UP(wr_len - sizeof(req->wr), 16));
1568
1569         idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM));
1570         idata->len = htonl(dlen);
1571 }
1572
1573 static struct sk_buff *
1574 ddp_ppod_init_idata(struct cxgbi_device *cdev,
1575                     struct cxgbi_ppm *ppm,
1576                     unsigned int idx, unsigned int npods,
1577                     unsigned int tid)
1578 {
1579         unsigned int pm_addr = (idx << PPOD_SIZE_SHIFT) + ppm->llimit;
1580         unsigned int dlen = npods << PPOD_SIZE_SHIFT;
1581         unsigned int wr_len = roundup(sizeof(struct ulp_mem_io) +
1582                                 sizeof(struct ulptx_idata) + dlen, 16);
1583         struct sk_buff *skb = alloc_wr(wr_len, 0, GFP_ATOMIC);
1584
1585         if (!skb) {
1586                 pr_err("%s: %s idx %u, npods %u, OOM.\n",
1587                        __func__, ppm->ndev->name, idx, npods);
1588                 return NULL;
1589         }
1590
1591         ulp_mem_io_set_hdr(cdev, (struct ulp_mem_io *)skb->head, wr_len, dlen,
1592                            pm_addr, tid);
1593
1594         return skb;
1595 }
1596
1597 static int ddp_ppod_write_idata(struct cxgbi_ppm *ppm, struct cxgbi_sock *csk,
1598                                 struct cxgbi_task_tag_info *ttinfo,
1599                                 unsigned int idx, unsigned int npods,
1600                                 struct scatterlist **sg_pp,
1601                                 unsigned int *sg_off)
1602 {
1603         struct cxgbi_device *cdev = csk->cdev;
1604         struct sk_buff *skb = ddp_ppod_init_idata(cdev, ppm, idx, npods,
1605                                                   csk->tid);
1606         struct ulp_mem_io *req;
1607         struct ulptx_idata *idata;
1608         struct cxgbi_pagepod *ppod;
1609         int i;
1610
1611         if (!skb)
1612                 return -ENOMEM;
1613
1614         req = (struct ulp_mem_io *)skb->head;
1615         idata = (struct ulptx_idata *)(req + 1);
1616         ppod = (struct cxgbi_pagepod *)(idata + 1);
1617
1618         for (i = 0; i < npods; i++, ppod++)
1619                 cxgbi_ddp_set_one_ppod(ppod, ttinfo, sg_pp, sg_off);
1620
1621         cxgbi_skcb_set_flag(skb, SKCBF_TX_MEM_WRITE);
1622         cxgbi_skcb_set_flag(skb, SKCBF_TX_FLAG_COMPL);
1623         set_wr_txq(skb, CPL_PRIORITY_DATA, csk->port_id);
1624
1625         spin_lock_bh(&csk->lock);
1626         cxgbi_sock_skb_entail(csk, skb);
1627         spin_unlock_bh(&csk->lock);
1628
1629         return 0;
1630 }
1631
1632 static int ddp_set_map(struct cxgbi_ppm *ppm, struct cxgbi_sock *csk,
1633                        struct cxgbi_task_tag_info *ttinfo)
1634 {
1635         unsigned int pidx = ttinfo->idx;
1636         unsigned int npods = ttinfo->npods;
1637         unsigned int i, cnt;
1638         int err = 0;
1639         struct scatterlist *sg = ttinfo->sgl;
1640         unsigned int offset = 0;
1641
1642         ttinfo->cid = csk->port_id;
1643
1644         for (i = 0; i < npods; i += cnt, pidx += cnt) {
1645                 cnt = npods - i;
1646
1647                 if (cnt > ULPMEM_IDATA_MAX_NPPODS)
1648                         cnt = ULPMEM_IDATA_MAX_NPPODS;
1649                 err = ddp_ppod_write_idata(ppm, csk, ttinfo, pidx, cnt,
1650                                            &sg, &offset);
1651                 if (err < 0)
1652                         break;
1653         }
1654
1655         return err;
1656 }
1657
1658 static int ddp_setup_conn_pgidx(struct cxgbi_sock *csk, unsigned int tid,
1659                                 int pg_idx, bool reply)
1660 {
1661         struct sk_buff *skb;
1662         struct cpl_set_tcb_field *req;
1663
1664         if (!pg_idx || pg_idx >= DDP_PGIDX_MAX)
1665                 return 0;
1666
1667         skb = alloc_wr(sizeof(*req), 0, GFP_KERNEL);
1668         if (!skb)
1669                 return -ENOMEM;
1670
1671         /*  set up ulp page size */
1672         req = (struct cpl_set_tcb_field *)skb->head;
1673         INIT_TP_WR(req, csk->tid);
1674         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, csk->tid));
1675         req->reply_ctrl = htons(NO_REPLY_V(reply) | QUEUENO_V(csk->rss_qid));
1676         req->word_cookie = htons(0);
1677         req->mask = cpu_to_be64(0x3 << 8);
1678         req->val = cpu_to_be64(pg_idx << 8);
1679         set_wr_txq(skb, CPL_PRIORITY_CONTROL, csk->port_id);
1680
1681         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
1682                 "csk 0x%p, tid 0x%x, pg_idx %u.\n", csk, csk->tid, pg_idx);
1683
1684         cxgb4_ofld_send(csk->cdev->ports[csk->port_id], skb);
1685         return 0;
1686 }
1687
1688 static int ddp_setup_conn_digest(struct cxgbi_sock *csk, unsigned int tid,
1689                                  int hcrc, int dcrc, int reply)
1690 {
1691         struct sk_buff *skb;
1692         struct cpl_set_tcb_field *req;
1693
1694         if (!hcrc && !dcrc)
1695                 return 0;
1696
1697         skb = alloc_wr(sizeof(*req), 0, GFP_KERNEL);
1698         if (!skb)
1699                 return -ENOMEM;
1700
1701         csk->hcrc_len = (hcrc ? 4 : 0);
1702         csk->dcrc_len = (dcrc ? 4 : 0);
1703         /*  set up ulp submode */
1704         req = (struct cpl_set_tcb_field *)skb->head;
1705         INIT_TP_WR(req, tid);
1706         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid));
1707         req->reply_ctrl = htons(NO_REPLY_V(reply) | QUEUENO_V(csk->rss_qid));
1708         req->word_cookie = htons(0);
1709         req->mask = cpu_to_be64(0x3 << 4);
1710         req->val = cpu_to_be64(((hcrc ? ULP_CRC_HEADER : 0) |
1711                                 (dcrc ? ULP_CRC_DATA : 0)) << 4);
1712         set_wr_txq(skb, CPL_PRIORITY_CONTROL, csk->port_id);
1713
1714         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
1715                 "csk 0x%p, tid 0x%x, crc %d,%d.\n", csk, csk->tid, hcrc, dcrc);
1716
1717         cxgb4_ofld_send(csk->cdev->ports[csk->port_id], skb);
1718         return 0;
1719 }
1720
1721 static struct cxgbi_ppm *cdev2ppm(struct cxgbi_device *cdev)
1722 {
1723         return (struct cxgbi_ppm *)(*((struct cxgb4_lld_info *)
1724                                        (cxgbi_cdev_priv(cdev)))->iscsi_ppm);
1725 }
1726
1727 static int cxgb4i_ddp_init(struct cxgbi_device *cdev)
1728 {
1729         struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
1730         struct net_device *ndev = cdev->ports[0];
1731         struct cxgbi_tag_format tformat;
1732         unsigned int ppmax;
1733         int i;
1734
1735         if (!lldi->vr->iscsi.size) {
1736                 pr_warn("%s, iscsi NOT enabled, check config!\n", ndev->name);
1737                 return -EACCES;
1738         }
1739
1740         cdev->flags |= CXGBI_FLAG_USE_PPOD_OFLDQ;
1741         ppmax = lldi->vr->iscsi.size >> PPOD_SIZE_SHIFT;
1742
1743         memset(&tformat, 0, sizeof(struct cxgbi_tag_format));
1744         for (i = 0; i < 4; i++)
1745                 tformat.pgsz_order[i] = (lldi->iscsi_pgsz_order >> (i << 3))
1746                                          & 0xF;
1747         cxgbi_tagmask_check(lldi->iscsi_tagmask, &tformat);
1748
1749         cxgbi_ddp_ppm_setup(lldi->iscsi_ppm, cdev, &tformat, ppmax,
1750                             lldi->iscsi_llimit, lldi->vr->iscsi.start, 2);
1751
1752         cdev->csk_ddp_setup_digest = ddp_setup_conn_digest;
1753         cdev->csk_ddp_setup_pgidx = ddp_setup_conn_pgidx;
1754         cdev->csk_ddp_set_map = ddp_set_map;
1755         cdev->tx_max_size = min_t(unsigned int, ULP2_MAX_PDU_PAYLOAD,
1756                                   lldi->iscsi_iolen - ISCSI_PDU_NONPAYLOAD_LEN);
1757         cdev->rx_max_size = min_t(unsigned int, ULP2_MAX_PDU_PAYLOAD,
1758                                   lldi->iscsi_iolen - ISCSI_PDU_NONPAYLOAD_LEN);
1759         cdev->cdev2ppm = cdev2ppm;
1760
1761         return 0;
1762 }
1763
1764 static void *t4_uld_add(const struct cxgb4_lld_info *lldi)
1765 {
1766         struct cxgbi_device *cdev;
1767         struct port_info *pi;
1768         int i, rc;
1769
1770         cdev = cxgbi_device_register(sizeof(*lldi), lldi->nports);
1771         if (!cdev) {
1772                 pr_info("t4 device 0x%p, register failed.\n", lldi);
1773                 return NULL;
1774         }
1775         pr_info("0x%p,0x%x, ports %u,%s, chan %u, q %u,%u, wr %u.\n",
1776                 cdev, lldi->adapter_type, lldi->nports,
1777                 lldi->ports[0]->name, lldi->nchan, lldi->ntxq,
1778                 lldi->nrxq, lldi->wr_cred);
1779         for (i = 0; i < lldi->nrxq; i++)
1780                 log_debug(1 << CXGBI_DBG_DEV,
1781                         "t4 0x%p, rxq id #%d: %u.\n",
1782                         cdev, i, lldi->rxq_ids[i]);
1783
1784         memcpy(cxgbi_cdev_priv(cdev), lldi, sizeof(*lldi));
1785         cdev->flags = CXGBI_FLAG_DEV_T4;
1786         cdev->pdev = lldi->pdev;
1787         cdev->ports = lldi->ports;
1788         cdev->nports = lldi->nports;
1789         cdev->mtus = lldi->mtus;
1790         cdev->nmtus = NMTUS;
1791         cdev->rx_credit_thres = cxgb4i_rx_credit_thres;
1792         cdev->skb_tx_rsvd = CXGB4I_TX_HEADER_LEN;
1793         cdev->skb_rx_extra = sizeof(struct cpl_iscsi_hdr);
1794         cdev->itp = &cxgb4i_iscsi_transport;
1795
1796         cdev->pfvf = FW_VIID_PFN_G(cxgb4_port_viid(lldi->ports[0]))
1797                         << FW_VIID_PFN_S;
1798         pr_info("cdev 0x%p,%s, pfvf %u.\n",
1799                 cdev, lldi->ports[0]->name, cdev->pfvf);
1800
1801         rc = cxgb4i_ddp_init(cdev);
1802         if (rc) {
1803                 pr_info("t4 0x%p ddp init failed.\n", cdev);
1804                 goto err_out;
1805         }
1806         rc = cxgb4i_ofld_init(cdev);
1807         if (rc) {
1808                 pr_info("t4 0x%p ofld init failed.\n", cdev);
1809                 goto err_out;
1810         }
1811
1812         rc = cxgbi_hbas_add(cdev, CXGB4I_MAX_LUN, CXGBI_MAX_CONN,
1813                                 &cxgb4i_host_template, cxgb4i_stt);
1814         if (rc)
1815                 goto err_out;
1816
1817         for (i = 0; i < cdev->nports; i++) {
1818                 pi = netdev_priv(lldi->ports[i]);
1819                 cdev->hbas[i]->port_id = pi->port_id;
1820         }
1821         return cdev;
1822
1823 err_out:
1824         cxgbi_device_unregister(cdev);
1825         return ERR_PTR(-ENOMEM);
1826 }
1827
1828 #define RX_PULL_LEN     128
1829 static int t4_uld_rx_handler(void *handle, const __be64 *rsp,
1830                                 const struct pkt_gl *pgl)
1831 {
1832         const struct cpl_act_establish *rpl;
1833         struct sk_buff *skb;
1834         unsigned int opc;
1835         struct cxgbi_device *cdev = handle;
1836
1837         if (pgl == NULL) {
1838                 unsigned int len = 64 - sizeof(struct rsp_ctrl) - 8;
1839
1840                 skb = alloc_wr(len, 0, GFP_ATOMIC);
1841                 if (!skb)
1842                         goto nomem;
1843                 skb_copy_to_linear_data(skb, &rsp[1], len);
1844         } else {
1845                 if (unlikely(*(u8 *)rsp != *(u8 *)pgl->va)) {
1846                         pr_info("? FL 0x%p,RSS%#llx,FL %#llx,len %u.\n",
1847                                 pgl->va, be64_to_cpu(*rsp),
1848                                 be64_to_cpu(*(u64 *)pgl->va),
1849                                 pgl->tot_len);
1850                         return 0;
1851                 }
1852                 skb = cxgb4_pktgl_to_skb(pgl, RX_PULL_LEN, RX_PULL_LEN);
1853                 if (unlikely(!skb))
1854                         goto nomem;
1855         }
1856
1857         rpl = (struct cpl_act_establish *)skb->data;
1858         opc = rpl->ot.opcode;
1859         log_debug(1 << CXGBI_DBG_TOE,
1860                 "cdev %p, opcode 0x%x(0x%x,0x%x), skb %p.\n",
1861                  cdev, opc, rpl->ot.opcode_tid, ntohl(rpl->ot.opcode_tid), skb);
1862         if (cxgb4i_cplhandlers[opc])
1863                 cxgb4i_cplhandlers[opc](cdev, skb);
1864         else {
1865                 pr_err("No handler for opcode 0x%x.\n", opc);
1866                 __kfree_skb(skb);
1867         }
1868         return 0;
1869 nomem:
1870         log_debug(1 << CXGBI_DBG_TOE, "OOM bailing out.\n");
1871         return 1;
1872 }
1873
1874 static int t4_uld_state_change(void *handle, enum cxgb4_state state)
1875 {
1876         struct cxgbi_device *cdev = handle;
1877
1878         switch (state) {
1879         case CXGB4_STATE_UP:
1880                 pr_info("cdev 0x%p, UP.\n", cdev);
1881                 break;
1882         case CXGB4_STATE_START_RECOVERY:
1883                 pr_info("cdev 0x%p, RECOVERY.\n", cdev);
1884                 /* close all connections */
1885                 break;
1886         case CXGB4_STATE_DOWN:
1887                 pr_info("cdev 0x%p, DOWN.\n", cdev);
1888                 break;
1889         case CXGB4_STATE_DETACH:
1890                 pr_info("cdev 0x%p, DETACH.\n", cdev);
1891                 cxgbi_device_unregister(cdev);
1892                 break;
1893         default:
1894                 pr_info("cdev 0x%p, unknown state %d.\n", cdev, state);
1895                 break;
1896         }
1897         return 0;
1898 }
1899
1900 static int __init cxgb4i_init_module(void)
1901 {
1902         int rc;
1903
1904         printk(KERN_INFO "%s", version);
1905
1906         rc = cxgbi_iscsi_init(&cxgb4i_iscsi_transport, &cxgb4i_stt);
1907         if (rc < 0)
1908                 return rc;
1909         cxgb4_register_uld(CXGB4_ULD_ISCSI, &cxgb4i_uld_info);
1910
1911         return 0;
1912 }
1913
1914 static void __exit cxgb4i_exit_module(void)
1915 {
1916         cxgb4_unregister_uld(CXGB4_ULD_ISCSI);
1917         cxgbi_device_unregister_all(CXGBI_FLAG_DEV_T4);
1918         cxgbi_iscsi_cleanup(&cxgb4i_iscsi_transport, &cxgb4i_stt);
1919 }
1920
1921 module_init(cxgb4i_init_module);
1922 module_exit(cxgb4i_exit_module);