drivers/infiniband/hw/hfi1/driver.c

   1 /*
   2  * Copyright(c) 2015, 2016 Intel Corporation.
   3  *
   4  * This file is provided under a dual BSD/GPLv2 license.  When using or
   5  * redistributing this file, you may do so under either license.
   6  *
   7  * GPL LICENSE SUMMARY
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of version 2 of the GNU General Public License as
  11  * published by the Free Software Foundation.
  12  *
  13  * This program is distributed in the hope that it will be useful, but
  14  * WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  16  * General Public License for more details.
  17  *
  18  * BSD LICENSE
  19  *
  20  * Redistribution and use in source and binary forms, with or without
  21  * modification, are permitted provided that the following conditions
  22  * are met:
  23  *
  24  *  - Redistributions of source code must retain the above copyright
  25  *    notice, this list of conditions and the following disclaimer.
  26  *  - Redistributions in binary form must reproduce the above copyright
  27  *    notice, this list of conditions and the following disclaimer in
  28  *    the documentation and/or other materials provided with the
  29  *    distribution.
  30  *  - Neither the name of Intel Corporation nor the names of its
  31  *    contributors may be used to endorse or promote products derived
  32  *    from this software without specific prior written permission.
  33  *
  34  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  35  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  36  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  37  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  38  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  39  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  40  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  41  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  42  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  43  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  44  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  45  *
  46  */
  47
  48 #include <linux/spinlock.h>
  49 #include <linux/pci.h>
  50 #include <linux/io.h>
  51 #include <linux/delay.h>
  52 #include <linux/netdevice.h>
  53 #include <linux/vmalloc.h>
  54 #include <linux/module.h>
  55 #include <linux/prefetch.h>
  56 #include <rdma/ib_verbs.h>
  57
  58 #include "hfi.h"
  59 #include "trace.h"
  60 #include "qp.h"
  61 #include "sdma.h"
  62
  63 #undef pr_fmt
  64 #define pr_fmt(fmt) DRIVER_NAME ": " fmt
  65
  66 /*
  67  * The size has to be longer than this string, so we can append
  68  * board/chip information to it in the initialization code.
  69  */
  70 const char ib_hfi1_version[] = HFI1_DRIVER_VERSION "\n";
  71
  72 DEFINE_SPINLOCK(hfi1_devs_lock);
  73 LIST_HEAD(hfi1_dev_list);
  74 DEFINE_MUTEX(hfi1_mutex);       /* general driver use */
  75
  76 unsigned int hfi1_max_mtu = HFI1_DEFAULT_MAX_MTU;
  77 module_param_named(max_mtu, hfi1_max_mtu, uint, S_IRUGO);
  78 MODULE_PARM_DESC(max_mtu, "Set max MTU bytes, default is " __stringify(
  79                  HFI1_DEFAULT_MAX_MTU));
  80
  81 unsigned int hfi1_cu = 1;
  82 module_param_named(cu, hfi1_cu, uint, S_IRUGO);
  83 MODULE_PARM_DESC(cu, "Credit return units");
  84
  85 unsigned long hfi1_cap_mask = HFI1_CAP_MASK_DEFAULT;
  86 static int hfi1_caps_set(const char *, const struct kernel_param *);
  87 static int hfi1_caps_get(char *, const struct kernel_param *);
  88 static const struct kernel_param_ops cap_ops = {
  89         .set = hfi1_caps_set,
  90         .get = hfi1_caps_get
  91 };
  92 module_param_cb(cap_mask, &cap_ops, &hfi1_cap_mask, S_IWUSR | S_IRUGO);
  93 MODULE_PARM_DESC(cap_mask, "Bit mask of enabled/disabled HW features");
  94
  95 MODULE_LICENSE("Dual BSD/GPL");
  96 MODULE_DESCRIPTION("Intel Omni-Path Architecture driver");
  97 MODULE_VERSION(HFI1_DRIVER_VERSION);
  98
  99 /*
 100  * MAX_PKT_RCV is the max # if packets processed per receive interrupt.
 101  */
 102 #define MAX_PKT_RECV 64
 103 #define EGR_HEAD_UPDATE_THRESHOLD 16
 104
 105 struct hfi1_ib_stats hfi1_stats;
 106
 107 static int hfi1_caps_set(const char *val, const struct kernel_param *kp)
 108 {
 109         int ret = 0;
 110         unsigned long *cap_mask_ptr = (unsigned long *)kp->arg,
 111                 cap_mask = *cap_mask_ptr, value, diff,
 112                 write_mask = ((HFI1_CAP_WRITABLE_MASK << HFI1_CAP_USER_SHIFT) |
 113                               HFI1_CAP_WRITABLE_MASK);
 114
 115         ret = kstrtoul(val, 0, &value);
 116         if (ret) {
 117                 pr_warn("Invalid module parameter value for 'cap_mask'\n");
 118                 goto done;
 119         }
 120         /* Get the changed bits (except the locked bit) */
 121         diff = value ^ (cap_mask & ~HFI1_CAP_LOCKED_SMASK);
 122
 123         /* Remove any bits that are not allowed to change after driver load */
 124         if (HFI1_CAP_LOCKED() && (diff & ~write_mask)) {
 125                 pr_warn("Ignoring non-writable capability bits %#lx\n",
 126                         diff & ~write_mask);
 127                 diff &= write_mask;
 128         }
 129
 130         /* Mask off any reserved bits */
 131         diff &= ~HFI1_CAP_RESERVED_MASK;
 132         /* Clear any previously set and changing bits */
 133         cap_mask &= ~diff;
 134         /* Update the bits with the new capability */
 135         cap_mask |= (value & diff);
 136         /* Check for any kernel/user restrictions */
 137         diff = (cap_mask & (HFI1_CAP_MUST_HAVE_KERN << HFI1_CAP_USER_SHIFT)) ^
 138                 ((cap_mask & HFI1_CAP_MUST_HAVE_KERN) << HFI1_CAP_USER_SHIFT);
 139         cap_mask &= ~diff;
 140         /* Set the bitmask to the final set */
 141         *cap_mask_ptr = cap_mask;
 142 done:
 143         return ret;
 144 }
 145
 146 static int hfi1_caps_get(char *buffer, const struct kernel_param *kp)
 147 {
 148         unsigned long cap_mask = *(unsigned long *)kp->arg;
 149
 150         cap_mask &= ~HFI1_CAP_LOCKED_SMASK;
 151         cap_mask |= ((cap_mask & HFI1_CAP_K2U) << HFI1_CAP_USER_SHIFT);
 152
 153         return scnprintf(buffer, PAGE_SIZE, "0x%lx", cap_mask);
 154 }
 155
 156 const char *get_unit_name(int unit)
 157 {
 158         static char iname[16];
 159
 160         snprintf(iname, sizeof(iname), DRIVER_NAME "_%u", unit);
 161         return iname;
 162 }
 163
 164 const char *get_card_name(struct rvt_dev_info *rdi)
 165 {
 166         struct hfi1_ibdev *ibdev = container_of(rdi, struct hfi1_ibdev, rdi);
 167         struct hfi1_devdata *dd = container_of(ibdev,
 168                                                struct hfi1_devdata, verbs_dev);
 169         return get_unit_name(dd->unit);
 170 }
 171
 172 struct pci_dev *get_pci_dev(struct rvt_dev_info *rdi)
 173 {
 174         struct hfi1_ibdev *ibdev = container_of(rdi, struct hfi1_ibdev, rdi);
 175         struct hfi1_devdata *dd = container_of(ibdev,
 176                                                struct hfi1_devdata, verbs_dev);
 177         return dd->pcidev;
 178 }
 179
 180 /*
 181  * Return count of units with at least one port ACTIVE.
 182  */
 183 int hfi1_count_active_units(void)
 184 {
 185         struct hfi1_devdata *dd;
 186         struct hfi1_pportdata *ppd;
 187         unsigned long flags;
 188         int pidx, nunits_active = 0;
 189
 190         spin_lock_irqsave(&hfi1_devs_lock, flags);
 191         list_for_each_entry(dd, &hfi1_dev_list, list) {
 192                 if (!(dd->flags & HFI1_PRESENT) || !dd->kregbase)
 193                         continue;
 194                 for (pidx = 0; pidx < dd->num_pports; ++pidx) {
 195                         ppd = dd->pport + pidx;
 196                         if (ppd->lid && ppd->linkup) {
 197                                 nunits_active++;
 198                                 break;
 199                         }
 200                 }
 201         }
 202         spin_unlock_irqrestore(&hfi1_devs_lock, flags);
 203         return nunits_active;
 204 }
 205
 206 /*
 207  * Return count of all units, optionally return in arguments
 208  * the number of usable (present) units, and the number of
 209  * ports that are up.
 210  */
 211 int hfi1_count_units(int *npresentp, int *nupp)
 212 {
 213         int nunits = 0, npresent = 0, nup = 0;
 214         struct hfi1_devdata *dd;
 215         unsigned long flags;
 216         int pidx;
 217         struct hfi1_pportdata *ppd;
 218
 219         spin_lock_irqsave(&hfi1_devs_lock, flags);
 220
 221         list_for_each_entry(dd, &hfi1_dev_list, list) {
 222                 nunits++;
 223                 if ((dd->flags & HFI1_PRESENT) && dd->kregbase)
 224                         npresent++;
 225                 for (pidx = 0; pidx < dd->num_pports; ++pidx) {
 226                         ppd = dd->pport + pidx;
 227                         if (ppd->lid && ppd->linkup)
 228                                 nup++;
 229                 }
 230         }
 231
 232         spin_unlock_irqrestore(&hfi1_devs_lock, flags);
 233
 234         if (npresentp)
 235                 *npresentp = npresent;
 236         if (nupp)
 237                 *nupp = nup;
 238
 239         return nunits;
 240 }
 241
 242 /*
 243  * Get address of eager buffer from it's index (allocated in chunks, not
 244  * contiguous).
 245  */
 246 static inline void *get_egrbuf(const struct hfi1_ctxtdata *rcd, u64 rhf,
 247                                u8 *update)
 248 {
 249         u32 idx = rhf_egr_index(rhf), offset = rhf_egr_buf_offset(rhf);
 250
 251         *update |= !(idx & (rcd->egrbufs.threshold - 1)) && !offset;
 252         return (void *)(((u64)(rcd->egrbufs.rcvtids[idx].addr)) +
 253                         (offset * RCV_BUF_BLOCK_SIZE));
 254 }
 255
 256 /*
 257  * Validate and encode the a given RcvArray Buffer size.
 258  * The function will check whether the given size falls within
 259  * allowed size ranges for the respective type and, optionally,
 260  * return the proper encoding.
 261  */
 262 inline int hfi1_rcvbuf_validate(u32 size, u8 type, u16 *encoded)
 263 {
 264         if (unlikely(!PAGE_ALIGNED(size)))
 265                 return 0;
 266         if (unlikely(size < MIN_EAGER_BUFFER))
 267                 return 0;
 268         if (size >
 269             (type == PT_EAGER ? MAX_EAGER_BUFFER : MAX_EXPECTED_BUFFER))
 270                 return 0;
 271         if (encoded)
 272                 *encoded = ilog2(size / PAGE_SIZE) + 1;
 273         return 1;
 274 }
 275
 276 static void rcv_hdrerr(struct hfi1_ctxtdata *rcd, struct hfi1_pportdata *ppd,
 277                        struct hfi1_packet *packet)
 278 {
 279         struct hfi1_message_header *rhdr = packet->hdr;
 280         u32 rte = rhf_rcv_type_err(packet->rhf);
 281         int lnh = be16_to_cpu(rhdr->lrh[0]) & 3;
 282         struct hfi1_ibport *ibp = &ppd->ibport_data;
 283         struct hfi1_devdata *dd = ppd->dd;
 284         struct rvt_dev_info *rdi = &dd->verbs_dev.rdi;
 285
 286         if (packet->rhf & (RHF_VCRC_ERR | RHF_ICRC_ERR))
 287                 return;
 288
 289         if (packet->rhf & RHF_TID_ERR) {
 290                 /* For TIDERR and RC QPs preemptively schedule a NAK */
 291                 struct hfi1_ib_header *hdr = (struct hfi1_ib_header *)rhdr;
 292                 struct hfi1_other_headers *ohdr = NULL;
 293                 u32 tlen = rhf_pkt_len(packet->rhf); /* in bytes */
 294                 u16 lid  = be16_to_cpu(hdr->lrh[1]);
 295                 u32 qp_num;
 296                 u32 rcv_flags = 0;
 297
 298                 /* Sanity check packet */
 299                 if (tlen < 24)
 300                         goto drop;
 301
 302                 /* Check for GRH */
 303                 if (lnh == HFI1_LRH_BTH) {
 304                         ohdr = &hdr->u.oth;
 305                 } else if (lnh == HFI1_LRH_GRH) {
 306                         u32 vtf;
 307
 308                         ohdr = &hdr->u.l.oth;
 309                         if (hdr->u.l.grh.next_hdr != IB_GRH_NEXT_HDR)
 310                                 goto drop;
 311                         vtf = be32_to_cpu(hdr->u.l.grh.version_tclass_flow);
 312                         if ((vtf >> IB_GRH_VERSION_SHIFT) != IB_GRH_VERSION)
 313                                 goto drop;
 314                         rcv_flags |= HFI1_HAS_GRH;
 315                 } else {
 316                         goto drop;
 317                 }
 318                 /* Get the destination QP number. */
 319                 qp_num = be32_to_cpu(ohdr->bth[1]) & RVT_QPN_MASK;
 320                 if (lid < be16_to_cpu(IB_MULTICAST_LID_BASE)) {
 321                         struct rvt_qp *qp;
 322                         unsigned long flags;
 323
 324                         rcu_read_lock();
 325                         qp = rvt_lookup_qpn(rdi, &ibp->rvp, qp_num);
 326                         if (!qp) {
 327                                 rcu_read_unlock();
 328                                 goto drop;
 329                         }
 330
 331                         /*
 332                          * Handle only RC QPs - for other QP types drop error
 333                          * packet.
 334                          */
 335                         spin_lock_irqsave(&qp->r_lock, flags);
 336
 337                         /* Check for valid receive state. */
 338                         if (!(ib_rvt_state_ops[qp->state] &
 339                               RVT_PROCESS_RECV_OK)) {
 340                                 ibp->rvp.n_pkt_drops++;
 341                         }
 342
 343                         switch (qp->ibqp.qp_type) {
 344                         case IB_QPT_RC:
 345                                 hfi1_rc_hdrerr(
 346                                         rcd,
 347                                         hdr,
 348                                         rcv_flags,
 349                                         qp);
 350                                 break;
 351                         default:
 352                                 /* For now don't handle any other QP types */
 353                                 break;
 354                         }
 355
 356                         spin_unlock_irqrestore(&qp->r_lock, flags);
 357                         rcu_read_unlock();
 358                 } /* Unicast QP */
 359         } /* Valid packet with TIDErr */
 360
 361         /* handle "RcvTypeErr" flags */
 362         switch (rte) {
 363         case RHF_RTE_ERROR_OP_CODE_ERR:
 364         {
 365                 u32 opcode;
 366                 void *ebuf = NULL;
 367                 __be32 *bth = NULL;
 368
 369                 if (rhf_use_egr_bfr(packet->rhf))
 370                         ebuf = packet->ebuf;
 371
 372                 if (!ebuf)
 373                         goto drop; /* this should never happen */
 374
 375                 if (lnh == HFI1_LRH_BTH)
 376                         bth = (__be32 *)ebuf;
 377                 else if (lnh == HFI1_LRH_GRH)
 378                         bth = (__be32 *)((char *)ebuf + sizeof(struct ib_grh));
 379                 else
 380                         goto drop;
 381
 382                 opcode = be32_to_cpu(bth[0]) >> 24;
 383                 opcode &= 0xff;
 384
 385                 if (opcode == IB_OPCODE_CNP) {
 386                         /*
 387                          * Only in pre-B0 h/w is the CNP_OPCODE handled
 388                          * via this code path.
 389                          */
 390                         struct rvt_qp *qp = NULL;
 391                         u32 lqpn, rqpn;
 392                         u16 rlid;
 393                         u8 svc_type, sl, sc5;
 394
 395                         sc5 = hdr2sc(rhdr, packet->rhf);
 396                         sl = ibp->sc_to_sl[sc5];
 397
 398                         lqpn = be32_to_cpu(bth[1]) & RVT_QPN_MASK;
 399                         rcu_read_lock();
 400                         qp = rvt_lookup_qpn(rdi, &ibp->rvp, lqpn);
 401                         if (!qp) {
 402                                 rcu_read_unlock();
 403                                 goto drop;
 404                         }
 405
 406                         switch (qp->ibqp.qp_type) {
 407                         case IB_QPT_UD:
 408                                 rlid = 0;
 409                                 rqpn = 0;
 410                                 svc_type = IB_CC_SVCTYPE_UD;
 411                                 break;
 412                         case IB_QPT_UC:
 413                                 rlid = be16_to_cpu(rhdr->lrh[3]);
 414                                 rqpn = qp->remote_qpn;
 415                                 svc_type = IB_CC_SVCTYPE_UC;
 416                                 break;
 417                         default:
 418                                 goto drop;
 419                         }
 420
 421                         process_becn(ppd, sl, rlid, lqpn, rqpn, svc_type);
 422                         rcu_read_unlock();
 423                 }
 424
 425                 packet->rhf &= ~RHF_RCV_TYPE_ERR_SMASK;
 426                 break;
 427         }
 428         default:
 429                 break;
 430         }
 431
 432 drop:
 433         return;
 434 }
 435
 436 static inline void init_packet(struct hfi1_ctxtdata *rcd,
 437                                struct hfi1_packet *packet)
 438 {
 439         packet->rsize = rcd->rcvhdrqentsize; /* words */
 440         packet->maxcnt = rcd->rcvhdrq_cnt * packet->rsize; /* words */
 441         packet->rcd = rcd;
 442         packet->updegr = 0;
 443         packet->etail = -1;
 444         packet->rhf_addr = get_rhf_addr(rcd);
 445         packet->rhf = rhf_to_cpu(packet->rhf_addr);
 446         packet->rhqoff = rcd->head;
 447         packet->numpkt = 0;
 448         packet->rcv_flags = 0;
 449 }
 450
 451 void hfi1_process_ecn_slowpath(struct rvt_qp *qp, struct hfi1_packet *pkt,
 452                                bool do_cnp)
 453 {
 454         struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num);
 455         struct hfi1_ib_header *hdr = pkt->hdr;
 456         struct hfi1_other_headers *ohdr = pkt->ohdr;
 457         struct ib_grh *grh = NULL;
 458         u32 rqpn = 0, bth1;
 459         u16 rlid, dlid = be16_to_cpu(hdr->lrh[1]);
 460         u8 sc, svc_type;
 461         bool is_mcast = false;
 462
 463         if (pkt->rcv_flags & HFI1_HAS_GRH)
 464                 grh = &hdr->u.l.grh;
 465
 466         switch (qp->ibqp.qp_type) {
 467         case IB_QPT_SMI:
 468         case IB_QPT_GSI:
 469         case IB_QPT_UD:
 470                 rlid = be16_to_cpu(hdr->lrh[3]);
 471                 rqpn = be32_to_cpu(ohdr->u.ud.deth[1]) & RVT_QPN_MASK;
 472                 svc_type = IB_CC_SVCTYPE_UD;
 473                 is_mcast = (dlid > be16_to_cpu(IB_MULTICAST_LID_BASE)) &&
 474                         (dlid != be16_to_cpu(IB_LID_PERMISSIVE));
 475                 break;
 476         case IB_QPT_UC:
 477                 rlid = qp->remote_ah_attr.dlid;
 478                 rqpn = qp->remote_qpn;
 479                 svc_type = IB_CC_SVCTYPE_UC;
 480                 break;
 481         case IB_QPT_RC:
 482                 rlid = qp->remote_ah_attr.dlid;
 483                 rqpn = qp->remote_qpn;
 484                 svc_type = IB_CC_SVCTYPE_RC;
 485                 break;
 486         default:
 487                 return;
 488         }
 489
 490         sc = hdr2sc((struct hfi1_message_header *)hdr, pkt->rhf);
 491
 492         bth1 = be32_to_cpu(ohdr->bth[1]);
 493         if (do_cnp && (bth1 & HFI1_FECN_SMASK)) {
 494                 u16 pkey = (u16)be32_to_cpu(ohdr->bth[0]);
 495
 496                 return_cnp(ibp, qp, rqpn, pkey, dlid, rlid, sc, grh);
 497         }
 498
 499         if (!is_mcast && (bth1 & HFI1_BECN_SMASK)) {
 500                 struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
 501                 u32 lqpn = bth1 & RVT_QPN_MASK;
 502                 u8 sl = ibp->sc_to_sl[sc];
 503
 504                 process_becn(ppd, sl, rlid, lqpn, rqpn, svc_type);
 505         }
 506
 507 }
 508
 509 struct ps_mdata {
 510         struct hfi1_ctxtdata *rcd;
 511         u32 rsize;
 512         u32 maxcnt;
 513         u32 ps_head;
 514         u32 ps_tail;
 515         u32 ps_seq;
 516 };
 517
 518 static inline void init_ps_mdata(struct ps_mdata *mdata,
 519                                  struct hfi1_packet *packet)
 520 {
 521         struct hfi1_ctxtdata *rcd = packet->rcd;
 522
 523         mdata->rcd = rcd;
 524         mdata->rsize = packet->rsize;
 525         mdata->maxcnt = packet->maxcnt;
 526         mdata->ps_head = packet->rhqoff;
 527
 528         if (HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL)) {
 529                 mdata->ps_tail = get_rcvhdrtail(rcd);
 530                 if (rcd->ctxt == HFI1_CTRL_CTXT)
 531                         mdata->ps_seq = rcd->seq_cnt;
 532                 else
 533                         mdata->ps_seq = 0; /* not used with DMA_RTAIL */
 534         } else {
 535                 mdata->ps_tail = 0; /* used only with DMA_RTAIL*/
 536                 mdata->ps_seq = rcd->seq_cnt;
 537         }
 538 }
 539
 540 static inline int ps_done(struct ps_mdata *mdata, u64 rhf,
 541                           struct hfi1_ctxtdata *rcd)
 542 {
 543         if (HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL))
 544                 return mdata->ps_head == mdata->ps_tail;
 545         return mdata->ps_seq != rhf_rcv_seq(rhf);
 546 }
 547
 548 static inline int ps_skip(struct ps_mdata *mdata, u64 rhf,
 549                           struct hfi1_ctxtdata *rcd)
 550 {
 551         /*
 552          * Control context can potentially receive an invalid rhf.
 553          * Drop such packets.
 554          */
 555         if ((rcd->ctxt == HFI1_CTRL_CTXT) && (mdata->ps_head != mdata->ps_tail))
 556                 return mdata->ps_seq != rhf_rcv_seq(rhf);
 557
 558         return 0;
 559 }
 560
 561 static inline void update_ps_mdata(struct ps_mdata *mdata,
 562                                    struct hfi1_ctxtdata *rcd)
 563 {
 564         mdata->ps_head += mdata->rsize;
 565         if (mdata->ps_head >= mdata->maxcnt)
 566                 mdata->ps_head = 0;
 567
 568         /* Control context must do seq counting */
 569         if (!HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL) ||
 570             (rcd->ctxt == HFI1_CTRL_CTXT)) {
 571                 if (++mdata->ps_seq > 13)
 572                         mdata->ps_seq = 1;
 573         }
 574 }
 575
 576 /*
 577  * prescan_rxq - search through the receive queue looking for packets
 578  * containing Excplicit Congestion Notifications (FECNs, or BECNs).
 579  * When an ECN is found, process the Congestion Notification, and toggle
 580  * it off.
 581  * This is declared as a macro to allow quick checking of the port to avoid
 582  * the overhead of a function call if not enabled.
 583  */
 584 #define prescan_rxq(rcd, packet) \
 585         do { \
 586                 if (rcd->ppd->cc_prescan) \
 587                         __prescan_rxq(packet); \
 588         } while (0)
 589 static void __prescan_rxq(struct hfi1_packet *packet)
 590 {
 591         struct hfi1_ctxtdata *rcd = packet->rcd;
 592         struct ps_mdata mdata;
 593
 594         init_ps_mdata(&mdata, packet);
 595
 596         while (1) {
 597                 struct hfi1_devdata *dd = rcd->dd;
 598                 struct hfi1_ibport *ibp = &rcd->ppd->ibport_data;
 599                 __le32 *rhf_addr = (__le32 *)rcd->rcvhdrq + mdata.ps_head +
 600                                          dd->rhf_offset;
 601                 struct rvt_qp *qp;
 602                 struct hfi1_ib_header *hdr;
 603                 struct hfi1_other_headers *ohdr;
 604                 struct rvt_dev_info *rdi = &dd->verbs_dev.rdi;
 605                 u64 rhf = rhf_to_cpu(rhf_addr);
 606                 u32 etype = rhf_rcv_type(rhf), qpn, bth1;
 607                 int is_ecn = 0;
 608                 u8 lnh;
 609
 610                 if (ps_done(&mdata, rhf, rcd))
 611                         break;
 612
 613                 if (ps_skip(&mdata, rhf, rcd))
 614                         goto next;
 615
 616                 if (etype != RHF_RCV_TYPE_IB)
 617                         goto next;
 618
 619                 hdr = (struct hfi1_ib_header *)
 620                         hfi1_get_msgheader(dd, rhf_addr);
 621                 lnh = be16_to_cpu(hdr->lrh[0]) & 3;
 622
 623                 if (lnh == HFI1_LRH_BTH)
 624                         ohdr = &hdr->u.oth;
 625                 else if (lnh == HFI1_LRH_GRH)
 626                         ohdr = &hdr->u.l.oth;
 627                 else
 628                         goto next; /* just in case */
 629
 630                 bth1 = be32_to_cpu(ohdr->bth[1]);
 631                 is_ecn = !!(bth1 & (HFI1_FECN_SMASK | HFI1_BECN_SMASK));
 632
 633                 if (!is_ecn)
 634                         goto next;
 635
 636                 qpn = bth1 & RVT_QPN_MASK;
 637                 rcu_read_lock();
 638                 qp = rvt_lookup_qpn(rdi, &ibp->rvp, qpn);
 639
 640                 if (!qp) {
 641                         rcu_read_unlock();
 642                         goto next;
 643                 }
 644
 645                 process_ecn(qp, packet, true);
 646                 rcu_read_unlock();
 647
 648                 /* turn off BECN, FECN */
 649                 bth1 &= ~(HFI1_FECN_SMASK | HFI1_BECN_SMASK);
 650                 ohdr->bth[1] = cpu_to_be32(bth1);
 651 next:
 652                 update_ps_mdata(&mdata, rcd);
 653         }
 654 }
 655
 656 static inline int skip_rcv_packet(struct hfi1_packet *packet, int thread)
 657 {
 658         int ret = RCV_PKT_OK;
 659
 660         /* Set up for the next packet */
 661         packet->rhqoff += packet->rsize;
 662         if (packet->rhqoff >= packet->maxcnt)
 663                 packet->rhqoff = 0;
 664
 665         packet->numpkt++;
 666         if (unlikely((packet->numpkt & (MAX_PKT_RECV - 1)) == 0)) {
 667                 if (thread) {
 668                         cond_resched();
 669                 } else {
 670                         ret = RCV_PKT_LIMIT;
 671                         this_cpu_inc(*packet->rcd->dd->rcv_limit);
 672                 }
 673         }
 674
 675         packet->rhf_addr = (__le32 *)packet->rcd->rcvhdrq + packet->rhqoff +
 676                                      packet->rcd->dd->rhf_offset;
 677         packet->rhf = rhf_to_cpu(packet->rhf_addr);
 678
 679         return ret;
 680 }
 681
 682 static inline int process_rcv_packet(struct hfi1_packet *packet, int thread)
 683 {
 684         int ret = RCV_PKT_OK;
 685
 686         packet->hdr = hfi1_get_msgheader(packet->rcd->dd,
 687                                          packet->rhf_addr);
 688         packet->hlen = (u8 *)packet->rhf_addr - (u8 *)packet->hdr;
 689         packet->etype = rhf_rcv_type(packet->rhf);
 690         /* total length */
 691         packet->tlen = rhf_pkt_len(packet->rhf); /* in bytes */
 692         /* retrieve eager buffer details */
 693         packet->ebuf = NULL;
 694         if (rhf_use_egr_bfr(packet->rhf)) {
 695                 packet->etail = rhf_egr_index(packet->rhf);
 696                 packet->ebuf = get_egrbuf(packet->rcd, packet->rhf,
 697                                  &packet->updegr);
 698                 /*
 699                  * Prefetch the contents of the eager buffer.  It is
 700                  * OK to send a negative length to prefetch_range().
 701                  * The +2 is the size of the RHF.
 702                  */
 703                 prefetch_range(packet->ebuf,
 704                                packet->tlen - ((packet->rcd->rcvhdrqentsize -
 705                                                (rhf_hdrq_offset(packet->rhf)
 706                                                 + 2)) * 4));
 707         }
 708
 709         /*
 710          * Call a type specific handler for the packet. We
 711          * should be able to trust that etype won't be beyond
 712          * the range of valid indexes. If so something is really
 713          * wrong and we can probably just let things come
 714          * crashing down. There is no need to eat another
 715          * comparison in this performance critical code.
 716          */
 717         packet->rcd->dd->rhf_rcv_function_map[packet->etype](packet);
 718         packet->numpkt++;
 719
 720         /* Set up for the next packet */
 721         packet->rhqoff += packet->rsize;
 722         if (packet->rhqoff >= packet->maxcnt)
 723                 packet->rhqoff = 0;
 724
 725         if (unlikely((packet->numpkt & (MAX_PKT_RECV - 1)) == 0)) {
 726                 if (thread) {
 727                         cond_resched();
 728                 } else {
 729                         ret = RCV_PKT_LIMIT;
 730                         this_cpu_inc(*packet->rcd->dd->rcv_limit);
 731                 }
 732         }
 733
 734         packet->rhf_addr = (__le32 *)packet->rcd->rcvhdrq + packet->rhqoff +
 735                                       packet->rcd->dd->rhf_offset;
 736         packet->rhf = rhf_to_cpu(packet->rhf_addr);
 737
 738         return ret;
 739 }
 740
 741 static inline void process_rcv_update(int last, struct hfi1_packet *packet)
 742 {
 743         /*
 744          * Update head regs etc., every 16 packets, if not last pkt,
 745          * to help prevent rcvhdrq overflows, when many packets
 746          * are processed and queue is nearly full.
 747          * Don't request an interrupt for intermediate updates.
 748          */
 749         if (!last && !(packet->numpkt & 0xf)) {
 750                 update_usrhead(packet->rcd, packet->rhqoff, packet->updegr,
 751                                packet->etail, 0, 0);
 752                 packet->updegr = 0;
 753         }
 754         packet->rcv_flags = 0;
 755 }
 756
 757 static inline void finish_packet(struct hfi1_packet *packet)
 758 {
 759         /*
 760          * Nothing we need to free for the packet.
 761          *
 762          * The only thing we need to do is a final update and call for an
 763          * interrupt
 764          */
 765         update_usrhead(packet->rcd, packet->rcd->head, packet->updegr,
 766                        packet->etail, rcv_intr_dynamic, packet->numpkt);
 767 }
 768
 769 static inline void process_rcv_qp_work(struct hfi1_packet *packet)
 770 {
 771         struct hfi1_ctxtdata *rcd;
 772         struct rvt_qp *qp, *nqp;
 773
 774         rcd = packet->rcd;
 775         rcd->head = packet->rhqoff;
 776
 777         /*
 778          * Iterate over all QPs waiting to respond.
 779          * The list won't change since the IRQ is only run on one CPU.
 780          */
 781         list_for_each_entry_safe(qp, nqp, &rcd->qp_wait_list, rspwait) {
 782                 list_del_init(&qp->rspwait);
 783                 if (qp->r_flags & RVT_R_RSP_NAK) {
 784                         qp->r_flags &= ~RVT_R_RSP_NAK;
 785                         hfi1_send_rc_ack(rcd, qp, 0);
 786                 }
 787                 if (qp->r_flags & RVT_R_RSP_SEND) {
 788                         unsigned long flags;
 789
 790                         qp->r_flags &= ~RVT_R_RSP_SEND;
 791                         spin_lock_irqsave(&qp->s_lock, flags);
 792                         if (ib_rvt_state_ops[qp->state] &
 793                                         RVT_PROCESS_OR_FLUSH_SEND)
 794                                 hfi1_schedule_send(qp);
 795                         spin_unlock_irqrestore(&qp->s_lock, flags);
 796                 }
 797                 if (atomic_dec_and_test(&qp->refcount))
 798                         wake_up(&qp->wait);
 799         }
 800 }
 801
 802 /*
 803  * Handle receive interrupts when using the no dma rtail option.
 804  */
 805 int handle_receive_interrupt_nodma_rtail(struct hfi1_ctxtdata *rcd, int thread)
 806 {
 807         u32 seq;
 808         int last = RCV_PKT_OK;
 809         struct hfi1_packet packet;
 810
 811         init_packet(rcd, &packet);
 812         seq = rhf_rcv_seq(packet.rhf);
 813         if (seq != rcd->seq_cnt) {
 814                 last = RCV_PKT_DONE;
 815                 goto bail;
 816         }
 817
 818         prescan_rxq(rcd, &packet);
 819
 820         while (last == RCV_PKT_OK) {
 821                 last = process_rcv_packet(&packet, thread);
 822                 seq = rhf_rcv_seq(packet.rhf);
 823                 if (++rcd->seq_cnt > 13)
 824                         rcd->seq_cnt = 1;
 825                 if (seq != rcd->seq_cnt)
 826                         last = RCV_PKT_DONE;
 827                 process_rcv_update(last, &packet);
 828         }
 829         process_rcv_qp_work(&packet);
 830 bail:
 831         finish_packet(&packet);
 832         return last;
 833 }
 834
 835 int handle_receive_interrupt_dma_rtail(struct hfi1_ctxtdata *rcd, int thread)
 836 {
 837         u32 hdrqtail;
 838         int last = RCV_PKT_OK;
 839         struct hfi1_packet packet;
 840
 841         init_packet(rcd, &packet);
 842         hdrqtail = get_rcvhdrtail(rcd);
 843         if (packet.rhqoff == hdrqtail) {
 844                 last = RCV_PKT_DONE;
 845                 goto bail;
 846         }
 847         smp_rmb();  /* prevent speculative reads of dma'ed hdrq */
 848
 849         prescan_rxq(rcd, &packet);
 850
 851         while (last == RCV_PKT_OK) {
 852                 last = process_rcv_packet(&packet, thread);
 853                 if (packet.rhqoff == hdrqtail)
 854                         last = RCV_PKT_DONE;
 855                 process_rcv_update(last, &packet);
 856         }
 857         process_rcv_qp_work(&packet);
 858 bail:
 859         finish_packet(&packet);
 860         return last;
 861 }
 862
 863 static inline void set_all_nodma_rtail(struct hfi1_devdata *dd)
 864 {
 865         int i;
 866
 867         for (i = HFI1_CTRL_CTXT + 1; i < dd->first_user_ctxt; i++)
 868                 dd->rcd[i]->do_interrupt =
 869                         &handle_receive_interrupt_nodma_rtail;
 870 }
 871
 872 static inline void set_all_dma_rtail(struct hfi1_devdata *dd)
 873 {
 874         int i;
 875
 876         for (i = HFI1_CTRL_CTXT + 1; i < dd->first_user_ctxt; i++)
 877                 dd->rcd[i]->do_interrupt =
 878                         &handle_receive_interrupt_dma_rtail;
 879 }
 880
 881 void set_all_slowpath(struct hfi1_devdata *dd)
 882 {
 883         int i;
 884
 885         /* HFI1_CTRL_CTXT must always use the slow path interrupt handler */
 886         for (i = HFI1_CTRL_CTXT + 1; i < dd->first_user_ctxt; i++)
 887                 dd->rcd[i]->do_interrupt = &handle_receive_interrupt;
 888 }
 889
 890 static inline int set_armed_to_active(struct hfi1_ctxtdata *rcd,
 891                                       struct hfi1_packet *packet,
 892                                       struct hfi1_devdata *dd)
 893 {
 894         struct work_struct *lsaw = &rcd->ppd->linkstate_active_work;
 895         struct hfi1_message_header *hdr = hfi1_get_msgheader(packet->rcd->dd,
 896                                                              packet->rhf_addr);
 897         u8 etype = rhf_rcv_type(packet->rhf);
 898
 899         if (etype == RHF_RCV_TYPE_IB && hdr2sc(hdr, packet->rhf) != 0xf) {
 900                 int hwstate = read_logical_state(dd);
 901
 902                 if (hwstate != LSTATE_ACTIVE) {
 903                         dd_dev_info(dd, "Unexpected link state %d\n", hwstate);
 904                         return 0;
 905                 }
 906
 907                 queue_work(rcd->ppd->hfi1_wq, lsaw);
 908                 return 1;
 909         }
 910         return 0;
 911 }
 912
 913 /*
 914  * handle_receive_interrupt - receive a packet
 915  * @rcd: the context
 916  *
 917  * Called from interrupt handler for errors or receive interrupt.
 918  * This is the slow path interrupt handler.
 919  */
 920 int handle_receive_interrupt(struct hfi1_ctxtdata *rcd, int thread)
 921 {
 922         struct hfi1_devdata *dd = rcd->dd;
 923         u32 hdrqtail;
 924         int needset, last = RCV_PKT_OK;
 925         struct hfi1_packet packet;
 926         int skip_pkt = 0;
 927
 928         /* Control context will always use the slow path interrupt handler */
 929         needset = (rcd->ctxt == HFI1_CTRL_CTXT) ? 0 : 1;
 930
 931         init_packet(rcd, &packet);
 932
 933         if (!HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL)) {
 934                 u32 seq = rhf_rcv_seq(packet.rhf);
 935
 936                 if (seq != rcd->seq_cnt) {
 937                         last = RCV_PKT_DONE;
 938                         goto bail;
 939                 }
 940                 hdrqtail = 0;
 941         } else {
 942                 hdrqtail = get_rcvhdrtail(rcd);
 943                 if (packet.rhqoff == hdrqtail) {
 944                         last = RCV_PKT_DONE;
 945                         goto bail;
 946                 }
 947                 smp_rmb();  /* prevent speculative reads of dma'ed hdrq */
 948
 949                 /*
 950                  * Control context can potentially receive an invalid
 951                  * rhf. Drop such packets.
 952                  */
 953                 if (rcd->ctxt == HFI1_CTRL_CTXT) {
 954                         u32 seq = rhf_rcv_seq(packet.rhf);
 955
 956                         if (seq != rcd->seq_cnt)
 957                                 skip_pkt = 1;
 958                 }
 959         }
 960
 961         prescan_rxq(rcd, &packet);
 962
 963         while (last == RCV_PKT_OK) {
 964                 if (unlikely(dd->do_drop &&
 965                              atomic_xchg(&dd->drop_packet, DROP_PACKET_OFF) ==
 966                              DROP_PACKET_ON)) {
 967                         dd->do_drop = 0;
 968
 969                         /* On to the next packet */
 970                         packet.rhqoff += packet.rsize;
 971                         packet.rhf_addr = (__le32 *)rcd->rcvhdrq +
 972                                           packet.rhqoff +
 973                                           dd->rhf_offset;
 974                         packet.rhf = rhf_to_cpu(packet.rhf_addr);
 975
 976                 } else if (skip_pkt) {
 977                         last = skip_rcv_packet(&packet, thread);
 978                         skip_pkt = 0;
 979                 } else {
 980                         /* Auto activate link on non-SC15 packet receive */
 981                         if (unlikely(rcd->ppd->host_link_state ==
 982                                      HLS_UP_ARMED) &&
 983                             set_armed_to_active(rcd, &packet, dd))
 984                                 goto bail;
 985                         last = process_rcv_packet(&packet, thread);
 986                 }
 987
 988                 if (!HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL)) {
 989                         u32 seq = rhf_rcv_seq(packet.rhf);
 990
 991                         if (++rcd->seq_cnt > 13)
 992                                 rcd->seq_cnt = 1;
 993                         if (seq != rcd->seq_cnt)
 994                                 last = RCV_PKT_DONE;
 995                         if (needset) {
 996                                 dd_dev_info(dd, "Switching to NO_DMA_RTAIL\n");
 997                                 set_all_nodma_rtail(dd);
 998                                 needset = 0;
 999                         }
1000                 } else {
1001                         if (packet.rhqoff == hdrqtail)
1002                                 last = RCV_PKT_DONE;
1003                         /*
1004                          * Control context can potentially receive an invalid
1005                          * rhf. Drop such packets.
1006                          */
1007                         if (rcd->ctxt == HFI1_CTRL_CTXT) {
1008                                 u32 seq = rhf_rcv_seq(packet.rhf);
1009
1010                                 if (++rcd->seq_cnt > 13)
1011                                         rcd->seq_cnt = 1;
1012                                 if (!last && (seq != rcd->seq_cnt))
1013                                         skip_pkt = 1;
1014                         }
1015
1016                         if (needset) {
1017                                 dd_dev_info(dd,
1018                                             "Switching to DMA_RTAIL\n");
1019                                 set_all_dma_rtail(dd);
1020                                 needset = 0;
1021                         }
1022                 }
1023
1024                 process_rcv_update(last, &packet);
1025         }
1026
1027         process_rcv_qp_work(&packet);
1028
1029 bail:
1030         /*
1031          * Always write head at end, and setup rcv interrupt, even
1032          * if no packets were processed.
1033          */
1034         finish_packet(&packet);
1035         return last;
1036 }
1037
1038 /*
1039  * We may discover in the interrupt that the hardware link state has
1040  * changed from ARMED to ACTIVE (due to the arrival of a non-SC15 packet),
1041  * and we need to update the driver's notion of the link state.  We cannot
1042  * run set_link_state from interrupt context, so we queue this function on
1043  * a workqueue.
1044  *
1045  * We delay the regular interrupt processing until after the state changes
1046  * so that the link will be in the correct state by the time any application
1047  * we wake up attempts to send a reply to any message it received.
1048  * (Subsequent receive interrupts may possibly force the wakeup before we
1049  * update the link state.)
1050  *
1051  * The rcd is freed in hfi1_free_ctxtdata after hfi1_postinit_cleanup invokes
1052  * dd->f_cleanup(dd) to disable the interrupt handler and flush workqueues,
1053  * so we're safe from use-after-free of the rcd.
1054  */
1055 void receive_interrupt_work(struct work_struct *work)
1056 {
1057         struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
1058                                                   linkstate_active_work);
1059         struct hfi1_devdata *dd = ppd->dd;
1060         int i;
1061
1062         /* Received non-SC15 packet implies neighbor_normal */
1063         ppd->neighbor_normal = 1;
1064         set_link_state(ppd, HLS_UP_ACTIVE);
1065
1066         /*
1067          * Interrupt all kernel contexts that could have had an
1068          * interrupt during auto activation.
1069          */
1070         for (i = HFI1_CTRL_CTXT; i < dd->first_user_ctxt; i++)
1071                 force_recv_intr(dd->rcd[i]);
1072 }
1073
1074 /*
1075  * Convert a given MTU size to the on-wire MAD packet enumeration.
1076  * Return -1 if the size is invalid.
1077  */
1078 int mtu_to_enum(u32 mtu, int default_if_bad)
1079 {
1080         switch (mtu) {
1081         case     0: return OPA_MTU_0;
1082         case   256: return OPA_MTU_256;
1083         case   512: return OPA_MTU_512;
1084         case  1024: return OPA_MTU_1024;
1085         case  2048: return OPA_MTU_2048;
1086         case  4096: return OPA_MTU_4096;
1087         case  8192: return OPA_MTU_8192;
1088         case 10240: return OPA_MTU_10240;
1089         }
1090         return default_if_bad;
1091 }
1092
1093 u16 enum_to_mtu(int mtu)
1094 {
1095         switch (mtu) {
1096         case OPA_MTU_0:     return 0;
1097         case OPA_MTU_256:   return 256;
1098         case OPA_MTU_512:   return 512;
1099         case OPA_MTU_1024:  return 1024;
1100         case OPA_MTU_2048:  return 2048;
1101         case OPA_MTU_4096:  return 4096;
1102         case OPA_MTU_8192:  return 8192;
1103         case OPA_MTU_10240: return 10240;
1104         default: return 0xffff;
1105         }
1106 }
1107
1108 /*
1109  * set_mtu - set the MTU
1110  * @ppd: the per port data
1111  *
1112  * We can handle "any" incoming size, the issue here is whether we
1113  * need to restrict our outgoing size.  We do not deal with what happens
1114  * to programs that are already running when the size changes.
1115  */
1116 int set_mtu(struct hfi1_pportdata *ppd)
1117 {
1118         struct hfi1_devdata *dd = ppd->dd;
1119         int i, drain, ret = 0, is_up = 0;
1120
1121         ppd->ibmtu = 0;
1122         for (i = 0; i < ppd->vls_supported; i++)
1123                 if (ppd->ibmtu < dd->vld[i].mtu)
1124                         ppd->ibmtu = dd->vld[i].mtu;
1125         ppd->ibmaxlen = ppd->ibmtu + lrh_max_header_bytes(ppd->dd);
1126
1127         mutex_lock(&ppd->hls_lock);
1128         if (ppd->host_link_state == HLS_UP_INIT ||
1129             ppd->host_link_state == HLS_UP_ARMED ||
1130             ppd->host_link_state == HLS_UP_ACTIVE)
1131                 is_up = 1;
1132
1133         drain = !is_ax(dd) && is_up;
1134
1135         if (drain)
1136                 /*
1137                  * MTU is specified per-VL. To ensure that no packet gets
1138                  * stuck (due, e.g., to the MTU for the packet's VL being
1139                  * reduced), empty the per-VL FIFOs before adjusting MTU.
1140                  */
1141                 ret = stop_drain_data_vls(dd);
1142
1143         if (ret) {
1144                 dd_dev_err(dd, "%s: cannot stop/drain VLs - refusing to change per-VL MTUs\n",
1145                            __func__);
1146                 goto err;
1147         }
1148
1149         hfi1_set_ib_cfg(ppd, HFI1_IB_CFG_MTU, 0);
1150
1151         if (drain)
1152                 open_fill_data_vls(dd); /* reopen all VLs */
1153
1154 err:
1155         mutex_unlock(&ppd->hls_lock);
1156
1157         return ret;
1158 }
1159
1160 int hfi1_set_lid(struct hfi1_pportdata *ppd, u32 lid, u8 lmc)
1161 {
1162         struct hfi1_devdata *dd = ppd->dd;
1163
1164         ppd->lid = lid;
1165         ppd->lmc = lmc;
1166         hfi1_set_ib_cfg(ppd, HFI1_IB_CFG_LIDLMC, 0);
1167
1168         dd_dev_info(dd, "port %u: got a lid: 0x%x\n", ppd->port, lid);
1169
1170         return 0;
1171 }
1172
1173 void shutdown_led_override(struct hfi1_pportdata *ppd)
1174 {
1175         struct hfi1_devdata *dd = ppd->dd;
1176
1177         /*
1178          * This pairs with the memory barrier in hfi1_start_led_override to
1179          * ensure that we read the correct state of LED beaconing represented
1180          * by led_override_timer_active
1181          */
1182         smp_rmb();
1183         if (atomic_read(&ppd->led_override_timer_active)) {
1184                 del_timer_sync(&ppd->led_override_timer);
1185                 atomic_set(&ppd->led_override_timer_active, 0);
1186                 /* Ensure the atomic_set is visible to all CPUs */
1187                 smp_wmb();
1188         }
1189
1190         /* Hand control of the LED to the DC for normal operation */
1191         write_csr(dd, DCC_CFG_LED_CNTRL, 0);
1192 }
1193
1194 static void run_led_override(unsigned long opaque)
1195 {
1196         struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)opaque;
1197         struct hfi1_devdata *dd = ppd->dd;
1198         unsigned long timeout;
1199         int phase_idx;
1200
1201         if (!(dd->flags & HFI1_INITTED))
1202                 return;
1203
1204         phase_idx = ppd->led_override_phase & 1;
1205
1206         setextled(dd, phase_idx);
1207
1208         timeout = ppd->led_override_vals[phase_idx];
1209
1210         /* Set up for next phase */
1211         ppd->led_override_phase = !ppd->led_override_phase;
1212
1213         mod_timer(&ppd->led_override_timer, jiffies + timeout);
1214 }
1215
1216 /*
1217  * To have the LED blink in a particular pattern, provide timeon and timeoff
1218  * in milliseconds.
1219  * To turn off custom blinking and return to normal operation, use
1220  * shutdown_led_override()
1221  */
1222 void hfi1_start_led_override(struct hfi1_pportdata *ppd, unsigned int timeon,
1223                              unsigned int timeoff)
1224 {
1225         if (!(ppd->dd->flags & HFI1_INITTED))
1226                 return;
1227
1228         /* Convert to jiffies for direct use in timer */
1229         ppd->led_override_vals[0] = msecs_to_jiffies(timeoff);
1230         ppd->led_override_vals[1] = msecs_to_jiffies(timeon);
1231
1232         /* Arbitrarily start from LED on phase */
1233         ppd->led_override_phase = 1;
1234
1235         /*
1236          * If the timer has not already been started, do so. Use a "quick"
1237          * timeout so the handler will be called soon to look at our request.
1238          */
1239         if (!timer_pending(&ppd->led_override_timer)) {
1240                 setup_timer(&ppd->led_override_timer, run_led_override,
1241                             (unsigned long)ppd);
1242                 ppd->led_override_timer.expires = jiffies + 1;
1243                 add_timer(&ppd->led_override_timer);
1244                 atomic_set(&ppd->led_override_timer_active, 1);
1245                 /* Ensure the atomic_set is visible to all CPUs */
1246                 smp_wmb();
1247         }
1248 }
1249
1250 /**
1251  * hfi1_reset_device - reset the chip if possible
1252  * @unit: the device to reset
1253  *
1254  * Whether or not reset is successful, we attempt to re-initialize the chip
1255  * (that is, much like a driver unload/reload).  We clear the INITTED flag
1256  * so that the various entry points will fail until we reinitialize.  For
1257  * now, we only allow this if no user contexts are open that use chip resources
1258  */
1259 int hfi1_reset_device(int unit)
1260 {
1261         int ret, i;
1262         struct hfi1_devdata *dd = hfi1_lookup(unit);
1263         struct hfi1_pportdata *ppd;
1264         unsigned long flags;
1265         int pidx;
1266
1267         if (!dd) {
1268                 ret = -ENODEV;
1269                 goto bail;
1270         }
1271
1272         dd_dev_info(dd, "Reset on unit %u requested\n", unit);
1273
1274         if (!dd->kregbase || !(dd->flags & HFI1_PRESENT)) {
1275                 dd_dev_info(dd,
1276                             "Invalid unit number %u or not initialized or not present\n",
1277                             unit);
1278                 ret = -ENXIO;
1279                 goto bail;
1280         }
1281
1282         spin_lock_irqsave(&dd->uctxt_lock, flags);
1283         if (dd->rcd)
1284                 for (i = dd->first_user_ctxt; i < dd->num_rcv_contexts; i++) {
1285                         if (!dd->rcd[i] || !dd->rcd[i]->cnt)
1286                                 continue;
1287                         spin_unlock_irqrestore(&dd->uctxt_lock, flags);
1288                         ret = -EBUSY;
1289                         goto bail;
1290                 }
1291         spin_unlock_irqrestore(&dd->uctxt_lock, flags);
1292
1293         for (pidx = 0; pidx < dd->num_pports; ++pidx) {
1294                 ppd = dd->pport + pidx;
1295
1296                 shutdown_led_override(ppd);
1297         }
1298         if (dd->flags & HFI1_HAS_SEND_DMA)
1299                 sdma_exit(dd);
1300
1301         hfi1_reset_cpu_counters(dd);
1302
1303         ret = hfi1_init(dd, 1);
1304
1305         if (ret)
1306                 dd_dev_err(dd,
1307                            "Reinitialize unit %u after reset failed with %d\n",
1308                            unit, ret);
1309         else
1310                 dd_dev_info(dd, "Reinitialized unit %u after resetting\n",
1311                             unit);
1312
1313 bail:
1314         return ret;
1315 }
1316
1317 void handle_eflags(struct hfi1_packet *packet)
1318 {
1319         struct hfi1_ctxtdata *rcd = packet->rcd;
1320         u32 rte = rhf_rcv_type_err(packet->rhf);
1321
1322         rcv_hdrerr(rcd, rcd->ppd, packet);
1323         if (rhf_err_flags(packet->rhf))
1324                 dd_dev_err(rcd->dd,
1325                            "receive context %d: rhf 0x%016llx, errs [ %s%s%s%s%s%s%s%s] rte 0x%x\n",
1326                            rcd->ctxt, packet->rhf,
1327                            packet->rhf & RHF_K_HDR_LEN_ERR ? "k_hdr_len " : "",
1328                            packet->rhf & RHF_DC_UNC_ERR ? "dc_unc " : "",
1329                            packet->rhf & RHF_DC_ERR ? "dc " : "",
1330                            packet->rhf & RHF_TID_ERR ? "tid " : "",
1331                            packet->rhf & RHF_LEN_ERR ? "len " : "",
1332                            packet->rhf & RHF_ECC_ERR ? "ecc " : "",
1333                            packet->rhf & RHF_VCRC_ERR ? "vcrc " : "",
1334                            packet->rhf & RHF_ICRC_ERR ? "icrc " : "",
1335                            rte);
1336 }
1337
1338 /*
1339  * The following functions are called by the interrupt handler. They are type
1340  * specific handlers for each packet type.
1341  */
1342 int process_receive_ib(struct hfi1_packet *packet)
1343 {
1344         trace_hfi1_rcvhdr(packet->rcd->ppd->dd,
1345                           packet->rcd->ctxt,
1346                           rhf_err_flags(packet->rhf),
1347                           RHF_RCV_TYPE_IB,
1348                           packet->hlen,
1349                           packet->tlen,
1350                           packet->updegr,
1351                           rhf_egr_index(packet->rhf));
1352
1353         if (unlikely(rhf_err_flags(packet->rhf))) {
1354                 handle_eflags(packet);
1355                 return RHF_RCV_CONTINUE;
1356         }
1357
1358         hfi1_ib_rcv(packet);
1359         return RHF_RCV_CONTINUE;
1360 }
1361
1362 int process_receive_bypass(struct hfi1_packet *packet)
1363 {
1364         if (unlikely(rhf_err_flags(packet->rhf)))
1365                 handle_eflags(packet);
1366
1367         dd_dev_err(packet->rcd->dd,
1368                    "Bypass packets are not supported in normal operation. Dropping\n");
1369         incr_cntr64(&packet->rcd->dd->sw_rcv_bypass_packet_errors);
1370         return RHF_RCV_CONTINUE;
1371 }
1372
1373 int process_receive_error(struct hfi1_packet *packet)
1374 {
1375         handle_eflags(packet);
1376
1377         if (unlikely(rhf_err_flags(packet->rhf)))
1378                 dd_dev_err(packet->rcd->dd,
1379                            "Unhandled error packet received. Dropping.\n");
1380
1381         return RHF_RCV_CONTINUE;
1382 }
1383
1384 int kdeth_process_expected(struct hfi1_packet *packet)
1385 {
1386         if (unlikely(rhf_err_flags(packet->rhf)))
1387                 handle_eflags(packet);
1388
1389         dd_dev_err(packet->rcd->dd,
1390                    "Unhandled expected packet received. Dropping.\n");
1391         return RHF_RCV_CONTINUE;
1392 }
1393
1394 int kdeth_process_eager(struct hfi1_packet *packet)
1395 {
1396         if (unlikely(rhf_err_flags(packet->rhf)))
1397                 handle_eflags(packet);
1398
1399         dd_dev_err(packet->rcd->dd,
1400                    "Unhandled eager packet received. Dropping.\n");
1401         return RHF_RCV_CONTINUE;
1402 }
1403
1404 int process_receive_invalid(struct hfi1_packet *packet)
1405 {
1406         dd_dev_err(packet->rcd->dd, "Invalid packet type %d. Dropping\n",
1407                    rhf_rcv_type(packet->rhf));
1408         return RHF_RCV_CONTINUE;
1409 }