drivers/net/ethernet/mellanox/mlx4/resource_tracker.c

   1 /*
   2  * Copyright (c) 2004, 2005 Topspin Communications.  All rights reserved.
   3  * Copyright (c) 2005, 2006, 2007, 2008 Mellanox Technologies.
   4  * All rights reserved.
   5  * Copyright (c) 2005, 2006, 2007 Cisco Systems, Inc.  All rights reserved.
   6  *
   7  * This software is available to you under a choice of one of two
   8  * licenses.  You may choose to be licensed under the terms of the GNU
   9  * General Public License (GPL) Version 2, available from the file
  10  * COPYING in the main directory of this source tree, or the
  11  * OpenIB.org BSD license below:
  12  *
  13  *     Redistribution and use in source and binary forms, with or
  14  *     without modification, are permitted provided that the following
  15  *     conditions are met:
  16  *
  17  *      - Redistributions of source code must retain the above
  18  *        copyright notice, this list of conditions and the following
  19  *        disclaimer.
  20  *
  21  *      - Redistributions in binary form must reproduce the above
  22  *        copyright notice, this list of conditions and the following
  23  *        disclaimer in the documentation and/or other materials
  24  *        provided with the distribution.
  25  *
  26  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  27  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  28  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  29  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  30  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  31  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  32  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  33  * SOFTWARE.
  34  */
  35
  36 #include <linux/sched.h>
  37 #include <linux/pci.h>
  38 #include <linux/errno.h>
  39 #include <linux/kernel.h>
  40 #include <linux/io.h>
  41 #include <linux/slab.h>
  42 #include <linux/mlx4/cmd.h>
  43 #include <linux/mlx4/qp.h>
  44 #include <linux/if_ether.h>
  45 #include <linux/etherdevice.h>
  46
  47 #include "mlx4.h"
  48 #include "fw.h"
  49 #include "mlx4_stats.h"
  50
  51 #define MLX4_MAC_VALID          (1ull << 63)
  52 #define MLX4_PF_COUNTERS_PER_PORT       2
  53 #define MLX4_VF_COUNTERS_PER_PORT       1
  54
  55 struct mac_res {
  56         struct list_head list;
  57         u64 mac;
  58         int ref_count;
  59         u8 smac_index;
  60         u8 port;
  61 };
  62
  63 struct vlan_res {
  64         struct list_head list;
  65         u16 vlan;
  66         int ref_count;
  67         int vlan_index;
  68         u8 port;
  69 };
  70
  71 struct res_common {
  72         struct list_head        list;
  73         struct rb_node          node;
  74         u64                     res_id;
  75         int                     owner;
  76         int                     state;
  77         int                     from_state;
  78         int                     to_state;
  79         int                     removing;
  80 };
  81
  82 enum {
  83         RES_ANY_BUSY = 1
  84 };
  85
  86 struct res_gid {
  87         struct list_head        list;
  88         u8                      gid[16];
  89         enum mlx4_protocol      prot;
  90         enum mlx4_steer_type    steer;
  91         u64                     reg_id;
  92 };
  93
  94 enum res_qp_states {
  95         RES_QP_BUSY = RES_ANY_BUSY,
  96
  97         /* QP number was allocated */
  98         RES_QP_RESERVED,
  99
 100         /* ICM memory for QP context was mapped */
 101         RES_QP_MAPPED,
 102
 103         /* QP is in hw ownership */
 104         RES_QP_HW
 105 };
 106
 107 struct res_qp {
 108         struct res_common       com;
 109         struct res_mtt         *mtt;
 110         struct res_cq          *rcq;
 111         struct res_cq          *scq;
 112         struct res_srq         *srq;
 113         struct list_head        mcg_list;
 114         spinlock_t              mcg_spl;
 115         int                     local_qpn;
 116         atomic_t                ref_count;
 117         u32                     qpc_flags;
 118         /* saved qp params before VST enforcement in order to restore on VGT */
 119         u8                      sched_queue;
 120         __be32                  param3;
 121         u8                      vlan_control;
 122         u8                      fvl_rx;
 123         u8                      pri_path_fl;
 124         u8                      vlan_index;
 125         u8                      feup;
 126 };
 127
 128 enum res_mtt_states {
 129         RES_MTT_BUSY = RES_ANY_BUSY,
 130         RES_MTT_ALLOCATED,
 131 };
 132
 133 static inline const char *mtt_states_str(enum res_mtt_states state)
 134 {
 135         switch (state) {
 136         case RES_MTT_BUSY: return "RES_MTT_BUSY";
 137         case RES_MTT_ALLOCATED: return "RES_MTT_ALLOCATED";
 138         default: return "Unknown";
 139         }
 140 }
 141
 142 struct res_mtt {
 143         struct res_common       com;
 144         int                     order;
 145         atomic_t                ref_count;
 146 };
 147
 148 enum res_mpt_states {
 149         RES_MPT_BUSY = RES_ANY_BUSY,
 150         RES_MPT_RESERVED,
 151         RES_MPT_MAPPED,
 152         RES_MPT_HW,
 153 };
 154
 155 struct res_mpt {
 156         struct res_common       com;
 157         struct res_mtt         *mtt;
 158         int                     key;
 159 };
 160
 161 enum res_eq_states {
 162         RES_EQ_BUSY = RES_ANY_BUSY,
 163         RES_EQ_RESERVED,
 164         RES_EQ_HW,
 165 };
 166
 167 struct res_eq {
 168         struct res_common       com;
 169         struct res_mtt         *mtt;
 170 };
 171
 172 enum res_cq_states {
 173         RES_CQ_BUSY = RES_ANY_BUSY,
 174         RES_CQ_ALLOCATED,
 175         RES_CQ_HW,
 176 };
 177
 178 struct res_cq {
 179         struct res_common       com;
 180         struct res_mtt         *mtt;
 181         atomic_t                ref_count;
 182 };
 183
 184 enum res_srq_states {
 185         RES_SRQ_BUSY = RES_ANY_BUSY,
 186         RES_SRQ_ALLOCATED,
 187         RES_SRQ_HW,
 188 };
 189
 190 struct res_srq {
 191         struct res_common       com;
 192         struct res_mtt         *mtt;
 193         struct res_cq          *cq;
 194         atomic_t                ref_count;
 195 };
 196
 197 enum res_counter_states {
 198         RES_COUNTER_BUSY = RES_ANY_BUSY,
 199         RES_COUNTER_ALLOCATED,
 200 };
 201
 202 struct res_counter {
 203         struct res_common       com;
 204         int                     port;
 205 };
 206
 207 enum res_xrcdn_states {
 208         RES_XRCD_BUSY = RES_ANY_BUSY,
 209         RES_XRCD_ALLOCATED,
 210 };
 211
 212 struct res_xrcdn {
 213         struct res_common       com;
 214         int                     port;
 215 };
 216
 217 enum res_fs_rule_states {
 218         RES_FS_RULE_BUSY = RES_ANY_BUSY,
 219         RES_FS_RULE_ALLOCATED,
 220 };
 221
 222 struct res_fs_rule {
 223         struct res_common       com;
 224         int                     qpn;
 225         /* VF DMFS mbox with port flipped */
 226         void                    *mirr_mbox;
 227         /* > 0 --> apply mirror when getting into HA mode      */
 228         /* = 0 --> un-apply mirror when getting out of HA mode */
 229         u32                     mirr_mbox_size;
 230         struct list_head        mirr_list;
 231         u64                     mirr_rule_id;
 232 };
 233
 234 static void *res_tracker_lookup(struct rb_root *root, u64 res_id)
 235 {
 236         struct rb_node *node = root->rb_node;
 237
 238         while (node) {
 239                 struct res_common *res = container_of(node, struct res_common,
 240                                                       node);
 241
 242                 if (res_id < res->res_id)
 243                         node = node->rb_left;
 244                 else if (res_id > res->res_id)
 245                         node = node->rb_right;
 246                 else
 247                         return res;
 248         }
 249         return NULL;
 250 }
 251
 252 static int res_tracker_insert(struct rb_root *root, struct res_common *res)
 253 {
 254         struct rb_node **new = &(root->rb_node), *parent = NULL;
 255
 256         /* Figure out where to put new node */
 257         while (*new) {
 258                 struct res_common *this = container_of(*new, struct res_common,
 259                                                        node);
 260
 261                 parent = *new;
 262                 if (res->res_id < this->res_id)
 263                         new = &((*new)->rb_left);
 264                 else if (res->res_id > this->res_id)
 265                         new = &((*new)->rb_right);
 266                 else
 267                         return -EEXIST;
 268         }
 269
 270         /* Add new node and rebalance tree. */
 271         rb_link_node(&res->node, parent, new);
 272         rb_insert_color(&res->node, root);
 273
 274         return 0;
 275 }
 276
 277 enum qp_transition {
 278         QP_TRANS_INIT2RTR,
 279         QP_TRANS_RTR2RTS,
 280         QP_TRANS_RTS2RTS,
 281         QP_TRANS_SQERR2RTS,
 282         QP_TRANS_SQD2SQD,
 283         QP_TRANS_SQD2RTS
 284 };
 285
 286 /* For Debug uses */
 287 static const char *resource_str(enum mlx4_resource rt)
 288 {
 289         switch (rt) {
 290         case RES_QP: return "RES_QP";
 291         case RES_CQ: return "RES_CQ";
 292         case RES_SRQ: return "RES_SRQ";
 293         case RES_MPT: return "RES_MPT";
 294         case RES_MTT: return "RES_MTT";
 295         case RES_MAC: return  "RES_MAC";
 296         case RES_VLAN: return  "RES_VLAN";
 297         case RES_EQ: return "RES_EQ";
 298         case RES_COUNTER: return "RES_COUNTER";
 299         case RES_FS_RULE: return "RES_FS_RULE";
 300         case RES_XRCD: return "RES_XRCD";
 301         default: return "Unknown resource type !!!";
 302         };
 303 }
 304
 305 static void rem_slave_vlans(struct mlx4_dev *dev, int slave);
 306 static inline int mlx4_grant_resource(struct mlx4_dev *dev, int slave,
 307                                       enum mlx4_resource res_type, int count,
 308                                       int port)
 309 {
 310         struct mlx4_priv *priv = mlx4_priv(dev);
 311         struct resource_allocator *res_alloc =
 312                 &priv->mfunc.master.res_tracker.res_alloc[res_type];
 313         int err = -EINVAL;
 314         int allocated, free, reserved, guaranteed, from_free;
 315         int from_rsvd;
 316
 317         if (slave > dev->persist->num_vfs)
 318                 return -EINVAL;
 319
 320         spin_lock(&res_alloc->alloc_lock);
 321         allocated = (port > 0) ?
 322                 res_alloc->allocated[(port - 1) *
 323                 (dev->persist->num_vfs + 1) + slave] :
 324                 res_alloc->allocated[slave];
 325         free = (port > 0) ? res_alloc->res_port_free[port - 1] :
 326                 res_alloc->res_free;
 327         reserved = (port > 0) ? res_alloc->res_port_rsvd[port - 1] :
 328                 res_alloc->res_reserved;
 329         guaranteed = res_alloc->guaranteed[slave];
 330
 331         if (allocated + count > res_alloc->quota[slave]) {
 332                 mlx4_warn(dev, "VF %d port %d res %s: quota exceeded, count %d alloc %d quota %d\n",
 333                           slave, port, resource_str(res_type), count,
 334                           allocated, res_alloc->quota[slave]);
 335                 goto out;
 336         }
 337
 338         if (allocated + count <= guaranteed) {
 339                 err = 0;
 340                 from_rsvd = count;
 341         } else {
 342                 /* portion may need to be obtained from free area */
 343                 if (guaranteed - allocated > 0)
 344                         from_free = count - (guaranteed - allocated);
 345                 else
 346                         from_free = count;
 347
 348                 from_rsvd = count - from_free;
 349
 350                 if (free - from_free >= reserved)
 351                         err = 0;
 352                 else
 353                         mlx4_warn(dev, "VF %d port %d res %s: free pool empty, free %d from_free %d rsvd %d\n",
 354                                   slave, port, resource_str(res_type), free,
 355                                   from_free, reserved);
 356         }
 357
 358         if (!err) {
 359                 /* grant the request */
 360                 if (port > 0) {
 361                         res_alloc->allocated[(port - 1) *
 362                         (dev->persist->num_vfs + 1) + slave] += count;
 363                         res_alloc->res_port_free[port - 1] -= count;
 364                         res_alloc->res_port_rsvd[port - 1] -= from_rsvd;
 365                 } else {
 366                         res_alloc->allocated[slave] += count;
 367                         res_alloc->res_free -= count;
 368                         res_alloc->res_reserved -= from_rsvd;
 369                 }
 370         }
 371
 372 out:
 373         spin_unlock(&res_alloc->alloc_lock);
 374         return err;
 375 }
 376
 377 static inline void mlx4_release_resource(struct mlx4_dev *dev, int slave,
 378                                     enum mlx4_resource res_type, int count,
 379                                     int port)
 380 {
 381         struct mlx4_priv *priv = mlx4_priv(dev);
 382         struct resource_allocator *res_alloc =
 383                 &priv->mfunc.master.res_tracker.res_alloc[res_type];
 384         int allocated, guaranteed, from_rsvd;
 385
 386         if (slave > dev->persist->num_vfs)
 387                 return;
 388
 389         spin_lock(&res_alloc->alloc_lock);
 390
 391         allocated = (port > 0) ?
 392                 res_alloc->allocated[(port - 1) *
 393                 (dev->persist->num_vfs + 1) + slave] :
 394                 res_alloc->allocated[slave];
 395         guaranteed = res_alloc->guaranteed[slave];
 396
 397         if (allocated - count >= guaranteed) {
 398                 from_rsvd = 0;
 399         } else {
 400                 /* portion may need to be returned to reserved area */
 401                 if (allocated - guaranteed > 0)
 402                         from_rsvd = count - (allocated - guaranteed);
 403                 else
 404                         from_rsvd = count;
 405         }
 406
 407         if (port > 0) {
 408                 res_alloc->allocated[(port - 1) *
 409                 (dev->persist->num_vfs + 1) + slave] -= count;
 410                 res_alloc->res_port_free[port - 1] += count;
 411                 res_alloc->res_port_rsvd[port - 1] += from_rsvd;
 412         } else {
 413                 res_alloc->allocated[slave] -= count;
 414                 res_alloc->res_free += count;
 415                 res_alloc->res_reserved += from_rsvd;
 416         }
 417
 418         spin_unlock(&res_alloc->alloc_lock);
 419         return;
 420 }
 421
 422 static inline void initialize_res_quotas(struct mlx4_dev *dev,
 423                                          struct resource_allocator *res_alloc,
 424                                          enum mlx4_resource res_type,
 425                                          int vf, int num_instances)
 426 {
 427         res_alloc->guaranteed[vf] = num_instances /
 428                                     (2 * (dev->persist->num_vfs + 1));
 429         res_alloc->quota[vf] = (num_instances / 2) + res_alloc->guaranteed[vf];
 430         if (vf == mlx4_master_func_num(dev)) {
 431                 res_alloc->res_free = num_instances;
 432                 if (res_type == RES_MTT) {
 433                         /* reserved mtts will be taken out of the PF allocation */
 434                         res_alloc->res_free += dev->caps.reserved_mtts;
 435                         res_alloc->guaranteed[vf] += dev->caps.reserved_mtts;
 436                         res_alloc->quota[vf] += dev->caps.reserved_mtts;
 437                 }
 438         }
 439 }
 440
 441 void mlx4_init_quotas(struct mlx4_dev *dev)
 442 {
 443         struct mlx4_priv *priv = mlx4_priv(dev);
 444         int pf;
 445
 446         /* quotas for VFs are initialized in mlx4_slave_cap */
 447         if (mlx4_is_slave(dev))
 448                 return;
 449
 450         if (!mlx4_is_mfunc(dev)) {
 451                 dev->quotas.qp = dev->caps.num_qps - dev->caps.reserved_qps -
 452                         mlx4_num_reserved_sqps(dev);
 453                 dev->quotas.cq = dev->caps.num_cqs - dev->caps.reserved_cqs;
 454                 dev->quotas.srq = dev->caps.num_srqs - dev->caps.reserved_srqs;
 455                 dev->quotas.mtt = dev->caps.num_mtts - dev->caps.reserved_mtts;
 456                 dev->quotas.mpt = dev->caps.num_mpts - dev->caps.reserved_mrws;
 457                 return;
 458         }
 459
 460         pf = mlx4_master_func_num(dev);
 461         dev->quotas.qp =
 462                 priv->mfunc.master.res_tracker.res_alloc[RES_QP].quota[pf];
 463         dev->quotas.cq =
 464                 priv->mfunc.master.res_tracker.res_alloc[RES_CQ].quota[pf];
 465         dev->quotas.srq =
 466                 priv->mfunc.master.res_tracker.res_alloc[RES_SRQ].quota[pf];
 467         dev->quotas.mtt =
 468                 priv->mfunc.master.res_tracker.res_alloc[RES_MTT].quota[pf];
 469         dev->quotas.mpt =
 470                 priv->mfunc.master.res_tracker.res_alloc[RES_MPT].quota[pf];
 471 }
 472
 473 static int get_max_gauranteed_vfs_counter(struct mlx4_dev *dev)
 474 {
 475         /* reduce the sink counter */
 476         return (dev->caps.max_counters - 1 -
 477                 (MLX4_PF_COUNTERS_PER_PORT * MLX4_MAX_PORTS))
 478                 / MLX4_MAX_PORTS;
 479 }
 480
 481 int mlx4_init_resource_tracker(struct mlx4_dev *dev)
 482 {
 483         struct mlx4_priv *priv = mlx4_priv(dev);
 484         int i, j;
 485         int t;
 486         int max_vfs_guarantee_counter = get_max_gauranteed_vfs_counter(dev);
 487
 488         priv->mfunc.master.res_tracker.slave_list =
 489                 kzalloc(dev->num_slaves * sizeof(struct slave_list),
 490                         GFP_KERNEL);
 491         if (!priv->mfunc.master.res_tracker.slave_list)
 492                 return -ENOMEM;
 493
 494         for (i = 0 ; i < dev->num_slaves; i++) {
 495                 for (t = 0; t < MLX4_NUM_OF_RESOURCE_TYPE; ++t)
 496                         INIT_LIST_HEAD(&priv->mfunc.master.res_tracker.
 497                                        slave_list[i].res_list[t]);
 498                 mutex_init(&priv->mfunc.master.res_tracker.slave_list[i].mutex);
 499         }
 500
 501         mlx4_dbg(dev, "Started init_resource_tracker: %ld slaves\n",
 502                  dev->num_slaves);
 503         for (i = 0 ; i < MLX4_NUM_OF_RESOURCE_TYPE; i++)
 504                 priv->mfunc.master.res_tracker.res_tree[i] = RB_ROOT;
 505
 506         for (i = 0; i < MLX4_NUM_OF_RESOURCE_TYPE; i++) {
 507                 struct resource_allocator *res_alloc =
 508                         &priv->mfunc.master.res_tracker.res_alloc[i];
 509                 res_alloc->quota = kmalloc((dev->persist->num_vfs + 1) *
 510                                            sizeof(int), GFP_KERNEL);
 511                 res_alloc->guaranteed = kmalloc((dev->persist->num_vfs + 1) *
 512                                                 sizeof(int), GFP_KERNEL);
 513                 if (i == RES_MAC || i == RES_VLAN)
 514                         res_alloc->allocated = kzalloc(MLX4_MAX_PORTS *
 515                                                        (dev->persist->num_vfs
 516                                                        + 1) *
 517                                                        sizeof(int), GFP_KERNEL);
 518                 else
 519                         res_alloc->allocated = kzalloc((dev->persist->
 520                                                         num_vfs + 1) *
 521                                                        sizeof(int), GFP_KERNEL);
 522                 /* Reduce the sink counter */
 523                 if (i == RES_COUNTER)
 524                         res_alloc->res_free = dev->caps.max_counters - 1;
 525
 526                 if (!res_alloc->quota || !res_alloc->guaranteed ||
 527                     !res_alloc->allocated)
 528                         goto no_mem_err;
 529
 530                 spin_lock_init(&res_alloc->alloc_lock);
 531                 for (t = 0; t < dev->persist->num_vfs + 1; t++) {
 532                         struct mlx4_active_ports actv_ports =
 533                                 mlx4_get_active_ports(dev, t);
 534                         switch (i) {
 535                         case RES_QP:
 536                                 initialize_res_quotas(dev, res_alloc, RES_QP,
 537                                                       t, dev->caps.num_qps -
 538                                                       dev->caps.reserved_qps -
 539                                                       mlx4_num_reserved_sqps(dev));
 540                                 break;
 541                         case RES_CQ:
 542                                 initialize_res_quotas(dev, res_alloc, RES_CQ,
 543                                                       t, dev->caps.num_cqs -
 544                                                       dev->caps.reserved_cqs);
 545                                 break;
 546                         case RES_SRQ:
 547                                 initialize_res_quotas(dev, res_alloc, RES_SRQ,
 548                                                       t, dev->caps.num_srqs -
 549                                                       dev->caps.reserved_srqs);
 550                                 break;
 551                         case RES_MPT:
 552                                 initialize_res_quotas(dev, res_alloc, RES_MPT,
 553                                                       t, dev->caps.num_mpts -
 554                                                       dev->caps.reserved_mrws);
 555                                 break;
 556                         case RES_MTT:
 557                                 initialize_res_quotas(dev, res_alloc, RES_MTT,
 558                                                       t, dev->caps.num_mtts -
 559                                                       dev->caps.reserved_mtts);
 560                                 break;
 561                         case RES_MAC:
 562                                 if (t == mlx4_master_func_num(dev)) {
 563                                         int max_vfs_pport = 0;
 564                                         /* Calculate the max vfs per port for */
 565                                         /* both ports.                        */
 566                                         for (j = 0; j < dev->caps.num_ports;
 567                                              j++) {
 568                                                 struct mlx4_slaves_pport slaves_pport =
 569                                                         mlx4_phys_to_slaves_pport(dev, j + 1);
 570                                                 unsigned current_slaves =
 571                                                         bitmap_weight(slaves_pport.slaves,
 572                                                                       dev->caps.num_ports) - 1;
 573                                                 if (max_vfs_pport < current_slaves)
 574                                                         max_vfs_pport =
 575                                                                 current_slaves;
 576                                         }
 577                                         res_alloc->quota[t] =
 578                                                 MLX4_MAX_MAC_NUM -
 579                                                 2 * max_vfs_pport;
 580                                         res_alloc->guaranteed[t] = 2;
 581                                         for (j = 0; j < MLX4_MAX_PORTS; j++)
 582                                                 res_alloc->res_port_free[j] =
 583                                                         MLX4_MAX_MAC_NUM;
 584                                 } else {
 585                                         res_alloc->quota[t] = MLX4_MAX_MAC_NUM;
 586                                         res_alloc->guaranteed[t] = 2;
 587                                 }
 588                                 break;
 589                         case RES_VLAN:
 590                                 if (t == mlx4_master_func_num(dev)) {
 591                                         res_alloc->quota[t] = MLX4_MAX_VLAN_NUM;
 592                                         res_alloc->guaranteed[t] = MLX4_MAX_VLAN_NUM / 2;
 593                                         for (j = 0; j < MLX4_MAX_PORTS; j++)
 594                                                 res_alloc->res_port_free[j] =
 595                                                         res_alloc->quota[t];
 596                                 } else {
 597                                         res_alloc->quota[t] = MLX4_MAX_VLAN_NUM / 2;
 598                                         res_alloc->guaranteed[t] = 0;
 599                                 }
 600                                 break;
 601                         case RES_COUNTER:
 602                                 res_alloc->quota[t] = dev->caps.max_counters;
 603                                 if (t == mlx4_master_func_num(dev))
 604                                         res_alloc->guaranteed[t] =
 605                                                 MLX4_PF_COUNTERS_PER_PORT *
 606                                                 MLX4_MAX_PORTS;
 607                                 else if (t <= max_vfs_guarantee_counter)
 608                                         res_alloc->guaranteed[t] =
 609                                                 MLX4_VF_COUNTERS_PER_PORT *
 610                                                 MLX4_MAX_PORTS;
 611                                 else
 612                                         res_alloc->guaranteed[t] = 0;
 613                                 res_alloc->res_free -= res_alloc->guaranteed[t];
 614                                 break;
 615                         default:
 616                                 break;
 617                         }
 618                         if (i == RES_MAC || i == RES_VLAN) {
 619                                 for (j = 0; j < dev->caps.num_ports; j++)
 620                                         if (test_bit(j, actv_ports.ports))
 621                                                 res_alloc->res_port_rsvd[j] +=
 622                                                         res_alloc->guaranteed[t];
 623                         } else {
 624                                 res_alloc->res_reserved += res_alloc->guaranteed[t];
 625                         }
 626                 }
 627         }
 628         spin_lock_init(&priv->mfunc.master.res_tracker.lock);
 629         return 0;
 630
 631 no_mem_err:
 632         for (i = 0; i < MLX4_NUM_OF_RESOURCE_TYPE; i++) {
 633                 kfree(priv->mfunc.master.res_tracker.res_alloc[i].allocated);
 634                 priv->mfunc.master.res_tracker.res_alloc[i].allocated = NULL;
 635                 kfree(priv->mfunc.master.res_tracker.res_alloc[i].guaranteed);
 636                 priv->mfunc.master.res_tracker.res_alloc[i].guaranteed = NULL;
 637                 kfree(priv->mfunc.master.res_tracker.res_alloc[i].quota);
 638                 priv->mfunc.master.res_tracker.res_alloc[i].quota = NULL;
 639         }
 640         return -ENOMEM;
 641 }
 642
 643 void mlx4_free_resource_tracker(struct mlx4_dev *dev,
 644                                 enum mlx4_res_tracker_free_type type)
 645 {
 646         struct mlx4_priv *priv = mlx4_priv(dev);
 647         int i;
 648
 649         if (priv->mfunc.master.res_tracker.slave_list) {
 650                 if (type != RES_TR_FREE_STRUCTS_ONLY) {
 651                         for (i = 0; i < dev->num_slaves; i++) {
 652                                 if (type == RES_TR_FREE_ALL ||
 653                                     dev->caps.function != i)
 654                                         mlx4_delete_all_resources_for_slave(dev, i);
 655                         }
 656                         /* free master's vlans */
 657                         i = dev->caps.function;
 658                         mlx4_reset_roce_gids(dev, i);
 659                         mutex_lock(&priv->mfunc.master.res_tracker.slave_list[i].mutex);
 660                         rem_slave_vlans(dev, i);
 661                         mutex_unlock(&priv->mfunc.master.res_tracker.slave_list[i].mutex);
 662                 }
 663
 664                 if (type != RES_TR_FREE_SLAVES_ONLY) {
 665                         for (i = 0; i < MLX4_NUM_OF_RESOURCE_TYPE; i++) {
 666                                 kfree(priv->mfunc.master.res_tracker.res_alloc[i].allocated);
 667                                 priv->mfunc.master.res_tracker.res_alloc[i].allocated = NULL;
 668                                 kfree(priv->mfunc.master.res_tracker.res_alloc[i].guaranteed);
 669                                 priv->mfunc.master.res_tracker.res_alloc[i].guaranteed = NULL;
 670                                 kfree(priv->mfunc.master.res_tracker.res_alloc[i].quota);
 671                                 priv->mfunc.master.res_tracker.res_alloc[i].quota = NULL;
 672                         }
 673                         kfree(priv->mfunc.master.res_tracker.slave_list);
 674                         priv->mfunc.master.res_tracker.slave_list = NULL;
 675                 }
 676         }
 677 }
 678
 679 static void update_pkey_index(struct mlx4_dev *dev, int slave,
 680                               struct mlx4_cmd_mailbox *inbox)
 681 {
 682         u8 sched = *(u8 *)(inbox->buf + 64);
 683         u8 orig_index = *(u8 *)(inbox->buf + 35);
 684         u8 new_index;
 685         struct mlx4_priv *priv = mlx4_priv(dev);
 686         int port;
 687
 688         port = (sched >> 6 & 1) + 1;
 689
 690         new_index = priv->virt2phys_pkey[slave][port - 1][orig_index];
 691         *(u8 *)(inbox->buf + 35) = new_index;
 692 }
 693
 694 static void update_gid(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *inbox,
 695                        u8 slave)
 696 {
 697         struct mlx4_qp_context  *qp_ctx = inbox->buf + 8;
 698         enum mlx4_qp_optpar     optpar = be32_to_cpu(*(__be32 *) inbox->buf);
 699         u32                     ts = (be32_to_cpu(qp_ctx->flags) >> 16) & 0xff;
 700         int port;
 701
 702         if (MLX4_QP_ST_UD == ts) {
 703                 port = (qp_ctx->pri_path.sched_queue >> 6 & 1) + 1;
 704                 if (mlx4_is_eth(dev, port))
 705                         qp_ctx->pri_path.mgid_index =
 706                                 mlx4_get_base_gid_ix(dev, slave, port) | 0x80;
 707                 else
 708                         qp_ctx->pri_path.mgid_index = slave | 0x80;
 709
 710         } else if (MLX4_QP_ST_RC == ts || MLX4_QP_ST_XRC == ts || MLX4_QP_ST_UC == ts) {
 711                 if (optpar & MLX4_QP_OPTPAR_PRIMARY_ADDR_PATH) {
 712                         port = (qp_ctx->pri_path.sched_queue >> 6 & 1) + 1;
 713                         if (mlx4_is_eth(dev, port)) {
 714                                 qp_ctx->pri_path.mgid_index +=
 715                                         mlx4_get_base_gid_ix(dev, slave, port);
 716                                 qp_ctx->pri_path.mgid_index &= 0x7f;
 717                         } else {
 718                                 qp_ctx->pri_path.mgid_index = slave & 0x7F;
 719                         }
 720                 }
 721                 if (optpar & MLX4_QP_OPTPAR_ALT_ADDR_PATH) {
 722                         port = (qp_ctx->alt_path.sched_queue >> 6 & 1) + 1;
 723                         if (mlx4_is_eth(dev, port)) {
 724                                 qp_ctx->alt_path.mgid_index +=
 725                                         mlx4_get_base_gid_ix(dev, slave, port);
 726                                 qp_ctx->alt_path.mgid_index &= 0x7f;
 727                         } else {
 728                                 qp_ctx->alt_path.mgid_index = slave & 0x7F;
 729                         }
 730                 }
 731         }
 732 }
 733
 734 static int handle_counter(struct mlx4_dev *dev, struct mlx4_qp_context *qpc,
 735                           u8 slave, int port);
 736
 737 static int update_vport_qp_param(struct mlx4_dev *dev,
 738                                  struct mlx4_cmd_mailbox *inbox,
 739                                  u8 slave, u32 qpn)
 740 {
 741         struct mlx4_qp_context  *qpc = inbox->buf + 8;
 742         struct mlx4_vport_oper_state *vp_oper;
 743         struct mlx4_priv *priv;
 744         u32 qp_type;
 745         int port, err = 0;
 746
 747         port = (qpc->pri_path.sched_queue & 0x40) ? 2 : 1;
 748         priv = mlx4_priv(dev);
 749         vp_oper = &priv->mfunc.master.vf_oper[slave].vport[port];
 750         qp_type = (be32_to_cpu(qpc->flags) >> 16) & 0xff;
 751
 752         err = handle_counter(dev, qpc, slave, port);
 753         if (err)
 754                 goto out;
 755
 756         if (MLX4_VGT != vp_oper->state.default_vlan) {
 757                 /* the reserved QPs (special, proxy, tunnel)
 758                  * do not operate over vlans
 759                  */
 760                 if (mlx4_is_qp_reserved(dev, qpn))
 761                         return 0;
 762
 763                 /* force strip vlan by clear vsd, MLX QP refers to Raw Ethernet */
 764                 if (qp_type == MLX4_QP_ST_UD ||
 765                     (qp_type == MLX4_QP_ST_MLX && mlx4_is_eth(dev, port))) {
 766                         if (dev->caps.bmme_flags & MLX4_BMME_FLAG_VSD_INIT2RTR) {
 767                                 *(__be32 *)inbox->buf =
 768                                         cpu_to_be32(be32_to_cpu(*(__be32 *)inbox->buf) |
 769                                         MLX4_QP_OPTPAR_VLAN_STRIPPING);
 770                                 qpc->param3 &= ~cpu_to_be32(MLX4_STRIP_VLAN);
 771                         } else {
 772                                 struct mlx4_update_qp_params params = {.flags = 0};
 773
 774                                 err = mlx4_update_qp(dev, qpn, MLX4_UPDATE_QP_VSD, &params);
 775                                 if (err)
 776                                         goto out;
 777                         }
 778                 }
 779
 780                 /* preserve IF_COUNTER flag */
 781                 qpc->pri_path.vlan_control &=
 782                         MLX4_CTRL_ETH_SRC_CHECK_IF_COUNTER;
 783                 if (vp_oper->state.link_state == IFLA_VF_LINK_STATE_DISABLE &&
 784                     dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_UPDATE_QP) {
 785                         qpc->pri_path.vlan_control |=
 786                                 MLX4_VLAN_CTRL_ETH_TX_BLOCK_TAGGED |
 787                                 MLX4_VLAN_CTRL_ETH_TX_BLOCK_PRIO_TAGGED |
 788                                 MLX4_VLAN_CTRL_ETH_TX_BLOCK_UNTAGGED |
 789                                 MLX4_VLAN_CTRL_ETH_RX_BLOCK_PRIO_TAGGED |
 790                                 MLX4_VLAN_CTRL_ETH_RX_BLOCK_UNTAGGED |
 791                                 MLX4_VLAN_CTRL_ETH_RX_BLOCK_TAGGED;
 792                 } else if (0 != vp_oper->state.default_vlan) {
 793                         if (vp_oper->state.vlan_proto == htons(ETH_P_8021AD)) {
 794                                 /* vst QinQ should block untagged on TX,
 795                                  * but cvlan is in payload and phv is set so
 796                                  * hw see it as untagged. Block tagged instead.
 797                                  */
 798                                 qpc->pri_path.vlan_control |=
 799                                         MLX4_VLAN_CTRL_ETH_TX_BLOCK_PRIO_TAGGED |
 800                                         MLX4_VLAN_CTRL_ETH_TX_BLOCK_TAGGED |
 801                                         MLX4_VLAN_CTRL_ETH_RX_BLOCK_PRIO_TAGGED |
 802                                         MLX4_VLAN_CTRL_ETH_RX_BLOCK_UNTAGGED;
 803                         } else { /* vst 802.1Q */
 804                                 qpc->pri_path.vlan_control |=
 805                                         MLX4_VLAN_CTRL_ETH_TX_BLOCK_TAGGED |
 806                                         MLX4_VLAN_CTRL_ETH_RX_BLOCK_PRIO_TAGGED |
 807                                         MLX4_VLAN_CTRL_ETH_RX_BLOCK_UNTAGGED;
 808                         }
 809                 } else { /* priority tagged */
 810                         qpc->pri_path.vlan_control |=
 811                                 MLX4_VLAN_CTRL_ETH_TX_BLOCK_TAGGED |
 812                                 MLX4_VLAN_CTRL_ETH_RX_BLOCK_TAGGED;
 813                 }
 814
 815                 qpc->pri_path.fvl_rx |= MLX4_FVL_RX_FORCE_ETH_VLAN;
 816                 qpc->pri_path.vlan_index = vp_oper->vlan_idx;
 817                 qpc->pri_path.fl |= MLX4_FL_ETH_HIDE_CQE_VLAN;
 818                 if (vp_oper->state.vlan_proto == htons(ETH_P_8021AD))
 819                         qpc->pri_path.fl |= MLX4_FL_SV;
 820                 else
 821                         qpc->pri_path.fl |= MLX4_FL_CV;
 822                 qpc->pri_path.feup |= MLX4_FEUP_FORCE_ETH_UP | MLX4_FVL_FORCE_ETH_VLAN;
 823                 qpc->pri_path.sched_queue &= 0xC7;
 824                 qpc->pri_path.sched_queue |= (vp_oper->state.default_qos) << 3;
 825                 qpc->qos_vport = vp_oper->state.qos_vport;
 826         }
 827         if (vp_oper->state.spoofchk) {
 828                 qpc->pri_path.feup |= MLX4_FSM_FORCE_ETH_SRC_MAC;
 829                 qpc->pri_path.grh_mylmc = (0x80 & qpc->pri_path.grh_mylmc) + vp_oper->mac_idx;
 830         }
 831 out:
 832         return err;
 833 }
 834
 835 static int mpt_mask(struct mlx4_dev *dev)
 836 {
 837         return dev->caps.num_mpts - 1;
 838 }
 839
 840 static void *find_res(struct mlx4_dev *dev, u64 res_id,
 841                       enum mlx4_resource type)
 842 {
 843         struct mlx4_priv *priv = mlx4_priv(dev);
 844
 845         return res_tracker_lookup(&priv->mfunc.master.res_tracker.res_tree[type],
 846                                   res_id);
 847 }
 848
 849 static int get_res(struct mlx4_dev *dev, int slave, u64 res_id,
 850                    enum mlx4_resource type,
 851                    void *res)
 852 {
 853         struct res_common *r;
 854         int err = 0;
 855
 856         spin_lock_irq(mlx4_tlock(dev));
 857         r = find_res(dev, res_id, type);
 858         if (!r) {
 859                 err = -ENONET;
 860                 goto exit;
 861         }
 862
 863         if (r->state == RES_ANY_BUSY) {
 864                 err = -EBUSY;
 865                 goto exit;
 866         }
 867
 868         if (r->owner != slave) {
 869                 err = -EPERM;
 870                 goto exit;
 871         }
 872
 873         r->from_state = r->state;
 874         r->state = RES_ANY_BUSY;
 875
 876         if (res)
 877                 *((struct res_common **)res) = r;
 878
 879 exit:
 880         spin_unlock_irq(mlx4_tlock(dev));
 881         return err;
 882 }
 883
 884 int mlx4_get_slave_from_resource_id(struct mlx4_dev *dev,
 885                                     enum mlx4_resource type,
 886                                     u64 res_id, int *slave)
 887 {
 888
 889         struct res_common *r;
 890         int err = -ENOENT;
 891         int id = res_id;
 892
 893         if (type == RES_QP)
 894                 id &= 0x7fffff;
 895         spin_lock(mlx4_tlock(dev));
 896
 897         r = find_res(dev, id, type);
 898         if (r) {
 899                 *slave = r->owner;
 900                 err = 0;
 901         }
 902         spin_unlock(mlx4_tlock(dev));
 903
 904         return err;
 905 }
 906
 907 static void put_res(struct mlx4_dev *dev, int slave, u64 res_id,
 908                     enum mlx4_resource type)
 909 {
 910         struct res_common *r;
 911
 912         spin_lock_irq(mlx4_tlock(dev));
 913         r = find_res(dev, res_id, type);
 914         if (r)
 915                 r->state = r->from_state;
 916         spin_unlock_irq(mlx4_tlock(dev));
 917 }
 918
 919 static int counter_alloc_res(struct mlx4_dev *dev, int slave, int op, int cmd,
 920                              u64 in_param, u64 *out_param, int port);
 921
 922 static int handle_existing_counter(struct mlx4_dev *dev, u8 slave, int port,
 923                                    int counter_index)
 924 {
 925         struct res_common *r;
 926         struct res_counter *counter;
 927         int ret = 0;
 928
 929         if (counter_index == MLX4_SINK_COUNTER_INDEX(dev))
 930                 return ret;
 931
 932         spin_lock_irq(mlx4_tlock(dev));
 933         r = find_res(dev, counter_index, RES_COUNTER);
 934         if (!r || r->owner != slave) {
 935                 ret = -EINVAL;
 936         } else {
 937                 counter = container_of(r, struct res_counter, com);
 938                 if (!counter->port)
 939                         counter->port = port;
 940         }
 941
 942         spin_unlock_irq(mlx4_tlock(dev));
 943         return ret;
 944 }
 945
 946 static int handle_unexisting_counter(struct mlx4_dev *dev,
 947                                      struct mlx4_qp_context *qpc, u8 slave,
 948                                      int port)
 949 {
 950         struct mlx4_priv *priv = mlx4_priv(dev);
 951         struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker;
 952         struct res_common *tmp;
 953         struct res_counter *counter;
 954         u64 counter_idx = MLX4_SINK_COUNTER_INDEX(dev);
 955         int err = 0;
 956
 957         spin_lock_irq(mlx4_tlock(dev));
 958         list_for_each_entry(tmp,
 959                             &tracker->slave_list[slave].res_list[RES_COUNTER],
 960                             list) {
 961                 counter = container_of(tmp, struct res_counter, com);
 962                 if (port == counter->port) {
 963                         qpc->pri_path.counter_index  = counter->com.res_id;
 964                         spin_unlock_irq(mlx4_tlock(dev));
 965                         return 0;
 966                 }
 967         }
 968         spin_unlock_irq(mlx4_tlock(dev));
 969
 970         /* No existing counter, need to allocate a new counter */
 971         err = counter_alloc_res(dev, slave, RES_OP_RESERVE, 0, 0, &counter_idx,
 972                                 port);
 973         if (err == -ENOENT) {
 974                 err = 0;
 975         } else if (err && err != -ENOSPC) {
 976                 mlx4_err(dev, "%s: failed to create new counter for slave %d err %d\n",
 977                          __func__, slave, err);
 978         } else {
 979                 qpc->pri_path.counter_index = counter_idx;
 980                 mlx4_dbg(dev, "%s: alloc new counter for slave %d index %d\n",
 981                          __func__, slave, qpc->pri_path.counter_index);
 982                 err = 0;
 983         }
 984
 985         return err;
 986 }
 987
 988 static int handle_counter(struct mlx4_dev *dev, struct mlx4_qp_context *qpc,
 989                           u8 slave, int port)
 990 {
 991         if (qpc->pri_path.counter_index != MLX4_SINK_COUNTER_INDEX(dev))
 992                 return handle_existing_counter(dev, slave, port,
 993                                                qpc->pri_path.counter_index);
 994
 995         return handle_unexisting_counter(dev, qpc, slave, port);
 996 }
 997
 998 static struct res_common *alloc_qp_tr(int id)
 999 {
1000         struct res_qp *ret;
1001
1002         ret = kzalloc(sizeof *ret, GFP_KERNEL);
1003         if (!ret)
1004                 return NULL;
1005
1006         ret->com.res_id = id;
1007         ret->com.state = RES_QP_RESERVED;
1008         ret->local_qpn = id;
1009         INIT_LIST_HEAD(&ret->mcg_list);
1010         spin_lock_init(&ret->mcg_spl);
1011         atomic_set(&ret->ref_count, 0);
1012
1013         return &ret->com;
1014 }
1015
1016 static struct res_common *alloc_mtt_tr(int id, int order)
1017 {
1018         struct res_mtt *ret;
1019
1020         ret = kzalloc(sizeof *ret, GFP_KERNEL);
1021         if (!ret)
1022                 return NULL;
1023
1024         ret->com.res_id = id;
1025         ret->order = order;
1026         ret->com.state = RES_MTT_ALLOCATED;
1027         atomic_set(&ret->ref_count, 0);
1028
1029         return &ret->com;
1030 }
1031
1032 static struct res_common *alloc_mpt_tr(int id, int key)
1033 {
1034         struct res_mpt *ret;
1035
1036         ret = kzalloc(sizeof *ret, GFP_KERNEL);
1037         if (!ret)
1038                 return NULL;
1039
1040         ret->com.res_id = id;
1041         ret->com.state = RES_MPT_RESERVED;
1042         ret->key = key;
1043
1044         return &ret->com;
1045 }
1046
1047 static struct res_common *alloc_eq_tr(int id)
1048 {
1049         struct res_eq *ret;
1050
1051         ret = kzalloc(sizeof *ret, GFP_KERNEL);
1052         if (!ret)
1053                 return NULL;
1054
1055         ret->com.res_id = id;
1056         ret->com.state = RES_EQ_RESERVED;
1057
1058         return &ret->com;
1059 }
1060
1061 static struct res_common *alloc_cq_tr(int id)
1062 {
1063         struct res_cq *ret;
1064
1065         ret = kzalloc(sizeof *ret, GFP_KERNEL);
1066         if (!ret)
1067                 return NULL;
1068
1069         ret->com.res_id = id;
1070         ret->com.state = RES_CQ_ALLOCATED;
1071         atomic_set(&ret->ref_count, 0);
1072
1073         return &ret->com;
1074 }
1075
1076 static struct res_common *alloc_srq_tr(int id)
1077 {
1078         struct res_srq *ret;
1079
1080         ret = kzalloc(sizeof *ret, GFP_KERNEL);
1081         if (!ret)
1082                 return NULL;
1083
1084         ret->com.res_id = id;
1085         ret->com.state = RES_SRQ_ALLOCATED;
1086         atomic_set(&ret->ref_count, 0);
1087
1088         return &ret->com;
1089 }
1090
1091 static struct res_common *alloc_counter_tr(int id, int port)
1092 {
1093         struct res_counter *ret;
1094
1095         ret = kzalloc(sizeof *ret, GFP_KERNEL);
1096         if (!ret)
1097                 return NULL;
1098
1099         ret->com.res_id = id;
1100         ret->com.state = RES_COUNTER_ALLOCATED;
1101         ret->port = port;
1102
1103         return &ret->com;
1104 }
1105
1106 static struct res_common *alloc_xrcdn_tr(int id)
1107 {
1108         struct res_xrcdn *ret;
1109
1110         ret = kzalloc(sizeof *ret, GFP_KERNEL);
1111         if (!ret)
1112                 return NULL;
1113
1114         ret->com.res_id = id;
1115         ret->com.state = RES_XRCD_ALLOCATED;
1116
1117         return &ret->com;
1118 }
1119
1120 static struct res_common *alloc_fs_rule_tr(u64 id, int qpn)
1121 {
1122         struct res_fs_rule *ret;
1123
1124         ret = kzalloc(sizeof *ret, GFP_KERNEL);
1125         if (!ret)
1126                 return NULL;
1127
1128         ret->com.res_id = id;
1129         ret->com.state = RES_FS_RULE_ALLOCATED;
1130         ret->qpn = qpn;
1131         return &ret->com;
1132 }
1133
1134 static struct res_common *alloc_tr(u64 id, enum mlx4_resource type, int slave,
1135                                    int extra)
1136 {
1137         struct res_common *ret;
1138
1139         switch (type) {
1140         case RES_QP:
1141                 ret = alloc_qp_tr(id);
1142                 break;
1143         case RES_MPT:
1144                 ret = alloc_mpt_tr(id, extra);
1145                 break;
1146         case RES_MTT:
1147                 ret = alloc_mtt_tr(id, extra);
1148                 break;
1149         case RES_EQ:
1150                 ret = alloc_eq_tr(id);
1151                 break;
1152         case RES_CQ:
1153                 ret = alloc_cq_tr(id);
1154                 break;
1155         case RES_SRQ:
1156                 ret = alloc_srq_tr(id);
1157                 break;
1158         case RES_MAC:
1159                 pr_err("implementation missing\n");
1160                 return NULL;
1161         case RES_COUNTER:
1162                 ret = alloc_counter_tr(id, extra);
1163                 break;
1164         case RES_XRCD:
1165                 ret = alloc_xrcdn_tr(id);
1166                 break;
1167         case RES_FS_RULE:
1168                 ret = alloc_fs_rule_tr(id, extra);
1169                 break;
1170         default:
1171                 return NULL;
1172         }
1173         if (ret)
1174                 ret->owner = slave;
1175
1176         return ret;
1177 }
1178
1179 int mlx4_calc_vf_counters(struct mlx4_dev *dev, int slave, int port,
1180                           struct mlx4_counter *data)
1181 {
1182         struct mlx4_priv *priv = mlx4_priv(dev);
1183         struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker;
1184         struct res_common *tmp;
1185         struct res_counter *counter;
1186         int *counters_arr;
1187         int i = 0, err = 0;
1188
1189         memset(data, 0, sizeof(*data));
1190
1191         counters_arr = kmalloc_array(dev->caps.max_counters,
1192                                      sizeof(*counters_arr), GFP_KERNEL);
1193         if (!counters_arr)
1194                 return -ENOMEM;
1195
1196         spin_lock_irq(mlx4_tlock(dev));
1197         list_for_each_entry(tmp,
1198                             &tracker->slave_list[slave].res_list[RES_COUNTER],
1199                             list) {
1200                 counter = container_of(tmp, struct res_counter, com);
1201                 if (counter->port == port) {
1202                         counters_arr[i] = (int)tmp->res_id;
1203                         i++;
1204                 }
1205         }
1206         spin_unlock_irq(mlx4_tlock(dev));
1207         counters_arr[i] = -1;
1208
1209         i = 0;
1210
1211         while (counters_arr[i] != -1) {
1212                 err = mlx4_get_counter_stats(dev, counters_arr[i], data,
1213                                              0);
1214                 if (err) {
1215                         memset(data, 0, sizeof(*data));
1216                         goto table_changed;
1217                 }
1218                 i++;
1219         }
1220
1221 table_changed:
1222         kfree(counters_arr);
1223         return 0;
1224 }
1225
1226 static int add_res_range(struct mlx4_dev *dev, int slave, u64 base, int count,
1227                          enum mlx4_resource type, int extra)
1228 {
1229         int i;
1230         int err;
1231         struct mlx4_priv *priv = mlx4_priv(dev);
1232         struct res_common **res_arr;
1233         struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker;
1234         struct rb_root *root = &tracker->res_tree[type];
1235
1236         res_arr = kzalloc(count * sizeof *res_arr, GFP_KERNEL);
1237         if (!res_arr)
1238                 return -ENOMEM;
1239
1240         for (i = 0; i < count; ++i) {
1241                 res_arr[i] = alloc_tr(base + i, type, slave, extra);
1242                 if (!res_arr[i]) {
1243                         for (--i; i >= 0; --i)
1244                                 kfree(res_arr[i]);
1245
1246                         kfree(res_arr);
1247                         return -ENOMEM;
1248                 }
1249         }
1250
1251         spin_lock_irq(mlx4_tlock(dev));
1252         for (i = 0; i < count; ++i) {
1253                 if (find_res(dev, base + i, type)) {
1254                         err = -EEXIST;
1255                         goto undo;
1256                 }
1257                 err = res_tracker_insert(root, res_arr[i]);
1258                 if (err)
1259                         goto undo;
1260                 list_add_tail(&res_arr[i]->list,
1261                               &tracker->slave_list[slave].res_list[type]);
1262         }
1263         spin_unlock_irq(mlx4_tlock(dev));
1264         kfree(res_arr);
1265
1266         return 0;
1267
1268 undo:
1269         for (--i; i >= 0; --i) {
1270                 rb_erase(&res_arr[i]->node, root);
1271                 list_del_init(&res_arr[i]->list);
1272         }
1273
1274         spin_unlock_irq(mlx4_tlock(dev));
1275
1276         for (i = 0; i < count; ++i)
1277                 kfree(res_arr[i]);
1278
1279         kfree(res_arr);
1280
1281         return err;
1282 }
1283
1284 static int remove_qp_ok(struct res_qp *res)
1285 {
1286         if (res->com.state == RES_QP_BUSY || atomic_read(&res->ref_count) ||
1287             !list_empty(&res->mcg_list)) {
1288                 pr_err("resource tracker: fail to remove qp, state %d, ref_count %d\n",
1289                        res->com.state, atomic_read(&res->ref_count));
1290                 return -EBUSY;
1291         } else if (res->com.state != RES_QP_RESERVED) {
1292                 return -EPERM;
1293         }
1294
1295         return 0;
1296 }
1297
1298 static int remove_mtt_ok(struct res_mtt *res, int order)
1299 {
1300         if (res->com.state == RES_MTT_BUSY ||
1301             atomic_read(&res->ref_count)) {
1302                 pr_devel("%s-%d: state %s, ref_count %d\n",
1303                          __func__, __LINE__,
1304                          mtt_states_str(res->com.state),
1305                          atomic_read(&res->ref_count));
1306                 return -EBUSY;
1307         } else if (res->com.state != RES_MTT_ALLOCATED)
1308                 return -EPERM;
1309         else if (res->order != order)
1310                 return -EINVAL;
1311
1312         return 0;
1313 }
1314
1315 static int remove_mpt_ok(struct res_mpt *res)
1316 {
1317         if (res->com.state == RES_MPT_BUSY)
1318                 return -EBUSY;
1319         else if (res->com.state != RES_MPT_RESERVED)
1320                 return -EPERM;
1321
1322         return 0;
1323 }
1324
1325 static int remove_eq_ok(struct res_eq *res)
1326 {
1327         if (res->com.state == RES_MPT_BUSY)
1328                 return -EBUSY;
1329         else if (res->com.state != RES_MPT_RESERVED)
1330                 return -EPERM;
1331
1332         return 0;
1333 }
1334
1335 static int remove_counter_ok(struct res_counter *res)
1336 {
1337         if (res->com.state == RES_COUNTER_BUSY)
1338                 return -EBUSY;
1339         else if (res->com.state != RES_COUNTER_ALLOCATED)
1340                 return -EPERM;
1341
1342         return 0;
1343 }
1344
1345 static int remove_xrcdn_ok(struct res_xrcdn *res)
1346 {
1347         if (res->com.state == RES_XRCD_BUSY)
1348                 return -EBUSY;
1349         else if (res->com.state != RES_XRCD_ALLOCATED)
1350                 return -EPERM;
1351
1352         return 0;
1353 }
1354
1355 static int remove_fs_rule_ok(struct res_fs_rule *res)
1356 {
1357         if (res->com.state == RES_FS_RULE_BUSY)
1358                 return -EBUSY;
1359         else if (res->com.state != RES_FS_RULE_ALLOCATED)
1360                 return -EPERM;
1361
1362         return 0;
1363 }
1364
1365 static int remove_cq_ok(struct res_cq *res)
1366 {
1367         if (res->com.state == RES_CQ_BUSY)
1368                 return -EBUSY;
1369         else if (res->com.state != RES_CQ_ALLOCATED)
1370                 return -EPERM;
1371
1372         return 0;
1373 }
1374
1375 static int remove_srq_ok(struct res_srq *res)
1376 {
1377         if (res->com.state == RES_SRQ_BUSY)
1378                 return -EBUSY;
1379         else if (res->com.state != RES_SRQ_ALLOCATED)
1380                 return -EPERM;
1381
1382         return 0;
1383 }
1384
1385 static int remove_ok(struct res_common *res, enum mlx4_resource type, int extra)
1386 {
1387         switch (type) {
1388         case RES_QP:
1389                 return remove_qp_ok((struct res_qp *)res);
1390         case RES_CQ:
1391                 return remove_cq_ok((struct res_cq *)res);
1392         case RES_SRQ:
1393                 return remove_srq_ok((struct res_srq *)res);
1394         case RES_MPT:
1395                 return remove_mpt_ok((struct res_mpt *)res);
1396         case RES_MTT:
1397                 return remove_mtt_ok((struct res_mtt *)res, extra);
1398         case RES_MAC:
1399                 return -ENOSYS;
1400         case RES_EQ:
1401                 return remove_eq_ok((struct res_eq *)res);
1402         case RES_COUNTER:
1403                 return remove_counter_ok((struct res_counter *)res);
1404         case RES_XRCD:
1405                 return remove_xrcdn_ok((struct res_xrcdn *)res);
1406         case RES_FS_RULE:
1407                 return remove_fs_rule_ok((struct res_fs_rule *)res);
1408         default:
1409                 return -EINVAL;
1410         }
1411 }
1412
1413 static int rem_res_range(struct mlx4_dev *dev, int slave, u64 base, int count,
1414                          enum mlx4_resource type, int extra)
1415 {
1416         u64 i;
1417         int err;
1418         struct mlx4_priv *priv = mlx4_priv(dev);
1419         struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker;
1420         struct res_common *r;
1421
1422         spin_lock_irq(mlx4_tlock(dev));
1423         for (i = base; i < base + count; ++i) {
1424                 r = res_tracker_lookup(&tracker->res_tree[type], i);
1425                 if (!r) {
1426                         err = -ENOENT;
1427                         goto out;
1428                 }
1429                 if (r->owner != slave) {
1430                         err = -EPERM;
1431                         goto out;
1432                 }
1433                 err = remove_ok(r, type, extra);
1434                 if (err)
1435                         goto out;
1436         }
1437
1438         for (i = base; i < base + count; ++i) {
1439                 r = res_tracker_lookup(&tracker->res_tree[type], i);
1440                 rb_erase(&r->node, &tracker->res_tree[type]);
1441                 list_del(&r->list);
1442                 kfree(r);
1443         }
1444         err = 0;
1445
1446 out:
1447         spin_unlock_irq(mlx4_tlock(dev));
1448
1449         return err;
1450 }
1451
1452 static int qp_res_start_move_to(struct mlx4_dev *dev, int slave, int qpn,
1453                                 enum res_qp_states state, struct res_qp **qp,
1454                                 int alloc)
1455 {
1456         struct mlx4_priv *priv = mlx4_priv(dev);
1457         struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker;
1458         struct res_qp *r;
1459         int err = 0;
1460
1461         spin_lock_irq(mlx4_tlock(dev));
1462         r = res_tracker_lookup(&tracker->res_tree[RES_QP], qpn);
1463         if (!r)
1464                 err = -ENOENT;
1465         else if (r->com.owner != slave)
1466                 err = -EPERM;
1467         else {
1468                 switch (state) {
1469                 case RES_QP_BUSY:
1470                         mlx4_dbg(dev, "%s: failed RES_QP, 0x%llx\n",
1471                                  __func__, r->com.res_id);
1472                         err = -EBUSY;
1473                         break;
1474
1475                 case RES_QP_RESERVED:
1476                         if (r->com.state == RES_QP_MAPPED && !alloc)
1477                                 break;
1478
1479                         mlx4_dbg(dev, "failed RES_QP, 0x%llx\n", r->com.res_id);
1480                         err = -EINVAL;
1481                         break;
1482
1483                 case RES_QP_MAPPED:
1484                         if ((r->com.state == RES_QP_RESERVED && alloc) ||
1485                             r->com.state == RES_QP_HW)
1486                                 break;
1487                         else {
1488                                 mlx4_dbg(dev, "failed RES_QP, 0x%llx\n",
1489                                           r->com.res_id);
1490                                 err = -EINVAL;
1491                         }
1492
1493                         break;
1494
1495                 case RES_QP_HW:
1496                         if (r->com.state != RES_QP_MAPPED)
1497                                 err = -EINVAL;
1498                         break;
1499                 default:
1500                         err = -EINVAL;
1501                 }
1502
1503                 if (!err) {
1504                         r->com.from_state = r->com.state;
1505                         r->com.to_state = state;
1506                         r->com.state = RES_QP_BUSY;
1507                         if (qp)
1508                                 *qp = r;
1509                 }
1510         }
1511
1512         spin_unlock_irq(mlx4_tlock(dev));
1513
1514         return err;
1515 }
1516
1517 static int mr_res_start_move_to(struct mlx4_dev *dev, int slave, int index,
1518                                 enum res_mpt_states state, struct res_mpt **mpt)
1519 {
1520         struct mlx4_priv *priv = mlx4_priv(dev);
1521         struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker;
1522         struct res_mpt *r;
1523         int err = 0;
1524
1525         spin_lock_irq(mlx4_tlock(dev));
1526         r = res_tracker_lookup(&tracker->res_tree[RES_MPT], index);
1527         if (!r)
1528                 err = -ENOENT;
1529         else if (r->com.owner != slave)
1530                 err = -EPERM;
1531         else {
1532                 switch (state) {
1533                 case RES_MPT_BUSY:
1534                         err = -EINVAL;
1535                         break;
1536
1537                 case RES_MPT_RESERVED:
1538                         if (r->com.state != RES_MPT_MAPPED)
1539                                 err = -EINVAL;
1540                         break;
1541
1542                 case RES_MPT_MAPPED:
1543                         if (r->com.state != RES_MPT_RESERVED &&
1544                             r->com.state != RES_MPT_HW)
1545                                 err = -EINVAL;
1546                         break;
1547
1548                 case RES_MPT_HW:
1549                         if (r->com.state != RES_MPT_MAPPED)
1550                                 err = -EINVAL;
1551                         break;
1552                 default:
1553                         err = -EINVAL;
1554                 }
1555
1556                 if (!err) {
1557                         r->com.from_state = r->com.state;
1558                         r->com.to_state = state;
1559                         r->com.state = RES_MPT_BUSY;
1560                         if (mpt)
1561                                 *mpt = r;
1562                 }
1563         }
1564
1565         spin_unlock_irq(mlx4_tlock(dev));
1566
1567         return err;
1568 }
1569
1570 static int eq_res_start_move_to(struct mlx4_dev *dev, int slave, int index,
1571                                 enum res_eq_states state, struct res_eq **eq)
1572 {
1573         struct mlx4_priv *priv = mlx4_priv(dev);
1574         struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker;
1575         struct res_eq *r;
1576         int err = 0;
1577
1578         spin_lock_irq(mlx4_tlock(dev));
1579         r = res_tracker_lookup(&tracker->res_tree[RES_EQ], index);
1580         if (!r)
1581                 err = -ENOENT;
1582         else if (r->com.owner != slave)
1583                 err = -EPERM;
1584         else {
1585                 switch (state) {
1586                 case RES_EQ_BUSY:
1587                         err = -EINVAL;
1588                         break;
1589
1590                 case RES_EQ_RESERVED:
1591                         if (r->com.state != RES_EQ_HW)
1592                                 err = -EINVAL;
1593                         break;
1594
1595                 case RES_EQ_HW:
1596                         if (r->com.state != RES_EQ_RESERVED)
1597                                 err = -EINVAL;
1598                         break;
1599
1600                 default:
1601                         err = -EINVAL;
1602                 }
1603
1604                 if (!err) {
1605                         r->com.from_state = r->com.state;
1606                         r->com.to_state = state;
1607                         r->com.state = RES_EQ_BUSY;
1608                         if (eq)
1609                                 *eq = r;
1610                 }
1611         }
1612
1613         spin_unlock_irq(mlx4_tlock(dev));
1614
1615         return err;
1616 }
1617
1618 static int cq_res_start_move_to(struct mlx4_dev *dev, int slave, int cqn,
1619                                 enum res_cq_states state, struct res_cq **cq)
1620 {
1621         struct mlx4_priv *priv = mlx4_priv(dev);
1622         struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker;
1623         struct res_cq *r;
1624         int err;
1625
1626         spin_lock_irq(mlx4_tlock(dev));
1627         r = res_tracker_lookup(&tracker->res_tree[RES_CQ], cqn);
1628         if (!r) {
1629                 err = -ENOENT;
1630         } else if (r->com.owner != slave) {
1631                 err = -EPERM;
1632         } else if (state == RES_CQ_ALLOCATED) {
1633                 if (r->com.state != RES_CQ_HW)
1634                         err = -EINVAL;
1635                 else if (atomic_read(&r->ref_count))
1636                         err = -EBUSY;
1637                 else
1638                         err = 0;
1639         } else if (state != RES_CQ_HW || r->com.state != RES_CQ_ALLOCATED) {
1640                 err = -EINVAL;
1641         } else {
1642                 err = 0;
1643         }
1644
1645         if (!err) {
1646                 r->com.from_state = r->com.state;
1647                 r->com.to_state = state;
1648                 r->com.state = RES_CQ_BUSY;
1649                 if (cq)
1650                         *cq = r;
1651         }
1652
1653         spin_unlock_irq(mlx4_tlock(dev));
1654
1655         return err;
1656 }
1657
1658 static int srq_res_start_move_to(struct mlx4_dev *dev, int slave, int index,
1659                                  enum res_srq_states state, struct res_srq **srq)
1660 {
1661         struct mlx4_priv *priv = mlx4_priv(dev);
1662         struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker;
1663         struct res_srq *r;
1664         int err = 0;
1665
1666         spin_lock_irq(mlx4_tlock(dev));
1667         r = res_tracker_lookup(&tracker->res_tree[RES_SRQ], index);
1668         if (!r) {
1669                 err = -ENOENT;
1670         } else if (r->com.owner != slave) {
1671                 err = -EPERM;
1672         } else if (state == RES_SRQ_ALLOCATED) {
1673                 if (r->com.state != RES_SRQ_HW)
1674                         err = -EINVAL;
1675                 else if (atomic_read(&r->ref_count))
1676                         err = -EBUSY;
1677         } else if (state != RES_SRQ_HW || r->com.state != RES_SRQ_ALLOCATED) {
1678                 err = -EINVAL;
1679         }
1680
1681         if (!err) {
1682                 r->com.from_state = r->com.state;
1683                 r->com.to_state = state;
1684                 r->com.state = RES_SRQ_BUSY;
1685                 if (srq)
1686                         *srq = r;
1687         }
1688
1689         spin_unlock_irq(mlx4_tlock(dev));
1690
1691         return err;
1692 }
1693
1694 static void res_abort_move(struct mlx4_dev *dev, int slave,
1695                            enum mlx4_resource type, int id)
1696 {
1697         struct mlx4_priv *priv = mlx4_priv(dev);
1698         struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker;
1699         struct res_common *r;
1700
1701         spin_lock_irq(mlx4_tlock(dev));
1702         r = res_tracker_lookup(&tracker->res_tree[type], id);
1703         if (r && (r->owner == slave))
1704                 r->state = r->from_state;
1705         spin_unlock_irq(mlx4_tlock(dev));
1706 }
1707
1708 static void res_end_move(struct mlx4_dev *dev, int slave,
1709                          enum mlx4_resource type, int id)
1710 {
1711         struct mlx4_priv *priv = mlx4_priv(dev);
1712         struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker;
1713         struct res_common *r;
1714
1715         spin_lock_irq(mlx4_tlock(dev));
1716         r = res_tracker_lookup(&tracker->res_tree[type], id);
1717         if (r && (r->owner == slave))
1718                 r->state = r->to_state;
1719         spin_unlock_irq(mlx4_tlock(dev));
1720 }
1721
1722 static int valid_reserved(struct mlx4_dev *dev, int slave, int qpn)
1723 {
1724         return mlx4_is_qp_reserved(dev, qpn) &&
1725                 (mlx4_is_master(dev) || mlx4_is_guest_proxy(dev, slave, qpn));
1726 }
1727
1728 static int fw_reserved(struct mlx4_dev *dev, int qpn)
1729 {
1730         return qpn < dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW];
1731 }
1732
1733 static int qp_alloc_res(struct mlx4_dev *dev, int slave, int op, int cmd,
1734                         u64 in_param, u64 *out_param)
1735 {
1736         int err;
1737         int count;
1738         int align;
1739         int base;
1740         int qpn;
1741         u8 flags;
1742
1743         switch (op) {
1744         case RES_OP_RESERVE:
1745                 count = get_param_l(&in_param) & 0xffffff;
1746                 /* Turn off all unsupported QP allocation flags that the
1747                  * slave tries to set.
1748                  */
1749                 flags = (get_param_l(&in_param) >> 24) & dev->caps.alloc_res_qp_mask;
1750                 align = get_param_h(&in_param);
1751                 err = mlx4_grant_resource(dev, slave, RES_QP, count, 0);
1752                 if (err)
1753                         return err;
1754
1755                 err = __mlx4_qp_reserve_range(dev, count, align, &base, flags);
1756                 if (err) {
1757                         mlx4_release_resource(dev, slave, RES_QP, count, 0);
1758                         return err;
1759                 }
1760
1761                 err = add_res_range(dev, slave, base, count, RES_QP, 0);
1762                 if (err) {
1763                         mlx4_release_resource(dev, slave, RES_QP, count, 0);
1764                         __mlx4_qp_release_range(dev, base, count);
1765                         return err;
1766                 }
1767                 set_param_l(out_param, base);
1768                 break;
1769         case RES_OP_MAP_ICM:
1770                 qpn = get_param_l(&in_param) & 0x7fffff;
1771                 if (valid_reserved(dev, slave, qpn)) {
1772                         err = add_res_range(dev, slave, qpn, 1, RES_QP, 0);
1773                         if (err)
1774                                 return err;
1775                 }
1776
1777                 err = qp_res_start_move_to(dev, slave, qpn, RES_QP_MAPPED,
1778                                            NULL, 1);
1779                 if (err)
1780                         return err;
1781
1782                 if (!fw_reserved(dev, qpn)) {
1783                         err = __mlx4_qp_alloc_icm(dev, qpn, GFP_KERNEL);
1784                         if (err) {
1785                                 res_abort_move(dev, slave, RES_QP, qpn);
1786                                 return err;
1787                         }
1788                 }
1789
1790                 res_end_move(dev, slave, RES_QP, qpn);
1791                 break;
1792
1793         default:
1794                 err = -EINVAL;
1795                 break;
1796         }
1797         return err;
1798 }
1799
1800 static int mtt_alloc_res(struct mlx4_dev *dev, int slave, int op, int cmd,
1801                          u64 in_param, u64 *out_param)
1802 {
1803         int err = -EINVAL;
1804         int base;
1805         int order;
1806
1807         if (op != RES_OP_RESERVE_AND_MAP)
1808                 return err;
1809
1810         order = get_param_l(&in_param);
1811
1812         err = mlx4_grant_resource(dev, slave, RES_MTT, 1 << order, 0);
1813         if (err)
1814                 return err;
1815
1816         base = __mlx4_alloc_mtt_range(dev, order);
1817         if (base == -1) {
1818                 mlx4_release_resource(dev, slave, RES_MTT, 1 << order, 0);
1819                 return -ENOMEM;
1820         }
1821
1822         err = add_res_range(dev, slave, base, 1, RES_MTT, order);
1823         if (err) {
1824                 mlx4_release_resource(dev, slave, RES_MTT, 1 << order, 0);
1825                 __mlx4_free_mtt_range(dev, base, order);
1826         } else {
1827                 set_param_l(out_param, base);
1828         }
1829
1830         return err;
1831 }
1832
1833 static int mpt_alloc_res(struct mlx4_dev *dev, int slave, int op, int cmd,
1834                          u64 in_param, u64 *out_param)
1835 {
1836         int err = -EINVAL;
1837         int index;
1838         int id;
1839         struct res_mpt *mpt;
1840
1841         switch (op) {
1842         case RES_OP_RESERVE:
1843                 err = mlx4_grant_resource(dev, slave, RES_MPT, 1, 0);
1844                 if (err)
1845                         break;
1846
1847                 index = __mlx4_mpt_reserve(dev);
1848                 if (index == -1) {
1849                         mlx4_release_resource(dev, slave, RES_MPT, 1, 0);
1850                         break;
1851                 }
1852                 id = index & mpt_mask(dev);
1853
1854                 err = add_res_range(dev, slave, id, 1, RES_MPT, index);
1855                 if (err) {
1856                         mlx4_release_resource(dev, slave, RES_MPT, 1, 0);
1857                         __mlx4_mpt_release(dev, index);
1858                         break;
1859                 }
1860                 set_param_l(out_param, index);
1861                 break;
1862         case RES_OP_MAP_ICM:
1863                 index = get_param_l(&in_param);
1864                 id = index & mpt_mask(dev);
1865                 err = mr_res_start_move_to(dev, slave, id,
1866                                            RES_MPT_MAPPED, &mpt);
1867                 if (err)
1868                         return err;
1869
1870                 err = __mlx4_mpt_alloc_icm(dev, mpt->key, GFP_KERNEL);
1871                 if (err) {
1872                         res_abort_move(dev, slave, RES_MPT, id);
1873                         return err;
1874                 }
1875
1876                 res_end_move(dev, slave, RES_MPT, id);
1877                 break;
1878         }
1879         return err;
1880 }
1881
1882 static int cq_alloc_res(struct mlx4_dev *dev, int slave, int op, int cmd,
1883                         u64 in_param, u64 *out_param)
1884 {
1885         int cqn;
1886         int err;
1887
1888         switch (op) {
1889         case RES_OP_RESERVE_AND_MAP:
1890                 err = mlx4_grant_resource(dev, slave, RES_CQ, 1, 0);
1891                 if (err)
1892                         break;
1893
1894                 err = __mlx4_cq_alloc_icm(dev, &cqn);
1895                 if (err) {
1896                         mlx4_release_resource(dev, slave, RES_CQ, 1, 0);
1897                         break;
1898                 }
1899
1900                 err = add_res_range(dev, slave, cqn, 1, RES_CQ, 0);
1901                 if (err) {
1902                         mlx4_release_resource(dev, slave, RES_CQ, 1, 0);
1903                         __mlx4_cq_free_icm(dev, cqn);
1904                         break;
1905                 }
1906
1907                 set_param_l(out_param, cqn);
1908                 break;
1909
1910         default:
1911                 err = -EINVAL;
1912         }
1913
1914         return err;
1915 }
1916
1917 static int srq_alloc_res(struct mlx4_dev *dev, int slave, int op, int cmd,
1918                          u64 in_param, u64 *out_param)
1919 {
1920         int srqn;
1921         int err;
1922
1923         switch (op) {
1924         case RES_OP_RESERVE_AND_MAP:
1925                 err = mlx4_grant_resource(dev, slave, RES_SRQ, 1, 0);
1926                 if (err)
1927                         break;
1928
1929                 err = __mlx4_srq_alloc_icm(dev, &srqn);
1930                 if (err) {
1931                         mlx4_release_resource(dev, slave, RES_SRQ, 1, 0);
1932                         break;
1933                 }
1934
1935                 err = add_res_range(dev, slave, srqn, 1, RES_SRQ, 0);
1936                 if (err) {
1937                         mlx4_release_resource(dev, slave, RES_SRQ, 1, 0);
1938                         __mlx4_srq_free_icm(dev, srqn);
1939                         break;
1940                 }
1941
1942                 set_param_l(out_param, srqn);
1943                 break;
1944
1945         default:
1946                 err = -EINVAL;
1947         }
1948
1949         return err;
1950 }
1951
1952 static int mac_find_smac_ix_in_slave(struct mlx4_dev *dev, int slave, int port,
1953                                      u8 smac_index, u64 *mac)
1954 {
1955         struct mlx4_priv *priv = mlx4_priv(dev);
1956         struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker;
1957         struct list_head *mac_list =
1958                 &tracker->slave_list[slave].res_list[RES_MAC];
1959         struct mac_res *res, *tmp;
1960
1961         list_for_each_entry_safe(res, tmp, mac_list, list) {
1962                 if (res->smac_index == smac_index && res->port == (u8) port) {
1963                         *mac = res->mac;
1964                         return 0;
1965                 }
1966         }
1967         return -ENOENT;
1968 }
1969
1970 static int mac_add_to_slave(struct mlx4_dev *dev, int slave, u64 mac, int port, u8 smac_index)
1971 {
1972         struct mlx4_priv *priv = mlx4_priv(dev);
1973         struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker;
1974         struct list_head *mac_list =
1975                 &tracker->slave_list[slave].res_list[RES_MAC];
1976         struct mac_res *res, *tmp;
1977
1978         list_for_each_entry_safe(res, tmp, mac_list, list) {
1979                 if (res->mac == mac && res->port == (u8) port) {
1980                         /* mac found. update ref count */
1981                         ++res->ref_count;
1982                         return 0;
1983                 }
1984         }
1985
1986         if (mlx4_grant_resource(dev, slave, RES_MAC, 1, port))
1987                 return -EINVAL;
1988         res = kzalloc(sizeof *res, GFP_KERNEL);
1989         if (!res) {
1990                 mlx4_release_resource(dev, slave, RES_MAC, 1, port);
1991                 return -ENOMEM;
1992         }
1993         res->mac = mac;
1994         res->port = (u8) port;
1995         res->smac_index = smac_index;
1996         res->ref_count = 1;
1997         list_add_tail(&res->list,
1998                       &tracker->slave_list[slave].res_list[RES_MAC]);
1999         return 0;
2000 }
2001
2002 static void mac_del_from_slave(struct mlx4_dev *dev, int slave, u64 mac,
2003                                int port)
2004 {
2005         struct mlx4_priv *priv = mlx4_priv(dev);
2006         struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker;
2007         struct list_head *mac_list =
2008                 &tracker->slave_list[slave].res_list[RES_MAC];
2009         struct mac_res *res, *tmp;
2010
2011         list_for_each_entry_safe(res, tmp, mac_list, list) {
2012                 if (res->mac == mac && res->port == (u8) port) {
2013                         if (!--res->ref_count) {
2014                                 list_del(&res->list);
2015                                 mlx4_release_resource(dev, slave, RES_MAC, 1, port);
2016                                 kfree(res);
2017                         }
2018                         break;
2019                 }
2020         }
2021 }
2022
2023 static void rem_slave_macs(struct mlx4_dev *dev, int slave)
2024 {
2025         struct mlx4_priv *priv = mlx4_priv(dev);
2026         struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker;
2027         struct list_head *mac_list =
2028                 &tracker->slave_list[slave].res_list[RES_MAC];
2029         struct mac_res *res, *tmp;
2030         int i;
2031
2032         list_for_each_entry_safe(res, tmp, mac_list, list) {
2033                 list_del(&res->list);
2034                 /* dereference the mac the num times the slave referenced it */
2035                 for (i = 0; i < res->ref_count; i++)
2036                         __mlx4_unregister_mac(dev, res->port, res->mac);
2037                 mlx4_release_resource(dev, slave, RES_MAC, 1, res->port);
2038                 kfree(res);
2039         }
2040 }
2041
2042 static int mac_alloc_res(struct mlx4_dev *dev, int slave, int op, int cmd,
2043                          u64 in_param, u64 *out_param, int in_port)
2044 {
2045         int err = -EINVAL;
2046         int port;
2047         u64 mac;
2048         u8 smac_index;
2049
2050         if (op != RES_OP_RESERVE_AND_MAP)
2051                 return err;
2052
2053         port = !in_port ? get_param_l(out_param) : in_port;
2054         port = mlx4_slave_convert_port(
2055                         dev, slave, port);
2056
2057         if (port < 0)
2058                 return -EINVAL;
2059         mac = in_param;
2060
2061         err = __mlx4_register_mac(dev, port, mac);
2062         if (err >= 0) {
2063                 smac_index = err;
2064                 set_param_l(out_param, err);
2065                 err = 0;
2066         }
2067
2068         if (!err) {
2069                 err = mac_add_to_slave(dev, slave, mac, port, smac_index);
2070                 if (err)
2071                         __mlx4_unregister_mac(dev, port, mac);
2072         }
2073         return err;
2074 }
2075
2076 static int vlan_add_to_slave(struct mlx4_dev *dev, int slave, u16 vlan,
2077                              int port, int vlan_index)
2078 {
2079         struct mlx4_priv *priv = mlx4_priv(dev);
2080         struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker;
2081         struct list_head *vlan_list =
2082                 &tracker->slave_list[slave].res_list[RES_VLAN];
2083         struct vlan_res *res, *tmp;
2084
2085         list_for_each_entry_safe(res, tmp, vlan_list, list) {
2086                 if (res->vlan == vlan && res->port == (u8) port) {
2087                         /* vlan found. update ref count */
2088                         ++res->ref_count;
2089                         return 0;
2090                 }
2091         }
2092
2093         if (mlx4_grant_resource(dev, slave, RES_VLAN, 1, port))
2094                 return -EINVAL;
2095         res = kzalloc(sizeof(*res), GFP_KERNEL);
2096         if (!res) {
2097                 mlx4_release_resource(dev, slave, RES_VLAN, 1, port);
2098                 return -ENOMEM;
2099         }
2100         res->vlan = vlan;
2101         res->port = (u8) port;
2102         res->vlan_index = vlan_index;
2103         res->ref_count = 1;
2104         list_add_tail(&res->list,
2105                       &tracker->slave_list[slave].res_list[RES_VLAN]);
2106         return 0;
2107 }
2108
2109
2110 static void vlan_del_from_slave(struct mlx4_dev *dev, int slave, u16 vlan,
2111                                 int port)
2112 {
2113         struct mlx4_priv *priv = mlx4_priv(dev);
2114         struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker;
2115         struct list_head *vlan_list =
2116                 &tracker->slave_list[slave].res_list[RES_VLAN];
2117         struct vlan_res *res, *tmp;
2118
2119         list_for_each_entry_safe(res, tmp, vlan_list, list) {
2120                 if (res->vlan == vlan && res->port == (u8) port) {
2121                         if (!--res->ref_count) {
2122                                 list_del(&res->list);
2123                                 mlx4_release_resource(dev, slave, RES_VLAN,
2124                                                       1, port);
2125                                 kfree(res);
2126                         }
2127                         break;
2128                 }
2129         }
2130 }
2131
2132 static void rem_slave_vlans(struct mlx4_dev *dev, int slave)
2133 {
2134         struct mlx4_priv *priv = mlx4_priv(dev);
2135         struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker;
2136         struct list_head *vlan_list =
2137                 &tracker->slave_list[slave].res_list[RES_VLAN];
2138         struct vlan_res *res, *tmp;
2139         int i;
2140
2141         list_for_each_entry_safe(res, tmp, vlan_list, list) {
2142                 list_del(&res->list);
2143                 /* dereference the vlan the num times the slave referenced it */
2144                 for (i = 0; i < res->ref_count; i++)
2145                         __mlx4_unregister_vlan(dev, res->port, res->vlan);
2146                 mlx4_release_resource(dev, slave, RES_VLAN, 1, res->port);
2147                 kfree(res);
2148         }
2149 }
2150
2151 static int vlan_alloc_res(struct mlx4_dev *dev, int slave, int op, int cmd,
2152                           u64 in_param, u64 *out_param, int in_port)
2153 {
2154         struct mlx4_priv *priv = mlx4_priv(dev);
2155         struct mlx4_slave_state *slave_state = priv->mfunc.master.slave_state;
2156         int err;
2157         u16 vlan;
2158         int vlan_index;
2159         int port;
2160
2161         port = !in_port ? get_param_l(out_param) : in_port;
2162
2163         if (!port || op != RES_OP_RESERVE_AND_MAP)
2164                 return -EINVAL;
2165
2166         port = mlx4_slave_convert_port(
2167                         dev, slave, port);
2168
2169         if (port < 0)
2170                 return -EINVAL;
2171         /* upstream kernels had NOP for reg/unreg vlan. Continue this. */
2172         if (!in_port && port > 0 && port <= dev->caps.num_ports) {
2173                 slave_state[slave].old_vlan_api = true;
2174                 return 0;
2175         }
2176
2177         vlan = (u16) in_param;
2178
2179         err = __mlx4_register_vlan(dev, port, vlan, &vlan_index);
2180         if (!err) {
2181                 set_param_l(out_param, (u32) vlan_index);
2182                 err = vlan_add_to_slave(dev, slave, vlan, port, vlan_index);
2183                 if (err)
2184                         __mlx4_unregister_vlan(dev, port, vlan);
2185         }
2186         return err;
2187 }
2188
2189 static int counter_alloc_res(struct mlx4_dev *dev, int slave, int op, int cmd,
2190                              u64 in_param, u64 *out_param, int port)
2191 {
2192         u32 index;
2193         int err;
2194
2195         if (op != RES_OP_RESERVE)
2196                 return -EINVAL;
2197
2198         err = mlx4_grant_resource(dev, slave, RES_COUNTER, 1, 0);
2199         if (err)
2200                 return err;
2201
2202         err = __mlx4_counter_alloc(dev, &index);
2203         if (err) {
2204                 mlx4_release_resource(dev, slave, RES_COUNTER, 1, 0);
2205                 return err;
2206         }
2207
2208         err = add_res_range(dev, slave, index, 1, RES_COUNTER, port);
2209         if (err) {
2210                 __mlx4_counter_free(dev, index);
2211                 mlx4_release_resource(dev, slave, RES_COUNTER, 1, 0);
2212         } else {
2213                 set_param_l(out_param, index);
2214         }
2215
2216         return err;
2217 }
2218
2219 static int xrcdn_alloc_res(struct mlx4_dev *dev, int slave, int op, int cmd,
2220                            u64 in_param, u64 *out_param)
2221 {
2222         u32 xrcdn;
2223         int err;
2224
2225         if (op != RES_OP_RESERVE)
2226                 return -EINVAL;
2227
2228         err = __mlx4_xrcd_alloc(dev, &xrcdn);
2229         if (err)
2230                 return err;
2231
2232         err = add_res_range(dev, slave, xrcdn, 1, RES_XRCD, 0);
2233         if (err)
2234                 __mlx4_xrcd_free(dev, xrcdn);
2235         else
2236                 set_param_l(out_param, xrcdn);
2237
2238         return err;
2239 }
2240
2241 int mlx4_ALLOC_RES_wrapper(struct mlx4_dev *dev, int slave,
2242                            struct mlx4_vhcr *vhcr,
2243                            struct mlx4_cmd_mailbox *inbox,
2244                            struct mlx4_cmd_mailbox *outbox,
2245                            struct mlx4_cmd_info *cmd)
2246 {
2247         int err;
2248         int alop = vhcr->op_modifier;
2249
2250         switch (vhcr->in_modifier & 0xFF) {
2251         case RES_QP:
2252                 err = qp_alloc_res(dev, slave, vhcr->op_modifier, alop,
2253                                    vhcr->in_param, &vhcr->out_param);
2254                 break;
2255
2256         case RES_MTT:
2257                 err = mtt_alloc_res(dev, slave, vhcr->op_modifier, alop,
2258                                     vhcr->in_param, &vhcr->out_param);
2259                 break;
2260
2261         case RES_MPT:
2262                 err = mpt_alloc_res(dev, slave, vhcr->op_modifier, alop,
2263                                     vhcr->in_param, &vhcr->out_param);
2264                 break;
2265
2266         case RES_CQ:
2267                 err = cq_alloc_res(dev, slave, vhcr->op_modifier, alop,
2268                                    vhcr->in_param, &vhcr->out_param);
2269                 break;
2270
2271         case RES_SRQ:
2272                 err = srq_alloc_res(dev, slave, vhcr->op_modifier, alop,
2273                                     vhcr->in_param, &vhcr->out_param);
2274                 break;
2275
2276         case RES_MAC:
2277                 err = mac_alloc_res(dev, slave, vhcr->op_modifier, alop,
2278                                     vhcr->in_param, &vhcr->out_param,
2279                                     (vhcr->in_modifier >> 8) & 0xFF);
2280                 break;
2281
2282         case RES_VLAN:
2283                 err = vlan_alloc_res(dev, slave, vhcr->op_modifier, alop,
2284                                      vhcr->in_param, &vhcr->out_param,
2285                                      (vhcr->in_modifier >> 8) & 0xFF);
2286                 break;
2287
2288         case RES_COUNTER:
2289                 err = counter_alloc_res(dev, slave, vhcr->op_modifier, alop,
2290                                         vhcr->in_param, &vhcr->out_param, 0);
2291                 break;
2292
2293         case RES_XRCD:
2294                 err = xrcdn_alloc_res(dev, slave, vhcr->op_modifier, alop,
2295                                       vhcr->in_param, &vhcr->out_param);
2296                 break;
2297
2298         default:
2299                 err = -EINVAL;
2300                 break;
2301         }
2302
2303         return err;
2304 }
2305
2306 static int qp_free_res(struct mlx4_dev *dev, int slave, int op, int cmd,
2307                        u64 in_param)
2308 {
2309         int err;
2310         int count;
2311         int base;
2312         int qpn;
2313
2314         switch (op) {
2315         case RES_OP_RESERVE:
2316                 base = get_param_l(&in_param) & 0x7fffff;
2317                 count = get_param_h(&in_param);
2318                 err = rem_res_range(dev, slave, base, count, RES_QP, 0);
2319                 if (err)
2320                         break;
2321                 mlx4_release_resource(dev, slave, RES_QP, count, 0);
2322                 __mlx4_qp_release_range(dev, base, count);
2323                 break;
2324         case RES_OP_MAP_ICM:
2325                 qpn = get_param_l(&in_param) & 0x7fffff;
2326                 err = qp_res_start_move_to(dev, slave, qpn, RES_QP_RESERVED,
2327                                            NULL, 0);
2328                 if (err)
2329                         return err;
2330
2331                 if (!fw_reserved(dev, qpn))
2332                         __mlx4_qp_free_icm(dev, qpn);
2333
2334                 res_end_move(dev, slave, RES_QP, qpn);
2335
2336                 if (valid_reserved(dev, slave, qpn))
2337                         err = rem_res_range(dev, slave, qpn, 1, RES_QP, 0);
2338                 break;
2339         default:
2340                 err = -EINVAL;
2341                 break;
2342         }
2343         return err;
2344 }
2345
2346 static int mtt_free_res(struct mlx4_dev *dev, int slave, int op, int cmd,
2347                         u64 in_param, u64 *out_param)
2348 {
2349         int err = -EINVAL;
2350         int base;
2351         int order;
2352
2353         if (op != RES_OP_RESERVE_AND_MAP)
2354                 return err;
2355
2356         base = get_param_l(&in_param);
2357         order = get_param_h(&in_param);
2358         err = rem_res_range(dev, slave, base, 1, RES_MTT, order);
2359         if (!err) {
2360                 mlx4_release_resource(dev, slave, RES_MTT, 1 << order, 0);
2361                 __mlx4_free_mtt_range(dev, base, order);
2362         }
2363         return err;
2364 }
2365
2366 static int mpt_free_res(struct mlx4_dev *dev, int slave, int op, int cmd,
2367                         u64 in_param)
2368 {
2369         int err = -EINVAL;
2370         int index;
2371         int id;
2372         struct res_mpt *mpt;
2373
2374         switch (op) {
2375         case RES_OP_RESERVE:
2376                 index = get_param_l(&in_param);
2377                 id = index & mpt_mask(dev);
2378                 err = get_res(dev, slave, id, RES_MPT, &mpt);
2379                 if (err)
2380                         break;
2381                 index = mpt->key;
2382                 put_res(dev, slave, id, RES_MPT);
2383
2384                 err = rem_res_range(dev, slave, id, 1, RES_MPT, 0);
2385                 if (err)
2386                         break;
2387                 mlx4_release_resource(dev, slave, RES_MPT, 1, 0);
2388                 __mlx4_mpt_release(dev, index);
2389                 break;
2390         case RES_OP_MAP_ICM:
2391                 index = get_param_l(&in_param);
2392                 id = index & mpt_mask(dev);
2393                 err = mr_res_start_move_to(dev, slave, id,
2394                                            RES_MPT_RESERVED, &mpt);
2395                 if (err)
2396                         return err;
2397
2398                 __mlx4_mpt_free_icm(dev, mpt->key);
2399                 res_end_move(dev, slave, RES_MPT, id);
2400                 break;
2401         default:
2402                 err = -EINVAL;
2403                 break;
2404         }
2405         return err;
2406 }
2407
2408 static int cq_free_res(struct mlx4_dev *dev, int slave, int op, int cmd,
2409                        u64 in_param, u64 *out_param)
2410 {
2411         int cqn;
2412         int err;
2413
2414         switch (op) {
2415         case RES_OP_RESERVE_AND_MAP:
2416                 cqn = get_param_l(&in_param);
2417                 err = rem_res_range(dev, slave, cqn, 1, RES_CQ, 0);
2418                 if (err)
2419                         break;
2420
2421                 mlx4_release_resource(dev, slave, RES_CQ, 1, 0);
2422                 __mlx4_cq_free_icm(dev, cqn);
2423                 break;
2424
2425         default:
2426                 err = -EINVAL;
2427                 break;
2428         }
2429
2430         return err;
2431 }
2432
2433 static int srq_free_res(struct mlx4_dev *dev, int slave, int op, int cmd,
2434                         u64 in_param, u64 *out_param)
2435 {
2436         int srqn;
2437         int err;
2438
2439         switch (op) {
2440         case RES_OP_RESERVE_AND_MAP:
2441                 srqn = get_param_l(&in_param);
2442                 err = rem_res_range(dev, slave, srqn, 1, RES_SRQ, 0);
2443                 if (err)
2444                         break;
2445
2446                 mlx4_release_resource(dev, slave, RES_SRQ, 1, 0);
2447                 __mlx4_srq_free_icm(dev, srqn);
2448                 break;
2449
2450         default:
2451                 err = -EINVAL;
2452                 break;
2453         }
2454
2455         return err;
2456 }
2457
2458 static int mac_free_res(struct mlx4_dev *dev, int slave, int op, int cmd,
2459                             u64 in_param, u64 *out_param, int in_port)
2460 {
2461         int port;
2462         int err = 0;
2463
2464         switch (op) {
2465         case RES_OP_RESERVE_AND_MAP:
2466                 port = !in_port ? get_param_l(out_param) : in_port;
2467                 port = mlx4_slave_convert_port(
2468                                 dev, slave, port);
2469
2470                 if (port < 0)
2471                         return -EINVAL;
2472                 mac_del_from_slave(dev, slave, in_param, port);
2473                 __mlx4_unregister_mac(dev, port, in_param);
2474                 break;
2475         default:
2476                 err = -EINVAL;
2477                 break;
2478         }
2479
2480         return err;
2481
2482 }
2483
2484 static int vlan_free_res(struct mlx4_dev *dev, int slave, int op, int cmd,
2485                             u64 in_param, u64 *out_param, int port)
2486 {
2487         struct mlx4_priv *priv = mlx4_priv(dev);
2488         struct mlx4_slave_state *slave_state = priv->mfunc.master.slave_state;
2489         int err = 0;
2490
2491         port = mlx4_slave_convert_port(
2492                         dev, slave, port);
2493
2494         if (port < 0)
2495                 return -EINVAL;
2496         switch (op) {
2497         case RES_OP_RESERVE_AND_MAP:
2498                 if (slave_state[slave].old_vlan_api)
2499                         return 0;
2500                 if (!port)
2501                         return -EINVAL;
2502                 vlan_del_from_slave(dev, slave, in_param, port);
2503                 __mlx4_unregister_vlan(dev, port, in_param);
2504                 break;
2505         default:
2506                 err = -EINVAL;
2507                 break;
2508         }
2509
2510         return err;
2511 }
2512
2513 static int counter_free_res(struct mlx4_dev *dev, int slave, int op, int cmd,
2514                             u64 in_param, u64 *out_param)
2515 {
2516         int index;
2517         int err;
2518
2519         if (op != RES_OP_RESERVE)
2520                 return -EINVAL;
2521
2522         index = get_param_l(&in_param);
2523         if (index == MLX4_SINK_COUNTER_INDEX(dev))
2524                 return 0;
2525
2526         err = rem_res_range(dev, slave, index, 1, RES_COUNTER, 0);
2527         if (err)
2528                 return err;
2529
2530         __mlx4_counter_free(dev, index);
2531         mlx4_release_resource(dev, slave, RES_COUNTER, 1, 0);
2532
2533         return err;
2534 }
2535
2536 static int xrcdn_free_res(struct mlx4_dev *dev, int slave, int op, int cmd,
2537                           u64 in_param, u64 *out_param)
2538 {
2539         int xrcdn;
2540         int err;
2541
2542         if (op != RES_OP_RESERVE)
2543                 return -EINVAL;
2544
2545         xrcdn = get_param_l(&in_param);
2546         err = rem_res_range(dev, slave, xrcdn, 1, RES_XRCD, 0);
2547         if (err)
2548                 return err;
2549
2550         __mlx4_xrcd_free(dev, xrcdn);
2551
2552         return err;
2553 }
2554
2555 int mlx4_FREE_RES_wrapper(struct mlx4_dev *dev, int slave,
2556                           struct mlx4_vhcr *vhcr,
2557                           struct mlx4_cmd_mailbox *inbox,
2558                           struct mlx4_cmd_mailbox *outbox,
2559                           struct mlx4_cmd_info *cmd)
2560 {
2561         int err = -EINVAL;
2562         int alop = vhcr->op_modifier;
2563
2564         switch (vhcr->in_modifier & 0xFF) {
2565         case RES_QP:
2566                 err = qp_free_res(dev, slave, vhcr->op_modifier, alop,
2567                                   vhcr->in_param);
2568                 break;
2569
2570         case RES_MTT:
2571                 err = mtt_free_res(dev, slave, vhcr->op_modifier, alop,
2572                                    vhcr->in_param, &vhcr->out_param);
2573                 break;
2574
2575         case RES_MPT:
2576                 err = mpt_free_res(dev, slave, vhcr->op_modifier, alop,
2577                                    vhcr->in_param);
2578                 break;
2579
2580         case RES_CQ:
2581                 err = cq_free_res(dev, slave, vhcr->op_modifier, alop,
2582                                   vhcr->in_param, &vhcr->out_param);
2583                 break;
2584
2585         case RES_SRQ:
2586                 err = srq_free_res(dev, slave, vhcr->op_modifier, alop,
2587                                    vhcr->in_param, &vhcr->out_param);
2588                 break;
2589
2590         case RES_MAC:
2591                 err = mac_free_res(dev, slave, vhcr->op_modifier, alop,
2592                                    vhcr->in_param, &vhcr->out_param,
2593                                    (vhcr->in_modifier >> 8) & 0xFF);
2594                 break;
2595
2596         case RES_VLAN:
2597                 err = vlan_free_res(dev, slave, vhcr->op_modifier, alop,
2598                                     vhcr->in_param, &vhcr->out_param,
2599                                     (vhcr->in_modifier >> 8) & 0xFF);
2600                 break;
2601
2602         case RES_COUNTER:
2603                 err = counter_free_res(dev, slave, vhcr->op_modifier, alop,
2604                                        vhcr->in_param, &vhcr->out_param);
2605                 break;
2606
2607         case RES_XRCD:
2608                 err = xrcdn_free_res(dev, slave, vhcr->op_modifier, alop,
2609                                      vhcr->in_param, &vhcr->out_param);
2610
2611         default:
2612                 break;
2613         }
2614         return err;
2615 }
2616
2617 /* ugly but other choices are uglier */
2618 static int mr_phys_mpt(struct mlx4_mpt_entry *mpt)
2619 {
2620         return (be32_to_cpu(mpt->flags) >> 9) & 1;
2621 }
2622
2623 static int mr_get_mtt_addr(struct mlx4_mpt_entry *mpt)
2624 {
2625         return (int)be64_to_cpu(mpt->mtt_addr) & 0xfffffff8;
2626 }
2627
2628 static int mr_get_mtt_size(struct mlx4_mpt_entry *mpt)
2629 {
2630         return be32_to_cpu(mpt->mtt_sz);
2631 }
2632
2633 static u32 mr_get_pd(struct mlx4_mpt_entry *mpt)
2634 {
2635         return be32_to_cpu(mpt->pd_flags) & 0x00ffffff;
2636 }
2637
2638 static int mr_is_fmr(struct mlx4_mpt_entry *mpt)
2639 {
2640         return be32_to_cpu(mpt->pd_flags) & MLX4_MPT_PD_FLAG_FAST_REG;
2641 }
2642
2643 static int mr_is_bind_enabled(struct mlx4_mpt_entry *mpt)
2644 {
2645         return be32_to_cpu(mpt->flags) & MLX4_MPT_FLAG_BIND_ENABLE;
2646 }
2647
2648 static int mr_is_region(struct mlx4_mpt_entry *mpt)
2649 {
2650         return be32_to_cpu(mpt->flags) & MLX4_MPT_FLAG_REGION;
2651 }
2652
2653 static int qp_get_mtt_addr(struct mlx4_qp_context *qpc)
2654 {
2655         return be32_to_cpu(qpc->mtt_base_addr_l) & 0xfffffff8;
2656 }
2657
2658 static int srq_get_mtt_addr(struct mlx4_srq_context *srqc)
2659 {
2660         return be32_to_cpu(srqc->mtt_base_addr_l) & 0xfffffff8;
2661 }
2662
2663 static int qp_get_mtt_size(struct mlx4_qp_context *qpc)
2664 {
2665         int page_shift = (qpc->log_page_size & 0x3f) + 12;
2666         int log_sq_size = (qpc->sq_size_stride >> 3) & 0xf;
2667         int log_sq_sride = qpc->sq_size_stride & 7;
2668         int log_rq_size = (qpc->rq_size_stride >> 3) & 0xf;
2669         int log_rq_stride = qpc->rq_size_stride & 7;
2670         int srq = (be32_to_cpu(qpc->srqn) >> 24) & 1;
2671         int rss = (be32_to_cpu(qpc->flags) >> 13) & 1;
2672         u32 ts = (be32_to_cpu(qpc->flags) >> 16) & 0xff;
2673         int xrc = (ts == MLX4_QP_ST_XRC) ? 1 : 0;
2674         int sq_size;
2675         int rq_size;
2676         int total_pages;
2677         int total_mem;
2678         int page_offset = (be32_to_cpu(qpc->params2) >> 6) & 0x3f;
2679
2680         sq_size = 1 << (log_sq_size + log_sq_sride + 4);
2681         rq_size = (srq|rss|xrc) ? 0 : (1 << (log_rq_size + log_rq_stride + 4));
2682         total_mem = sq_size + rq_size;
2683         total_pages =
2684                 roundup_pow_of_two((total_mem + (page_offset << 6)) >>
2685                                    page_shift);
2686
2687         return total_pages;
2688 }
2689
2690 static int check_mtt_range(struct mlx4_dev *dev, int slave, int start,
2691                            int size, struct res_mtt *mtt)
2692 {
2693         int res_start = mtt->com.res_id;
2694         int res_size = (1 << mtt->order);
2695
2696         if (start < res_start || start + size > res_start + res_size)
2697                 return -EPERM;
2698         return 0;
2699 }
2700
2701 int mlx4_SW2HW_MPT_wrapper(struct mlx4_dev *dev, int slave,
2702                            struct mlx4_vhcr *vhcr,
2703                            struct mlx4_cmd_mailbox *inbox,
2704                            struct mlx4_cmd_mailbox *outbox,
2705                            struct mlx4_cmd_info *cmd)
2706 {
2707         int err;
2708         int index = vhcr->in_modifier;
2709         struct res_mtt *mtt;
2710         struct res_mpt *mpt;
2711         int mtt_base = mr_get_mtt_addr(inbox->buf) / dev->caps.mtt_entry_sz;
2712         int phys;
2713         int id;
2714         u32 pd;
2715         int pd_slave;
2716
2717         id = index & mpt_mask(dev);
2718         err = mr_res_start_move_to(dev, slave, id, RES_MPT_HW, &mpt);
2719         if (err)
2720                 return err;
2721
2722         /* Disable memory windows for VFs. */
2723         if (!mr_is_region(inbox->buf)) {
2724                 err = -EPERM;
2725                 goto ex_abort;
2726         }
2727
2728         /* Make sure that the PD bits related to the slave id are zeros. */
2729         pd = mr_get_pd(inbox->buf);
2730         pd_slave = (pd >> 17) & 0x7f;
2731         if (pd_slave != 0 && --pd_slave != slave) {
2732                 err = -EPERM;
2733                 goto ex_abort;
2734         }
2735
2736         if (mr_is_fmr(inbox->buf)) {
2737                 /* FMR and Bind Enable are forbidden in slave devices. */
2738                 if (mr_is_bind_enabled(inbox->buf)) {
2739                         err = -EPERM;
2740                         goto ex_abort;
2741                 }
2742                 /* FMR and Memory Windows are also forbidden. */
2743                 if (!mr_is_region(inbox->buf)) {
2744                         err = -EPERM;
2745                         goto ex_abort;
2746                 }
2747         }
2748
2749         phys = mr_phys_mpt(inbox->buf);
2750         if (!phys) {
2751                 err = get_res(dev, slave, mtt_base, RES_MTT, &mtt);
2752                 if (err)
2753                         goto ex_abort;
2754
2755                 err = check_mtt_range(dev, slave, mtt_base,
2756                                       mr_get_mtt_size(inbox->buf), mtt);
2757                 if (err)
2758                         goto ex_put;
2759
2760                 mpt->mtt = mtt;
2761         }
2762
2763         err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd);
2764         if (err)
2765                 goto ex_put;
2766
2767         if (!phys) {
2768                 atomic_inc(&mtt->ref_count);
2769                 put_res(dev, slave, mtt->com.res_id, RES_MTT);
2770         }
2771
2772         res_end_move(dev, slave, RES_MPT, id);
2773         return 0;
2774
2775 ex_put:
2776         if (!phys)
2777                 put_res(dev, slave, mtt->com.res_id, RES_MTT);
2778 ex_abort:
2779         res_abort_move(dev, slave, RES_MPT, id);
2780
2781         return err;
2782 }
2783
2784 int mlx4_HW2SW_MPT_wrapper(struct mlx4_dev *dev, int slave,
2785                            struct mlx4_vhcr *vhcr,
2786                            struct mlx4_cmd_mailbox *inbox,
2787                            struct mlx4_cmd_mailbox *outbox,
2788                            struct mlx4_cmd_info *cmd)
2789 {
2790         int err;
2791         int index = vhcr->in_modifier;
2792         struct res_mpt *mpt;
2793         int id;
2794
2795         id = index & mpt_mask(dev);
2796         err = mr_res_start_move_to(dev, slave, id, RES_MPT_MAPPED, &mpt);
2797         if (err)
2798                 return err;
2799
2800         err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd);
2801         if (err)
2802                 goto ex_abort;
2803
2804         if (mpt->mtt)
2805                 atomic_dec(&mpt->mtt->ref_count);
2806
2807         res_end_move(dev, slave, RES_MPT, id);
2808         return 0;
2809
2810 ex_abort:
2811         res_abort_move(dev, slave, RES_MPT, id);
2812
2813         return err;
2814 }
2815
2816 int mlx4_QUERY_MPT_wrapper(struct mlx4_dev *dev, int slave,
2817                            struct mlx4_vhcr *vhcr,
2818                            struct mlx4_cmd_mailbox *inbox,
2819                            struct mlx4_cmd_mailbox *outbox,
2820                            struct mlx4_cmd_info *cmd)
2821 {
2822         int err;
2823         int index = vhcr->in_modifier;
2824         struct res_mpt *mpt;
2825         int id;
2826
2827         id = index & mpt_mask(dev);
2828         err = get_res(dev, slave, id, RES_MPT, &mpt);
2829         if (err)
2830                 return err;
2831
2832         if (mpt->com.from_state == RES_MPT_MAPPED) {
2833                 /* In order to allow rereg in SRIOV, we need to alter the MPT entry. To do
2834                  * that, the VF must read the MPT. But since the MPT entry memory is not
2835                  * in the VF's virtual memory space, it must use QUERY_MPT to obtain the
2836                  * entry contents. To guarantee that the MPT cannot be changed, the driver
2837                  * must perform HW2SW_MPT before this query and return the MPT entry to HW
2838                  * ownership fofollowing the change. The change here allows the VF to
2839                  * perform QUERY_MPT also when the entry is in SW ownership.
2840                  */
2841                 struct mlx4_mpt_entry *mpt_entry = mlx4_table_find(
2842                                         &mlx4_priv(dev)->mr_table.dmpt_table,
2843                                         mpt->key, NULL);
2844
2845                 if (NULL == mpt_entry || NULL == outbox->buf) {
2846                         err = -EINVAL;
2847                         goto out;
2848                 }
2849
2850                 memcpy(outbox->buf, mpt_entry, sizeof(*mpt_entry));
2851
2852                 err = 0;
2853         } else if (mpt->com.from_state == RES_MPT_HW) {
2854                 err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd);
2855         } else {
2856                 err = -EBUSY;
2857                 goto out;
2858         }
2859
2860
2861 out:
2862         put_res(dev, slave, id, RES_MPT);
2863         return err;
2864 }
2865
2866 static int qp_get_rcqn(struct mlx4_qp_context *qpc)
2867 {
2868         return be32_to_cpu(qpc->cqn_recv) & 0xffffff;
2869 }
2870
2871 static int qp_get_scqn(struct mlx4_qp_context *qpc)
2872 {
2873         return be32_to_cpu(qpc->cqn_send) & 0xffffff;
2874 }
2875
2876 static u32 qp_get_srqn(struct mlx4_qp_context *qpc)
2877 {
2878         return be32_to_cpu(qpc->srqn) & 0x1ffffff;
2879 }
2880
2881 static void adjust_proxy_tun_qkey(struct mlx4_dev *dev, struct mlx4_vhcr *vhcr,
2882                                   struct mlx4_qp_context *context)
2883 {
2884         u32 qpn = vhcr->in_modifier & 0xffffff;
2885         u32 qkey = 0;
2886
2887         if (mlx4_get_parav_qkey(dev, qpn, &qkey))
2888                 return;
2889
2890         /* adjust qkey in qp context */
2891         context->qkey = cpu_to_be32(qkey);
2892 }
2893
2894 static int adjust_qp_sched_queue(struct mlx4_dev *dev, int slave,
2895                                  struct mlx4_qp_context *qpc,
2896                                  struct mlx4_cmd_mailbox *inbox);
2897
2898 int mlx4_RST2INIT_QP_wrapper(struct mlx4_dev *dev, int slave,
2899                              struct mlx4_vhcr *vhcr,
2900                              struct mlx4_cmd_mailbox *inbox,
2901                              struct mlx4_cmd_mailbox *outbox,
2902                              struct mlx4_cmd_info *cmd)
2903 {
2904         int err;
2905         int qpn = vhcr->in_modifier & 0x7fffff;
2906         struct res_mtt *mtt;
2907         struct res_qp *qp;
2908         struct mlx4_qp_context *qpc = inbox->buf + 8;
2909         int mtt_base = qp_get_mtt_addr(qpc) / dev->caps.mtt_entry_sz;
2910         int mtt_size = qp_get_mtt_size(qpc);
2911         struct res_cq *rcq;
2912         struct res_cq *scq;
2913         int rcqn = qp_get_rcqn(qpc);
2914         int scqn = qp_get_scqn(qpc);
2915         u32 srqn = qp_get_srqn(qpc) & 0xffffff;
2916         int use_srq = (qp_get_srqn(qpc) >> 24) & 1;
2917         struct res_srq *srq;
2918         int local_qpn = be32_to_cpu(qpc->local_qpn) & 0xffffff;
2919
2920         err = adjust_qp_sched_queue(dev, slave, qpc, inbox);
2921         if (err)
2922                 return err;
2923
2924         err = qp_res_start_move_to(dev, slave, qpn, RES_QP_HW, &qp, 0);
2925         if (err)
2926                 return err;
2927         qp->local_qpn = local_qpn;
2928         qp->sched_queue = 0;
2929         qp->param3 = 0;
2930         qp->vlan_control = 0;
2931         qp->fvl_rx = 0;
2932         qp->pri_path_fl = 0;
2933         qp->vlan_index = 0;
2934         qp->feup = 0;
2935         qp->qpc_flags = be32_to_cpu(qpc->flags);
2936
2937         err = get_res(dev, slave, mtt_base, RES_MTT, &mtt);
2938         if (err)
2939                 goto ex_abort;
2940
2941         err = check_mtt_range(dev, slave, mtt_base, mtt_size, mtt);
2942         if (err)
2943                 goto ex_put_mtt;
2944
2945         err = get_res(dev, slave, rcqn, RES_CQ, &rcq);
2946         if (err)
2947                 goto ex_put_mtt;
2948
2949         if (scqn != rcqn) {
2950                 err = get_res(dev, slave, scqn, RES_CQ, &scq);
2951                 if (err)
2952                         goto ex_put_rcq;
2953         } else
2954                 scq = rcq;
2955
2956         if (use_srq) {
2957                 err = get_res(dev, slave, srqn, RES_SRQ, &srq);
2958                 if (err)
2959                         goto ex_put_scq;
2960         }
2961
2962         adjust_proxy_tun_qkey(dev, vhcr, qpc);
2963         update_pkey_index(dev, slave, inbox);
2964         err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd);
2965         if (err)
2966                 goto ex_put_srq;
2967         atomic_inc(&mtt->ref_count);
2968         qp->mtt = mtt;
2969         atomic_inc(&rcq->ref_count);
2970         qp->rcq = rcq;
2971         atomic_inc(&scq->ref_count);
2972         qp->scq = scq;
2973
2974         if (scqn != rcqn)
2975                 put_res(dev, slave, scqn, RES_CQ);
2976
2977         if (use_srq) {
2978                 atomic_inc(&srq->ref_count);
2979                 put_res(dev, slave, srqn, RES_SRQ);
2980                 qp->srq = srq;
2981         }
2982         put_res(dev, slave, rcqn, RES_CQ);
2983         put_res(dev, slave, mtt_base, RES_MTT);
2984         res_end_move(dev, slave, RES_QP, qpn);
2985
2986         return 0;
2987
2988 ex_put_srq:
2989         if (use_srq)
2990                 put_res(dev, slave, srqn, RES_SRQ);
2991 ex_put_scq:
2992         if (scqn != rcqn)
2993                 put_res(dev, slave, scqn, RES_CQ);
2994 ex_put_rcq:
2995         put_res(dev, slave, rcqn, RES_CQ);
2996 ex_put_mtt:
2997         put_res(dev, slave, mtt_base, RES_MTT);
2998 ex_abort:
2999         res_abort_move(dev, slave, RES_QP, qpn);
3000
3001         return err;
3002 }
3003
3004 static int eq_get_mtt_addr(struct mlx4_eq_context *eqc)
3005 {
3006         return be32_to_cpu(eqc->mtt_base_addr_l) & 0xfffffff8;
3007 }
3008
3009 static int eq_get_mtt_size(struct mlx4_eq_context *eqc)
3010 {
3011         int log_eq_size = eqc->log_eq_size & 0x1f;
3012         int page_shift = (eqc->log_page_size & 0x3f) + 12;
3013
3014         if (log_eq_size + 5 < page_shift)
3015                 return 1;
3016
3017         return 1 << (log_eq_size + 5 - page_shift);
3018 }
3019
3020 static int cq_get_mtt_addr(struct mlx4_cq_context *cqc)
3021 {
3022         return be32_to_cpu(cqc->mtt_base_addr_l) & 0xfffffff8;
3023 }
3024
3025 static int cq_get_mtt_size(struct mlx4_cq_context *cqc)
3026 {
3027         int log_cq_size = (be32_to_cpu(cqc->logsize_usrpage) >> 24) & 0x1f;
3028         int page_shift = (cqc->log_page_size & 0x3f) + 12;
3029
3030         if (log_cq_size + 5 < page_shift)
3031                 return 1;
3032
3033         return 1 << (log_cq_size + 5 - page_shift);
3034 }
3035
3036 int mlx4_SW2HW_EQ_wrapper(struct mlx4_dev *dev, int slave,
3037                           struct mlx4_vhcr *vhcr,
3038                           struct mlx4_cmd_mailbox *inbox,
3039                           struct mlx4_cmd_mailbox *outbox,
3040                           struct mlx4_cmd_info *cmd)
3041 {
3042         int err;
3043         int eqn = vhcr->in_modifier;
3044         int res_id = (slave << 10) | eqn;
3045         struct mlx4_eq_context *eqc = inbox->buf;
3046         int mtt_base = eq_get_mtt_addr(eqc) / dev->caps.mtt_entry_sz;
3047         int mtt_size = eq_get_mtt_size(eqc);
3048         struct res_eq *eq;
3049         struct res_mtt *mtt;
3050
3051         err = add_res_range(dev, slave, res_id, 1, RES_EQ, 0);
3052         if (err)
3053                 return err;
3054         err = eq_res_start_move_to(dev, slave, res_id, RES_EQ_HW, &eq);
3055         if (err)
3056                 goto out_add;
3057
3058         err = get_res(dev, slave, mtt_base, RES_MTT, &mtt);
3059         if (err)
3060                 goto out_move;
3061
3062         err = check_mtt_range(dev, slave, mtt_base, mtt_size, mtt);
3063         if (err)
3064                 goto out_put;
3065
3066         err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd);
3067         if (err)
3068                 goto out_put;
3069
3070         atomic_inc(&mtt->ref_count);
3071         eq->mtt = mtt;
3072         put_res(dev, slave, mtt->com.res_id, RES_MTT);
3073         res_end_move(dev, slave, RES_EQ, res_id);
3074         return 0;
3075
3076 out_put:
3077         put_res(dev, slave, mtt->com.res_id, RES_MTT);
3078 out_move:
3079         res_abort_move(dev, slave, RES_EQ, res_id);
3080 out_add:
3081         rem_res_range(dev, slave, res_id, 1, RES_EQ, 0);
3082         return err;
3083 }
3084
3085 int mlx4_CONFIG_DEV_wrapper(struct mlx4_dev *dev, int slave,
3086                             struct mlx4_vhcr *vhcr,
3087                             struct mlx4_cmd_mailbox *inbox,
3088                             struct mlx4_cmd_mailbox *outbox,
3089                             struct mlx4_cmd_info *cmd)
3090 {
3091         int err;
3092         u8 get = vhcr->op_modifier;
3093
3094         if (get != 1)
3095                 return -EPERM;
3096
3097         err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd);
3098
3099         return err;
3100 }
3101
3102 static int get_containing_mtt(struct mlx4_dev *dev, int slave, int start,
3103                               int len, struct res_mtt **res)
3104 {
3105         struct mlx4_priv *priv = mlx4_priv(dev);
3106         struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker;
3107         struct res_mtt *mtt;
3108         int err = -EINVAL;
3109
3110         spin_lock_irq(mlx4_tlock(dev));
3111         list_for_each_entry(mtt, &tracker->slave_list[slave].res_list[RES_MTT],
3112                             com.list) {
3113                 if (!check_mtt_range(dev, slave, start, len, mtt)) {
3114                         *res = mtt;
3115                         mtt->com.from_state = mtt->com.state;
3116                         mtt->com.state = RES_MTT_BUSY;
3117                         err = 0;
3118                         break;
3119                 }
3120         }
3121         spin_unlock_irq(mlx4_tlock(dev));
3122
3123         return err;
3124 }
3125
3126 static int verify_qp_parameters(struct mlx4_dev *dev,
3127                                 struct mlx4_vhcr *vhcr,
3128                                 struct mlx4_cmd_mailbox *inbox,
3129                                 enum qp_transition transition, u8 slave)
3130 {
3131         u32                     qp_type;
3132         u32                     qpn;
3133         struct mlx4_qp_context  *qp_ctx;
3134         enum mlx4_qp_optpar     optpar;
3135         int port;
3136         int num_gids;
3137
3138         qp_ctx  = inbox->buf + 8;
3139         qp_type = (be32_to_cpu(qp_ctx->flags) >> 16) & 0xff;
3140         optpar  = be32_to_cpu(*(__be32 *) inbox->buf);
3141
3142         if (slave != mlx4_master_func_num(dev)) {
3143                 qp_ctx->params2 &= ~MLX4_QP_BIT_FPP;
3144                 /* setting QP rate-limit is disallowed for VFs */
3145                 if (qp_ctx->rate_limit_params)
3146                         return -EPERM;
3147         }
3148
3149         switch (qp_type) {
3150         case MLX4_QP_ST_RC:
3151         case MLX4_QP_ST_XRC:
3152         case MLX4_QP_ST_UC:
3153                 switch (transition) {
3154                 case QP_TRANS_INIT2RTR:
3155                 case QP_TRANS_RTR2RTS:
3156                 case QP_TRANS_RTS2RTS:
3157                 case QP_TRANS_SQD2SQD:
3158                 case QP_TRANS_SQD2RTS:
3159                         if (slave != mlx4_master_func_num(dev)) {
3160                                 if (optpar & MLX4_QP_OPTPAR_PRIMARY_ADDR_PATH) {
3161                                         port = (qp_ctx->pri_path.sched_queue >> 6 & 1) + 1;
3162                                         if (dev->caps.port_mask[port] != MLX4_PORT_TYPE_IB)
3163                                                 num_gids = mlx4_get_slave_num_gids(dev, slave, port);
3164                                         else
3165                                                 num_gids = 1;
3166                                         if (qp_ctx->pri_path.mgid_index >= num_gids)
3167                                                 return -EINVAL;
3168                                 }
3169                                 if (optpar & MLX4_QP_OPTPAR_ALT_ADDR_PATH) {
3170                                         port = (qp_ctx->alt_path.sched_queue >> 6 & 1) + 1;
3171                                         if (dev->caps.port_mask[port] != MLX4_PORT_TYPE_IB)
3172                                                 num_gids = mlx4_get_slave_num_gids(dev, slave, port);
3173                                         else
3174                                                 num_gids = 1;
3175                                         if (qp_ctx->alt_path.mgid_index >= num_gids)
3176                                                 return -EINVAL;
3177                                 }
3178                         }
3179                         break;
3180                 default:
3181                         break;
3182                 }
3183                 break;
3184
3185         case MLX4_QP_ST_MLX:
3186                 qpn = vhcr->in_modifier & 0x7fffff;
3187                 port = (qp_ctx->pri_path.sched_queue >> 6 & 1) + 1;
3188                 if (transition == QP_TRANS_INIT2RTR &&
3189                     slave != mlx4_master_func_num(dev) &&
3190                     mlx4_is_qp_reserved(dev, qpn) &&
3191                     !mlx4_vf_smi_enabled(dev, slave, port)) {
3192                         /* only enabled VFs may create MLX proxy QPs */
3193                         mlx4_err(dev, "%s: unprivileged slave %d attempting to create an MLX proxy special QP on port %d\n",
3194                                  __func__, slave, port);
3195                         return -EPERM;
3196                 }
3197                 break;
3198
3199         default:
3200                 break;
3201         }
3202
3203         return 0;
3204 }
3205
3206 int mlx4_WRITE_MTT_wrapper(struct mlx4_dev *dev, int slave,
3207                            struct mlx4_vhcr *vhcr,
3208                            struct mlx4_cmd_mailbox *inbox,
3209                            struct mlx4_cmd_mailbox *outbox,
3210                            struct mlx4_cmd_info *cmd)
3211 {
3212         struct mlx4_mtt mtt;
3213         __be64 *page_list = inbox->buf;
3214         u64 *pg_list = (u64 *)page_list;
3215         int i;
3216         struct res_mtt *rmtt = NULL;
3217         int start = be64_to_cpu(page_list[0]);
3218         int npages = vhcr->in_modifier;
3219         int err;
3220
3221         err = get_containing_mtt(dev, slave, start, npages, &rmtt);
3222         if (err)
3223                 return err;
3224
3225         /* Call the SW implementation of write_mtt:
3226          * - Prepare a dummy mtt struct
3227          * - Translate inbox contents to simple addresses in host endianness */
3228         mtt.offset = 0;  /* TBD this is broken but I don't handle it since
3229                             we don't really use it */
3230         mtt.order = 0;
3231         mtt.page_shift = 0;
3232         for (i = 0; i < npages; ++i)
3233                 pg_list[i + 2] = (be64_to_cpu(page_list[i + 2]) & ~1ULL);
3234
3235         err = __mlx4_write_mtt(dev, &mtt, be64_to_cpu(page_list[0]), npages,
3236                                ((u64 *)page_list + 2));
3237
3238         if (rmtt)
3239                 put_res(dev, slave, rmtt->com.res_id, RES_MTT);
3240
3241         return err;
3242 }
3243
3244 int mlx4_HW2SW_EQ_wrapper(struct mlx4_dev *dev, int slave,
3245                           struct mlx4_vhcr *vhcr,
3246                           struct mlx4_cmd_mailbox *inbox,
3247                           struct mlx4_cmd_mailbox *outbox,
3248                           struct mlx4_cmd_info *cmd)
3249 {
3250         int eqn = vhcr->in_modifier;
3251         int res_id = eqn | (slave << 10);
3252         struct res_eq *eq;
3253         int err;
3254
3255         err = eq_res_start_move_to(dev, slave, res_id, RES_EQ_RESERVED, &eq);
3256         if (err)
3257                 return err;
3258
3259         err = get_res(dev, slave, eq->mtt->com.res_id, RES_MTT, NULL);
3260         if (err)
3261                 goto ex_abort;
3262
3263         err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd);
3264         if (err)
3265                 goto ex_put;
3266
3267         atomic_dec(&eq->mtt->ref_count);
3268         put_res(dev, slave, eq->mtt->com.res_id, RES_MTT);
3269         res_end_move(dev, slave, RES_EQ, res_id);
3270         rem_res_range(dev, slave, res_id, 1, RES_EQ, 0);
3271
3272         return 0;
3273
3274 ex_put:
3275         put_res(dev, slave, eq->mtt->com.res_id, RES_MTT);
3276 ex_abort:
3277         res_abort_move(dev, slave, RES_EQ, res_id);
3278
3279         return err;
3280 }
3281
3282 int mlx4_GEN_EQE(struct mlx4_dev *dev, int slave, struct mlx4_eqe *eqe)
3283 {
3284         struct mlx4_priv *priv = mlx4_priv(dev);
3285         struct mlx4_slave_event_eq_info *event_eq;
3286         struct mlx4_cmd_mailbox *mailbox;
3287         u32 in_modifier = 0;
3288         int err;
3289         int res_id;
3290         struct res_eq *req;
3291
3292         if (!priv->mfunc.master.slave_state)
3293                 return -EINVAL;
3294
3295         /* check for slave valid, slave not PF, and slave active */
3296         if (slave < 0 || slave > dev->persist->num_vfs ||
3297             slave == dev->caps.function ||
3298             !priv->mfunc.master.slave_state[slave].active)
3299                 return 0;
3300
3301         event_eq = &priv->mfunc.master.slave_state[slave].event_eq[eqe->type];
3302
3303         /* Create the event only if the slave is registered */
3304         if (event_eq->eqn < 0)
3305                 return 0;
3306
3307         mutex_lock(&priv->mfunc.master.gen_eqe_mutex[slave]);
3308         res_id = (slave << 10) | event_eq->eqn;
3309         err = get_res(dev, slave, res_id, RES_EQ, &req);
3310         if (err)
3311                 goto unlock;
3312
3313         if (req->com.from_state != RES_EQ_HW) {
3314                 err = -EINVAL;
3315                 goto put;
3316         }
3317
3318         mailbox = mlx4_alloc_cmd_mailbox(dev);
3319         if (IS_ERR(mailbox)) {
3320                 err = PTR_ERR(mailbox);
3321                 goto put;
3322         }
3323
3324         if (eqe->type == MLX4_EVENT_TYPE_CMD) {
3325                 ++event_eq->token;
3326                 eqe->event.cmd.token = cpu_to_be16(event_eq->token);
3327         }
3328
3329         memcpy(mailbox->buf, (u8 *) eqe, 28);
3330
3331         in_modifier = (slave & 0xff) | ((event_eq->eqn & 0x3ff) << 16);
3332
3333         err = mlx4_cmd(dev, mailbox->dma, in_modifier, 0,
3334                        MLX4_CMD_GEN_EQE, MLX4_CMD_TIME_CLASS_B,
3335                        MLX4_CMD_NATIVE);
3336
3337         put_res(dev, slave, res_id, RES_EQ);
3338         mutex_unlock(&priv->mfunc.master.gen_eqe_mutex[slave]);
3339         mlx4_free_cmd_mailbox(dev, mailbox);
3340         return err;
3341
3342 put:
3343         put_res(dev, slave, res_id, RES_EQ);
3344
3345 unlock:
3346         mutex_unlock(&priv->mfunc.master.gen_eqe_mutex[slave]);
3347         return err;
3348 }
3349
3350 int mlx4_QUERY_EQ_wrapper(struct mlx4_dev *dev, int slave,
3351                           struct mlx4_vhcr *vhcr,
3352                           struct mlx4_cmd_mailbox *inbox,
3353                           struct mlx4_cmd_mailbox *outbox,
3354                           struct mlx4_cmd_info *cmd)
3355 {
3356         int eqn = vhcr->in_modifier;
3357         int res_id = eqn | (slave << 10);
3358         struct res_eq *eq;
3359         int err;
3360
3361         err = get_res(dev, slave, res_id, RES_EQ, &eq);
3362         if (err)
3363                 return err;
3364
3365         if (eq->com.from_state != RES_EQ_HW) {
3366                 err = -EINVAL;
3367                 goto ex_put;
3368         }
3369
3370         err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd);
3371
3372 ex_put:
3373         put_res(dev, slave, res_id, RES_EQ);
3374         return err;
3375 }
3376
3377 int mlx4_SW2HW_CQ_wrapper(struct mlx4_dev *dev, int slave,
3378                           struct mlx4_vhcr *vhcr,
3379                           struct mlx4_cmd_mailbox *inbox,
3380                           struct mlx4_cmd_mailbox *outbox,
3381                           struct mlx4_cmd_info *cmd)
3382 {
3383         int err;
3384         int cqn = vhcr->in_modifier;
3385         struct mlx4_cq_context *cqc = inbox->buf;
3386         int mtt_base = cq_get_mtt_addr(cqc) / dev->caps.mtt_entry_sz;
3387         struct res_cq *cq = NULL;
3388         struct res_mtt *mtt;
3389
3390         err = cq_res_start_move_to(dev, slave, cqn, RES_CQ_HW, &cq);
3391         if (err)
3392                 return err;
3393         err = get_res(dev, slave, mtt_base, RES_MTT, &mtt);
3394         if (err)
3395                 goto out_move;
3396         err = check_mtt_range(dev, slave, mtt_base, cq_get_mtt_size(cqc), mtt);
3397         if (err)
3398                 goto out_put;
3399         err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd);
3400         if (err)
3401                 goto out_put;
3402         atomic_inc(&mtt->ref_count);
3403         cq->mtt = mtt;
3404         put_res(dev, slave, mtt->com.res_id, RES_MTT);
3405         res_end_move(dev, slave, RES_CQ, cqn);
3406         return 0;
3407
3408 out_put:
3409         put_res(dev, slave, mtt->com.res_id, RES_MTT);
3410 out_move:
3411         res_abort_move(dev, slave, RES_CQ, cqn);
3412         return err;
3413 }
3414
3415 int mlx4_HW2SW_CQ_wrapper(struct mlx4_dev *dev, int slave,
3416                           struct mlx4_vhcr *vhcr,
3417                           struct mlx4_cmd_mailbox *inbox,
3418                           struct mlx4_cmd_mailbox *outbox,
3419                           struct mlx4_cmd_info *cmd)
3420 {
3421         int err;
3422         int cqn = vhcr->in_modifier;
3423         struct res_cq *cq = NULL;
3424
3425         err = cq_res_start_move_to(dev, slave, cqn, RES_CQ_ALLOCATED, &cq);
3426         if (err)
3427                 return err;
3428         err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd);
3429         if (err)
3430                 goto out_move;
3431         atomic_dec(&cq->mtt->ref_count);
3432         res_end_move(dev, slave, RES_CQ, cqn);
3433         return 0;
3434
3435 out_move:
3436         res_abort_move(dev, slave, RES_CQ, cqn);
3437         return err;
3438 }
3439
3440 int mlx4_QUERY_CQ_wrapper(struct mlx4_dev *dev, int slave,
3441                           struct mlx4_vhcr *vhcr,
3442                           struct mlx4_cmd_mailbox *inbox,
3443                           struct mlx4_cmd_mailbox *outbox,
3444                           struct mlx4_cmd_info *cmd)
3445 {
3446         int cqn = vhcr->in_modifier;
3447         struct res_cq *cq;
3448         int err;
3449
3450         err = get_res(dev, slave, cqn, RES_CQ, &cq);
3451         if (err)
3452                 return err;
3453
3454         if (cq->com.from_state != RES_CQ_HW)
3455                 goto ex_put;
3456
3457         err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd);
3458 ex_put:
3459         put_res(dev, slave, cqn, RES_CQ);
3460
3461         return err;
3462 }
3463
3464 static int handle_resize(struct mlx4_dev *dev, int slave,
3465                          struct mlx4_vhcr *vhcr,
3466                          struct mlx4_cmd_mailbox *inbox,
3467                          struct mlx4_cmd_mailbox *outbox,
3468                          struct mlx4_cmd_info *cmd,
3469                          struct res_cq *cq)
3470 {
3471         int err;
3472         struct res_mtt *orig_mtt;
3473         struct res_mtt *mtt;
3474         struct mlx4_cq_context *cqc = inbox->buf;
3475         int mtt_base = cq_get_mtt_addr(cqc) / dev->caps.mtt_entry_sz;
3476
3477         err = get_res(dev, slave, cq->mtt->com.res_id, RES_MTT, &orig_mtt);
3478         if (err)
3479                 return err;
3480
3481         if (orig_mtt != cq->mtt) {
3482                 err = -EINVAL;
3483                 goto ex_put;
3484         }
3485
3486         err = get_res(dev, slave, mtt_base, RES_MTT, &mtt);
3487         if (err)
3488                 goto ex_put;
3489
3490         err = check_mtt_range(dev, slave, mtt_base, cq_get_mtt_size(cqc), mtt);
3491         if (err)
3492                 goto ex_put1;
3493         err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd);
3494         if (err)
3495                 goto ex_put1;
3496         atomic_dec(&orig_mtt->ref_count);
3497         put_res(dev, slave, orig_mtt->com.res_id, RES_MTT);
3498         atomic_inc(&mtt->ref_count);
3499         cq->mtt = mtt;
3500         put_res(dev, slave, mtt->com.res_id, RES_MTT);
3501         return 0;
3502
3503 ex_put1:
3504         put_res(dev, slave, mtt->com.res_id, RES_MTT);
3505 ex_put:
3506         put_res(dev, slave, orig_mtt->com.res_id, RES_MTT);
3507
3508         return err;
3509
3510 }
3511
3512 int mlx4_MODIFY_CQ_wrapper(struct mlx4_dev *dev, int slave,
3513                            struct mlx4_vhcr *vhcr,
3514                            struct mlx4_cmd_mailbox *inbox,
3515                            struct mlx4_cmd_mailbox *outbox,
3516                            struct mlx4_cmd_info *cmd)
3517 {
3518         int cqn = vhcr->in_modifier;
3519         struct res_cq *cq;
3520         int err;
3521
3522         err = get_res(dev, slave, cqn, RES_CQ, &cq);
3523         if (err)
3524                 return err;
3525
3526         if (cq->com.from_state != RES_CQ_HW)
3527                 goto ex_put;
3528
3529         if (vhcr->op_modifier == 0) {
3530                 err = handle_resize(dev, slave, vhcr, inbox, outbox, cmd, cq);
3531                 goto ex_put;
3532         }
3533
3534         err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd);
3535 ex_put:
3536         put_res(dev, slave, cqn, RES_CQ);
3537
3538         return err;
3539 }
3540
3541 static int srq_get_mtt_size(struct mlx4_srq_context *srqc)
3542 {
3543         int log_srq_size = (be32_to_cpu(srqc->state_logsize_srqn) >> 24) & 0xf;
3544         int log_rq_stride = srqc->logstride & 7;
3545         int page_shift = (srqc->log_page_size & 0x3f) + 12;
3546
3547         if (log_srq_size + log_rq_stride + 4 < page_shift)
3548                 return 1;
3549
3550         return 1 << (log_srq_size + log_rq_stride + 4 - page_shift);
3551 }
3552
3553 int mlx4_SW2HW_SRQ_wrapper(struct mlx4_dev *dev, int slave,
3554                            struct mlx4_vhcr *vhcr,
3555                            struct mlx4_cmd_mailbox *inbox,
3556                            struct mlx4_cmd_mailbox *outbox,
3557                            struct mlx4_cmd_info *cmd)
3558 {
3559         int err;
3560         int srqn = vhcr->in_modifier;
3561         struct res_mtt *mtt;
3562         struct res_srq *srq = NULL;
3563         struct mlx4_srq_context *srqc = inbox->buf;
3564         int mtt_base = srq_get_mtt_addr(srqc) / dev->caps.mtt_entry_sz;
3565
3566         if (srqn != (be32_to_cpu(srqc->state_logsize_srqn) & 0xffffff))
3567                 return -EINVAL;
3568
3569         err = srq_res_start_move_to(dev, slave, srqn, RES_SRQ_HW, &srq);
3570         if (err)
3571                 return err;
3572         err = get_res(dev, slave, mtt_base, RES_MTT, &mtt);
3573         if (err)
3574                 goto ex_abort;
3575         err = check_mtt_range(dev, slave, mtt_base, srq_get_mtt_size(srqc),
3576                               mtt);
3577         if (err)
3578                 goto ex_put_mtt;
3579
3580         err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd);
3581         if (err)
3582                 goto ex_put_mtt;
3583
3584         atomic_inc(&mtt->ref_count);
3585         srq->mtt = mtt;
3586         put_res(dev, slave, mtt->com.res_id, RES_MTT);
3587         res_end_move(dev, slave, RES_SRQ, srqn);
3588         return 0;
3589
3590 ex_put_mtt:
3591         put_res(dev, slave, mtt->com.res_id, RES_MTT);
3592 ex_abort:
3593         res_abort_move(dev, slave, RES_SRQ, srqn);
3594
3595         return err;
3596 }
3597
3598 int mlx4_HW2SW_SRQ_wrapper(struct mlx4_dev *dev, int slave,
3599                            struct mlx4_vhcr *vhcr,
3600                            struct mlx4_cmd_mailbox *inbox,
3601                            struct mlx4_cmd_mailbox *outbox,
3602                            struct mlx4_cmd_info *cmd)
3603 {
3604         int err;
3605         int srqn = vhcr->in_modifier;
3606         struct res_srq *srq = NULL;
3607
3608         err = srq_res_start_move_to(dev, slave, srqn, RES_SRQ_ALLOCATED, &srq);
3609         if (err)
3610                 return err;
3611         err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd);
3612         if (err)
3613                 goto ex_abort;
3614         atomic_dec(&srq->mtt->ref_count);
3615         if (srq->cq)
3616                 atomic_dec(&srq->cq->ref_count);
3617         res_end_move(dev, slave, RES_SRQ, srqn);
3618
3619         return 0;
3620
3621 ex_abort:
3622         res_abort_move(dev, slave, RES_SRQ, srqn);
3623
3624         return err;
3625 }
3626
3627 int mlx4_QUERY_SRQ_wrapper(struct mlx4_dev *dev, int slave,
3628                            struct mlx4_vhcr *vhcr,
3629                            struct mlx4_cmd_mailbox *inbox,
3630                            struct mlx4_cmd_mailbox *outbox,
3631                            struct mlx4_cmd_info *cmd)
3632 {
3633         int err;
3634         int srqn = vhcr->in_modifier;
3635         struct res_srq *srq;
3636
3637         err = get_res(dev, slave, srqn, RES_SRQ, &srq);
3638         if (err)
3639                 return err;
3640         if (srq->com.from_state != RES_SRQ_HW) {
3641                 err = -EBUSY;
3642                 goto out;
3643         }
3644         err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd);
3645 out:
3646         put_res(dev, slave, srqn, RES_SRQ);
3647         return err;
3648 }
3649
3650 int mlx4_ARM_SRQ_wrapper(struct mlx4_dev *dev, int slave,
3651                          struct mlx4_vhcr *vhcr,
3652                          struct mlx4_cmd_mailbox *inbox,
3653                          struct mlx4_cmd_mailbox *outbox,
3654                          struct mlx4_cmd_info *cmd)
3655 {
3656         int err;
3657         int srqn = vhcr->in_modifier;
3658         struct res_srq *srq;
3659
3660         err = get_res(dev, slave, srqn, RES_SRQ, &srq);
3661         if (err)
3662                 return err;
3663
3664         if (srq->com.from_state != RES_SRQ_HW) {
3665                 err = -EBUSY;
3666                 goto out;
3667         }
3668
3669         err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd);
3670 out:
3671         put_res(dev, slave, srqn, RES_SRQ);
3672         return err;
3673 }
3674
3675 int mlx4_GEN_QP_wrapper(struct mlx4_dev *dev, int slave,
3676                         struct mlx4_vhcr *vhcr,
3677                         struct mlx4_cmd_mailbox *inbox,
3678                         struct mlx4_cmd_mailbox *outbox,
3679                         struct mlx4_cmd_info *cmd)
3680 {
3681         int err;
3682         int qpn = vhcr->in_modifier & 0x7fffff;
3683         struct res_qp *qp;
3684
3685         err = get_res(dev, slave, qpn, RES_QP, &qp);
3686         if (err)
3687                 return err;
3688         if (qp->com.from_state != RES_QP_HW) {
3689                 err = -EBUSY;
3690                 goto out;
3691         }
3692
3693         err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd);
3694 out:
3695         put_res(dev, slave, qpn, RES_QP);
3696         return err;
3697 }
3698
3699 int mlx4_INIT2INIT_QP_wrapper(struct mlx4_dev *dev, int slave,
3700                               struct mlx4_vhcr *vhcr,
3701                               struct mlx4_cmd_mailbox *inbox,
3702                               struct mlx4_cmd_mailbox *outbox,
3703                               struct mlx4_cmd_info *cmd)
3704 {
3705         struct mlx4_qp_context *context = inbox->buf + 8;
3706         adjust_proxy_tun_qkey(dev, vhcr, context);
3707         update_pkey_index(dev, slave, inbox);
3708         return mlx4_GEN_QP_wrapper(dev, slave, vhcr, inbox, outbox, cmd);
3709 }
3710
3711 static int adjust_qp_sched_queue(struct mlx4_dev *dev, int slave,
3712                                   struct mlx4_qp_context *qpc,
3713                                   struct mlx4_cmd_mailbox *inbox)
3714 {
3715         enum mlx4_qp_optpar optpar = be32_to_cpu(*(__be32 *)inbox->buf);
3716         u8 pri_sched_queue;
3717         int port = mlx4_slave_convert_port(
3718                    dev, slave, (qpc->pri_path.sched_queue >> 6 & 1) + 1) - 1;
3719
3720         if (port < 0)
3721                 return -EINVAL;
3722
3723         pri_sched_queue = (qpc->pri_path.sched_queue & ~(1 << 6)) |
3724                           ((port & 1) << 6);
3725
3726         if (optpar & (MLX4_QP_OPTPAR_PRIMARY_ADDR_PATH | MLX4_QP_OPTPAR_SCHED_QUEUE) ||
3727             qpc->pri_path.sched_queue || mlx4_is_eth(dev, port + 1)) {
3728                 qpc->pri_path.sched_queue = pri_sched_queue;
3729         }
3730
3731         if (optpar & MLX4_QP_OPTPAR_ALT_ADDR_PATH) {
3732                 port = mlx4_slave_convert_port(
3733                                 dev, slave, (qpc->alt_path.sched_queue >> 6 & 1)
3734                                 + 1) - 1;
3735                 if (port < 0)
3736                         return -EINVAL;
3737                 qpc->alt_path.sched_queue =
3738                         (qpc->alt_path.sched_queue & ~(1 << 6)) |
3739                         (port & 1) << 6;
3740         }
3741         return 0;
3742 }
3743
3744 static int roce_verify_mac(struct mlx4_dev *dev, int slave,
3745                                 struct mlx4_qp_context *qpc,
3746                                 struct mlx4_cmd_mailbox *inbox)
3747 {
3748         u64 mac;
3749         int port;
3750         u32 ts = (be32_to_cpu(qpc->flags) >> 16) & 0xff;
3751         u8 sched = *(u8 *)(inbox->buf + 64);
3752         u8 smac_ix;
3753
3754         port = (sched >> 6 & 1) + 1;
3755         if (mlx4_is_eth(dev, port) && (ts != MLX4_QP_ST_MLX)) {
3756                 smac_ix = qpc->pri_path.grh_mylmc & 0x7f;
3757                 if (mac_find_smac_ix_in_slave(dev, slave, port, smac_ix, &mac))
3758                         return -ENOENT;
3759         }
3760         return 0;
3761 }
3762
3763 int mlx4_INIT2RTR_QP_wrapper(struct mlx4_dev *dev, int slave,
3764                              struct mlx4_vhcr *vhcr,
3765                              struct mlx4_cmd_mailbox *inbox,
3766                              struct mlx4_cmd_mailbox *outbox,
3767                              struct mlx4_cmd_info *cmd)
3768 {
3769         int err;
3770         struct mlx4_qp_context *qpc = inbox->buf + 8;
3771         int qpn = vhcr->in_modifier & 0x7fffff;
3772         struct res_qp *qp;
3773         u8 orig_sched_queue;
3774         __be32  orig_param3 = qpc->param3;
3775         u8 orig_vlan_control = qpc->pri_path.vlan_control;
3776         u8 orig_fvl_rx = qpc->pri_path.fvl_rx;
3777         u8 orig_pri_path_fl = qpc->pri_path.fl;
3778         u8 orig_vlan_index = qpc->pri_path.vlan_index;
3779         u8 orig_feup = qpc->pri_path.feup;
3780
3781         err = adjust_qp_sched_queue(dev, slave, qpc, inbox);
3782         if (err)
3783                 return err;
3784         err = verify_qp_parameters(dev, vhcr, inbox, QP_TRANS_INIT2RTR, slave);
3785         if (err)
3786                 return err;
3787
3788         if (roce_verify_mac(dev, slave, qpc, inbox))
3789                 return -EINVAL;
3790
3791         update_pkey_index(dev, slave, inbox);
3792         update_gid(dev, inbox, (u8)slave);
3793         adjust_proxy_tun_qkey(dev, vhcr, qpc);
3794         orig_sched_queue = qpc->pri_path.sched_queue;
3795
3796         err = get_res(dev, slave, qpn, RES_QP, &qp);
3797         if (err)
3798                 return err;
3799         if (qp->com.from_state != RES_QP_HW) {
3800                 err = -EBUSY;
3801                 goto out;
3802         }
3803
3804         err = update_vport_qp_param(dev, inbox, slave, qpn);
3805         if (err)
3806                 goto out;
3807
3808         err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd);
3809 out:
3810         /* if no error, save sched queue value passed in by VF. This is
3811          * essentially the QOS value provided by the VF. This will be useful
3812          * if we allow dynamic changes from VST back to VGT
3813          */
3814         if (!err) {
3815                 qp->sched_queue = orig_sched_queue;
3816                 qp->param3      = orig_param3;
3817                 qp->vlan_control = orig_vlan_control;
3818                 qp->fvl_rx      =  orig_fvl_rx;
3819                 qp->pri_path_fl = orig_pri_path_fl;
3820                 qp->vlan_index  = orig_vlan_index;
3821                 qp->feup        = orig_feup;
3822         }
3823         put_res(dev, slave, qpn, RES_QP);
3824         return err;
3825 }
3826
3827 int mlx4_RTR2RTS_QP_wrapper(struct mlx4_dev *dev, int slave,
3828                             struct mlx4_vhcr *vhcr,
3829                             struct mlx4_cmd_mailbox *inbox,
3830                             struct mlx4_cmd_mailbox *outbox,
3831                             struct mlx4_cmd_info *cmd)
3832 {
3833         int err;
3834         struct mlx4_qp_context *context = inbox->buf + 8;
3835
3836         err = adjust_qp_sched_queue(dev, slave, context, inbox);
3837         if (err)
3838                 return err;
3839         err = verify_qp_parameters(dev, vhcr, inbox, QP_TRANS_RTR2RTS, slave);
3840         if (err)
3841                 return err;
3842
3843         update_pkey_index(dev, slave, inbox);
3844         update_gid(dev, inbox, (u8)slave);
3845         adjust_proxy_tun_qkey(dev, vhcr, context);
3846         return mlx4_GEN_QP_wrapper(dev, slave, vhcr, inbox, outbox, cmd);
3847 }
3848
3849 int mlx4_RTS2RTS_QP_wrapper(struct mlx4_dev *dev, int slave,
3850                             struct mlx4_vhcr *vhcr,
3851                             struct mlx4_cmd_mailbox *inbox,
3852                             struct mlx4_cmd_mailbox *outbox,
3853                             struct mlx4_cmd_info *cmd)
3854 {
3855         int err;
3856         struct mlx4_qp_context *context = inbox->buf + 8;
3857
3858         err = adjust_qp_sched_queue(dev, slave, context, inbox);
3859         if (err)
3860                 return err;
3861         err = verify_qp_parameters(dev, vhcr, inbox, QP_TRANS_RTS2RTS, slave);
3862         if (err)
3863                 return err;
3864
3865         update_pkey_index(dev, slave, inbox);
3866         update_gid(dev, inbox, (u8)slave);
3867         adjust_proxy_tun_qkey(dev, vhcr, context);
3868         return mlx4_GEN_QP_wrapper(dev, slave, vhcr, inbox, outbox, cmd);
3869 }
3870
3871
3872 int mlx4_SQERR2RTS_QP_wrapper(struct mlx4_dev *dev, int slave,
3873                               struct mlx4_vhcr *vhcr,
3874                               struct mlx4_cmd_mailbox *inbox,
3875                               struct mlx4_cmd_mailbox *outbox,
3876                               struct mlx4_cmd_info *cmd)
3877 {
3878         struct mlx4_qp_context *context = inbox->buf + 8;
3879         int err = adjust_qp_sched_queue(dev, slave, context, inbox);
3880         if (err)
3881                 return err;
3882         adjust_proxy_tun_qkey(dev, vhcr, context);
3883         return mlx4_GEN_QP_wrapper(dev, slave, vhcr, inbox, outbox, cmd);
3884 }
3885
3886 int mlx4_SQD2SQD_QP_wrapper(struct mlx4_dev *dev, int slave,
3887                             struct mlx4_vhcr *vhcr,
3888                             struct mlx4_cmd_mailbox *inbox,
3889                             struct mlx4_cmd_mailbox *outbox,
3890                             struct mlx4_cmd_info *cmd)
3891 {
3892         int err;
3893         struct mlx4_qp_context *context = inbox->buf + 8;
3894
3895         err = adjust_qp_sched_queue(dev, slave, context, inbox);
3896         if (err)
3897                 return err;
3898         err = verify_qp_parameters(dev, vhcr, inbox, QP_TRANS_SQD2SQD, slave);
3899         if (err)
3900                 return err;
3901
3902         adjust_proxy_tun_qkey(dev, vhcr, context);
3903         update_gid(dev, inbox, (u8)slave);
3904         update_pkey_index(dev, slave, inbox);
3905         return mlx4_GEN_QP_wrapper(dev, slave, vhcr, inbox, outbox, cmd);
3906 }
3907
3908 int mlx4_SQD2RTS_QP_wrapper(struct mlx4_dev *dev, int slave,
3909                             struct mlx4_vhcr *vhcr,
3910                             struct mlx4_cmd_mailbox *inbox,
3911                             struct mlx4_cmd_mailbox *outbox,
3912                             struct mlx4_cmd_info *cmd)
3913 {
3914         int err;
3915         struct mlx4_qp_context *context = inbox->buf + 8;
3916
3917         err = adjust_qp_sched_queue(dev, slave, context, inbox);
3918         if (err)
3919                 return err;
3920         err = verify_qp_parameters(dev, vhcr, inbox, QP_TRANS_SQD2RTS, slave);
3921         if (err)
3922                 return err;
3923
3924         adjust_proxy_tun_qkey(dev, vhcr, context);
3925         update_gid(dev, inbox, (u8)slave);
3926         update_pkey_index(dev, slave, inbox);
3927         return mlx4_GEN_QP_wrapper(dev, slave, vhcr, inbox, outbox, cmd);
3928 }
3929
3930 int mlx4_2RST_QP_wrapper(struct mlx4_dev *dev, int slave,
3931                          struct mlx4_vhcr *vhcr,
3932                          struct mlx4_cmd_mailbox *inbox,
3933                          struct mlx4_cmd_mailbox *outbox,
3934                          struct mlx4_cmd_info *cmd)
3935 {
3936         int err;
3937         int qpn = vhcr->in_modifier & 0x7fffff;
3938         struct res_qp *qp;
3939
3940         err = qp_res_start_move_to(dev, slave, qpn, RES_QP_MAPPED, &qp, 0);
3941         if (err)
3942                 return err;
3943         err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd);
3944         if (err)
3945                 goto ex_abort;
3946
3947         atomic_dec(&qp->mtt->ref_count);
3948         atomic_dec(&qp->rcq->ref_count);
3949         atomic_dec(&qp->scq->ref_count);
3950         if (qp->srq)
3951                 atomic_dec(&qp->srq->ref_count);
3952         res_end_move(dev, slave, RES_QP, qpn);
3953         return 0;
3954
3955 ex_abort:
3956         res_abort_move(dev, slave, RES_QP, qpn);
3957
3958         return err;
3959 }
3960
3961 static struct res_gid *find_gid(struct mlx4_dev *dev, int slave,
3962                                 struct res_qp *rqp, u8 *gid)
3963 {
3964         struct res_gid *res;
3965
3966         list_for_each_entry(res, &rqp->mcg_list, list) {
3967                 if (!memcmp(res->gid, gid, 16))
3968                         return res;
3969         }
3970         return NULL;
3971 }
3972
3973 static int add_mcg_res(struct mlx4_dev *dev, int slave, struct res_qp *rqp,
3974                        u8 *gid, enum mlx4_protocol prot,
3975                        enum mlx4_steer_type steer, u64 reg_id)
3976 {
3977         struct res_gid *res;
3978         int err;
3979
3980         res = kzalloc(sizeof *res, GFP_KERNEL);
3981         if (!res)
3982                 return -ENOMEM;
3983
3984         spin_lock_irq(&rqp->mcg_spl);
3985         if (find_gid(dev, slave, rqp, gid)) {
3986                 kfree(res);
3987                 err = -EEXIST;
3988         } else {
3989                 memcpy(res->gid, gid, 16);
3990                 res->prot = prot;
3991                 res->steer = steer;
3992                 res->reg_id = reg_id;
3993                 list_add_tail(&res->list, &rqp->mcg_list);
3994                 err = 0;
3995         }
3996         spin_unlock_irq(&rqp->mcg_spl);
3997
3998         return err;
3999 }
4000
4001 static int rem_mcg_res(struct mlx4_dev *dev, int slave, struct res_qp *rqp,
4002                        u8 *gid, enum mlx4_protocol prot,
4003                        enum mlx4_steer_type steer, u64 *reg_id)
4004 {
4005         struct res_gid *res;
4006         int err;
4007
4008         spin_lock_irq(&rqp->mcg_spl);
4009         res = find_gid(dev, slave, rqp, gid);
4010         if (!res || res->prot != prot || res->steer != steer)
4011                 err = -EINVAL;
4012         else {
4013                 *reg_id = res->reg_id;
4014                 list_del(&res->list);
4015                 kfree(res);
4016                 err = 0;
4017         }
4018         spin_unlock_irq(&rqp->mcg_spl);
4019
4020         return err;
4021 }
4022
4023 static int qp_attach(struct mlx4_dev *dev, int slave, struct mlx4_qp *qp,
4024                      u8 gid[16], int block_loopback, enum mlx4_protocol prot,
4025                      enum mlx4_steer_type type, u64 *reg_id)
4026 {
4027         switch (dev->caps.steering_mode) {
4028         case MLX4_STEERING_MODE_DEVICE_MANAGED: {
4029                 int port = mlx4_slave_convert_port(dev, slave, gid[5]);
4030                 if (port < 0)
4031                         return port;
4032                 return mlx4_trans_to_dmfs_attach(dev, qp, gid, port,
4033                                                 block_loopback, prot,
4034                                                 reg_id);
4035         }
4036         case MLX4_STEERING_MODE_B0:
4037                 if (prot == MLX4_PROT_ETH) {
4038                         int port = mlx4_slave_convert_port(dev, slave, gid[5]);
4039                         if (port < 0)
4040                                 return port;
4041                         gid[5] = port;
4042                 }
4043                 return mlx4_qp_attach_common(dev, qp, gid,
4044                                             block_loopback, prot, type);
4045         default:
4046                 return -EINVAL;
4047         }
4048 }
4049
4050 static int qp_detach(struct mlx4_dev *dev, struct mlx4_qp *qp,
4051                      u8 gid[16], enum mlx4_protocol prot,
4052                      enum mlx4_steer_type type, u64 reg_id)
4053 {
4054         switch (dev->caps.steering_mode) {
4055         case MLX4_STEERING_MODE_DEVICE_MANAGED:
4056                 return mlx4_flow_detach(dev, reg_id);
4057         case MLX4_STEERING_MODE_B0:
4058                 return mlx4_qp_detach_common(dev, qp, gid, prot, type);
4059         default:
4060                 return -EINVAL;
4061         }
4062 }
4063
4064 static int mlx4_adjust_port(struct mlx4_dev *dev, int slave,
4065                             u8 *gid, enum mlx4_protocol prot)
4066 {
4067         int real_port;
4068
4069         if (prot != MLX4_PROT_ETH)
4070                 return 0;
4071
4072         if (dev->caps.steering_mode == MLX4_STEERING_MODE_B0 ||
4073             dev->caps.steering_mode == MLX4_STEERING_MODE_DEVICE_MANAGED) {
4074                 real_port = mlx4_slave_convert_port(dev, slave, gid[5]);
4075                 if (real_port < 0)
4076                         return -EINVAL;
4077                 gid[5] = real_port;
4078         }
4079
4080         return 0;
4081 }
4082
4083 int mlx4_QP_ATTACH_wrapper(struct mlx4_dev *dev, int slave,
4084                                struct mlx4_vhcr *vhcr,
4085                                struct mlx4_cmd_mailbox *inbox,
4086                                struct mlx4_cmd_mailbox *outbox,
4087                                struct mlx4_cmd_info *cmd)
4088 {
4089         struct mlx4_qp qp; /* dummy for calling attach/detach */
4090         u8 *gid = inbox->buf;
4091         enum mlx4_protocol prot = (vhcr->in_modifier >> 28) & 0x7;
4092         int err;
4093         int qpn;
4094         struct res_qp *rqp;
4095         u64 reg_id = 0;
4096         int attach = vhcr->op_modifier;
4097         int block_loopback = vhcr->in_modifier >> 31;
4098         u8 steer_type_mask = 2;
4099         enum mlx4_steer_type type = (gid[7] & steer_type_mask) >> 1;
4100
4101         qpn = vhcr->in_modifier & 0xffffff;
4102         err = get_res(dev, slave, qpn, RES_QP, &rqp);
4103         if (err)
4104                 return err;
4105
4106         qp.qpn = qpn;
4107         if (attach) {
4108                 err = qp_attach(dev, slave, &qp, gid, block_loopback, prot,
4109                                 type, &reg_id);
4110                 if (err) {
4111                         pr_err("Fail to attach rule to qp 0x%x\n", qpn);
4112                         goto ex_put;
4113                 }
4114                 err = add_mcg_res(dev, slave, rqp, gid, prot, type, reg_id);
4115                 if (err)
4116                         goto ex_detach;
4117         } else {
4118                 err = mlx4_adjust_port(dev, slave, gid, prot);
4119                 if (err)
4120                         goto ex_put;
4121
4122                 err = rem_mcg_res(dev, slave, rqp, gid, prot, type, &reg_id);
4123                 if (err)
4124                         goto ex_put;
4125
4126                 err = qp_detach(dev, &qp, gid, prot, type, reg_id);
4127                 if (err)
4128                         pr_err("Fail to detach rule from qp 0x%x reg_id = 0x%llx\n",
4129                                qpn, reg_id);
4130         }
4131         put_res(dev, slave, qpn, RES_QP);
4132         return err;
4133
4134 ex_detach:
4135         qp_detach(dev, &qp, gid, prot, type, reg_id);
4136 ex_put:
4137         put_res(dev, slave, qpn, RES_QP);
4138         return err;
4139 }
4140
4141 /*
4142  * MAC validation for Flow Steering rules.
4143  * VF can attach rules only with a mac address which is assigned to it.
4144  */
4145 static int validate_eth_header_mac(int slave, struct _rule_hw *eth_header,
4146                                    struct list_head *rlist)
4147 {
4148         struct mac_res *res, *tmp;
4149         __be64 be_mac;
4150
4151         /* make sure it isn't multicast or broadcast mac*/
4152         if (!is_multicast_ether_addr(eth_header->eth.dst_mac) &&
4153             !is_broadcast_ether_addr(eth_header->eth.dst_mac)) {
4154                 list_for_each_entry_safe(res, tmp, rlist, list) {
4155                         be_mac = cpu_to_be64(res->mac << 16);
4156                         if (ether_addr_equal((u8 *)&be_mac, eth_header->eth.dst_mac))
4157                                 return 0;
4158                 }
4159                 pr_err("MAC %pM doesn't belong to VF %d, Steering rule rejected\n",
4160                        eth_header->eth.dst_mac, slave);
4161                 return -EINVAL;
4162         }
4163         return 0;
4164 }
4165
4166 static void handle_eth_header_mcast_prio(struct mlx4_net_trans_rule_hw_ctrl *ctrl,
4167                                          struct _rule_hw *eth_header)
4168 {
4169         if (is_multicast_ether_addr(eth_header->eth.dst_mac) ||
4170             is_broadcast_ether_addr(eth_header->eth.dst_mac)) {
4171                 struct mlx4_net_trans_rule_hw_eth *eth =
4172                         (struct mlx4_net_trans_rule_hw_eth *)eth_header;
4173                 struct _rule_hw *next_rule = (struct _rule_hw *)(eth + 1);
4174                 bool last_rule = next_rule->size == 0 && next_rule->id == 0 &&
4175                         next_rule->rsvd == 0;
4176
4177                 if (last_rule)
4178                         ctrl->prio = cpu_to_be16(MLX4_DOMAIN_NIC);
4179         }
4180 }
4181
4182 /*
4183  * In case of missing eth header, append eth header with a MAC address
4184  * assigned to the VF.
4185  */
4186 static int add_eth_header(struct mlx4_dev *dev, int slave,
4187                           struct mlx4_cmd_mailbox *inbox,
4188                           struct list_head *rlist, int header_id)
4189 {
4190         struct mac_res *res, *tmp;
4191         u8 port;
4192         struct mlx4_net_trans_rule_hw_ctrl *ctrl;
4193         struct mlx4_net_trans_rule_hw_eth *eth_header;
4194         struct mlx4_net_trans_rule_hw_ipv4 *ip_header;
4195         struct mlx4_net_trans_rule_hw_tcp_udp *l4_header;
4196         __be64 be_mac = 0;
4197         __be64 mac_msk = cpu_to_be64(MLX4_MAC_MASK << 16);
4198
4199         ctrl = (struct mlx4_net_trans_rule_hw_ctrl *)inbox->buf;
4200         port = ctrl->port;
4201         eth_header = (struct mlx4_net_trans_rule_hw_eth *)(ctrl + 1);
4202
4203         /* Clear a space in the inbox for eth header */
4204         switch (header_id) {
4205         case MLX4_NET_TRANS_RULE_ID_IPV4:
4206                 ip_header =
4207                         (struct mlx4_net_trans_rule_hw_ipv4 *)(eth_header + 1);
4208                 memmove(ip_header, eth_header,
4209                         sizeof(*ip_header) + sizeof(*l4_header));
4210                 break;
4211         case MLX4_NET_TRANS_RULE_ID_TCP:
4212         case MLX4_NET_TRANS_RULE_ID_UDP:
4213                 l4_header = (struct mlx4_net_trans_rule_hw_tcp_udp *)
4214                             (eth_header + 1);
4215                 memmove(l4_header, eth_header, sizeof(*l4_header));
4216                 break;
4217         default:
4218                 return -EINVAL;
4219         }
4220         list_for_each_entry_safe(res, tmp, rlist, list) {
4221                 if (port == res->port) {
4222                         be_mac = cpu_to_be64(res->mac << 16);
4223                         break;
4224                 }
4225         }
4226         if (!be_mac) {
4227                 pr_err("Failed adding eth header to FS rule, Can't find matching MAC for port %d\n",
4228                        port);
4229                 return -EINVAL;
4230         }
4231
4232         memset(eth_header, 0, sizeof(*eth_header));
4233         eth_header->size = sizeof(*eth_header) >> 2;
4234         eth_header->id = cpu_to_be16(__sw_id_hw[MLX4_NET_TRANS_RULE_ID_ETH]);
4235         memcpy(eth_header->dst_mac, &be_mac, ETH_ALEN);
4236         memcpy(eth_header->dst_mac_msk, &mac_msk, ETH_ALEN);
4237
4238         return 0;
4239
4240 }
4241
4242 #define MLX4_UPD_QP_PATH_MASK_SUPPORTED      (                                \
4243         1ULL << MLX4_UPD_QP_PATH_MASK_MAC_INDEX                     |\
4244         1ULL << MLX4_UPD_QP_PATH_MASK_ETH_SRC_CHECK_MC_LB)
4245 int mlx4_UPDATE_QP_wrapper(struct mlx4_dev *dev, int slave,
4246                            struct mlx4_vhcr *vhcr,
4247                            struct mlx4_cmd_mailbox *inbox,
4248                            struct mlx4_cmd_mailbox *outbox,
4249                            struct mlx4_cmd_info *cmd_info)
4250 {
4251         int err;
4252         u32 qpn = vhcr->in_modifier & 0xffffff;
4253         struct res_qp *rqp;
4254         u64 mac;
4255         unsigned port;
4256         u64 pri_addr_path_mask;
4257         struct mlx4_update_qp_context *cmd;
4258         int smac_index;
4259
4260         cmd = (struct mlx4_update_qp_context *)inbox->buf;
4261
4262         pri_addr_path_mask = be64_to_cpu(cmd->primary_addr_path_mask);
4263         if (cmd->qp_mask || cmd->secondary_addr_path_mask ||
4264             (pri_addr_path_mask & ~MLX4_UPD_QP_PATH_MASK_SUPPORTED))
4265                 return -EPERM;
4266
4267         if ((pri_addr_path_mask &
4268              (1ULL << MLX4_UPD_QP_PATH_MASK_ETH_SRC_CHECK_MC_LB)) &&
4269                 !(dev->caps.flags2 &
4270                   MLX4_DEV_CAP_FLAG2_UPDATE_QP_SRC_CHECK_LB)) {
4271                 mlx4_warn(dev, "Src check LB for slave %d isn't supported\n",
4272                           slave);
4273                 return -ENOTSUPP;
4274         }
4275
4276         /* Just change the smac for the QP */
4277         err = get_res(dev, slave, qpn, RES_QP, &rqp);
4278         if (err) {
4279                 mlx4_err(dev, "Updating qpn 0x%x for slave %d rejected\n", qpn, slave);
4280                 return err;
4281         }
4282
4283         port = (rqp->sched_queue >> 6 & 1) + 1;
4284
4285         if (pri_addr_path_mask & (1ULL << MLX4_UPD_QP_PATH_MASK_MAC_INDEX)) {
4286                 smac_index = cmd->qp_context.pri_path.grh_mylmc;
4287                 err = mac_find_smac_ix_in_slave(dev, slave, port,
4288                                                 smac_index, &mac);
4289
4290                 if (err) {
4291                         mlx4_err(dev, "Failed to update qpn 0x%x, MAC is invalid. smac_ix: %d\n",
4292                                  qpn, smac_index);
4293                         goto err_mac;
4294                 }
4295         }
4296
4297         err = mlx4_cmd(dev, inbox->dma,
4298                        vhcr->in_modifier, 0,
4299                        MLX4_CMD_UPDATE_QP, MLX4_CMD_TIME_CLASS_A,
4300                        MLX4_CMD_NATIVE);
4301         if (err) {
4302                 mlx4_err(dev, "Failed to update qpn on qpn 0x%x, command failed\n", qpn);
4303                 goto err_mac;
4304         }
4305
4306 err_mac:
4307         put_res(dev, slave, qpn, RES_QP);
4308         return err;
4309 }
4310
4311 static u32 qp_attach_mbox_size(void *mbox)
4312 {
4313         u32 size = sizeof(struct mlx4_net_trans_rule_hw_ctrl);
4314         struct _rule_hw  *rule_header;
4315
4316         rule_header = (struct _rule_hw *)(mbox + size);
4317
4318         while (rule_header->size) {
4319                 size += rule_header->size * sizeof(u32);
4320                 rule_header += 1;
4321         }
4322         return size;
4323 }
4324
4325 static int mlx4_do_mirror_rule(struct mlx4_dev *dev, struct res_fs_rule *fs_rule);
4326
4327 int mlx4_QP_FLOW_STEERING_ATTACH_wrapper(struct mlx4_dev *dev, int slave,
4328                                          struct mlx4_vhcr *vhcr,
4329                                          struct mlx4_cmd_mailbox *inbox,
4330                                          struct mlx4_cmd_mailbox *outbox,
4331                                          struct mlx4_cmd_info *cmd)
4332 {
4333
4334         struct mlx4_priv *priv = mlx4_priv(dev);
4335         struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker;
4336         struct list_head *rlist = &tracker->slave_list[slave].res_list[RES_MAC];
4337         int err;
4338         int qpn;
4339         struct res_qp *rqp;
4340         struct mlx4_net_trans_rule_hw_ctrl *ctrl;
4341         struct _rule_hw  *rule_header;
4342         int header_id;
4343         struct res_fs_rule *rrule;
4344         u32 mbox_size;
4345
4346         if (dev->caps.steering_mode !=
4347             MLX4_STEERING_MODE_DEVICE_MANAGED)
4348                 return -EOPNOTSUPP;
4349
4350         ctrl = (struct mlx4_net_trans_rule_hw_ctrl *)inbox->buf;
4351         err = mlx4_slave_convert_port(dev, slave, ctrl->port);
4352         if (err <= 0)
4353                 return -EINVAL;
4354         ctrl->port = err;
4355         qpn = be32_to_cpu(ctrl->qpn) & 0xffffff;
4356         err = get_res(dev, slave, qpn, RES_QP, &rqp);
4357         if (err) {
4358                 pr_err("Steering rule with qpn 0x%x rejected\n", qpn);
4359                 return err;
4360         }
4361         rule_header = (struct _rule_hw *)(ctrl + 1);
4362         header_id = map_hw_to_sw_id(be16_to_cpu(rule_header->id));
4363
4364         if (header_id == MLX4_NET_TRANS_RULE_ID_ETH)
4365                 handle_eth_header_mcast_prio(ctrl, rule_header);
4366
4367         if (slave == dev->caps.function)
4368                 goto execute;
4369
4370         switch (header_id) {
4371         case MLX4_NET_TRANS_RULE_ID_ETH:
4372                 if (validate_eth_header_mac(slave, rule_header, rlist)) {
4373                         err = -EINVAL;
4374                         goto err_put_qp;
4375                 }
4376                 break;
4377         case MLX4_NET_TRANS_RULE_ID_IB:
4378                 break;
4379         case MLX4_NET_TRANS_RULE_ID_IPV4:
4380         case MLX4_NET_TRANS_RULE_ID_TCP:
4381         case MLX4_NET_TRANS_RULE_ID_UDP:
4382                 pr_warn("Can't attach FS rule without L2 headers, adding L2 header\n");
4383                 if (add_eth_header(dev, slave, inbox, rlist, header_id)) {
4384                         err = -EINVAL;
4385                         goto err_put_qp;
4386                 }
4387                 vhcr->in_modifier +=
4388                         sizeof(struct mlx4_net_trans_rule_hw_eth) >> 2;
4389                 break;
4390         default:
4391                 pr_err("Corrupted mailbox\n");
4392                 err = -EINVAL;
4393                 goto err_put_qp;
4394         }
4395
4396 execute:
4397         err = mlx4_cmd_imm(dev, inbox->dma, &vhcr->out_param,
4398                            vhcr->in_modifier, 0,
4399                            MLX4_QP_FLOW_STEERING_ATTACH, MLX4_CMD_TIME_CLASS_A,
4400                            MLX4_CMD_NATIVE);
4401         if (err)
4402                 goto err_put_qp;
4403
4404
4405         err = add_res_range(dev, slave, vhcr->out_param, 1, RES_FS_RULE, qpn);
4406         if (err) {
4407                 mlx4_err(dev, "Fail to add flow steering resources\n");
4408                 goto err_detach;
4409         }
4410
4411         err = get_res(dev, slave, vhcr->out_param, RES_FS_RULE, &rrule);
4412         if (err)
4413                 goto err_detach;
4414
4415         mbox_size = qp_attach_mbox_size(inbox->buf);
4416         rrule->mirr_mbox = kmalloc(mbox_size, GFP_KERNEL);
4417         if (!rrule->mirr_mbox) {
4418                 err = -ENOMEM;
4419                 goto err_put_rule;
4420         }
4421         rrule->mirr_mbox_size = mbox_size;
4422         rrule->mirr_rule_id = 0;
4423         memcpy(rrule->mirr_mbox, inbox->buf, mbox_size);
4424
4425         /* set different port */
4426         ctrl = (struct mlx4_net_trans_rule_hw_ctrl *)rrule->mirr_mbox;
4427         if (ctrl->port == 1)
4428                 ctrl->port = 2;
4429         else
4430                 ctrl->port = 1;
4431
4432         if (mlx4_is_bonded(dev))
4433                 mlx4_do_mirror_rule(dev, rrule);
4434
4435         atomic_inc(&rqp->ref_count);
4436
4437 err_put_rule:
4438         put_res(dev, slave, vhcr->out_param, RES_FS_RULE);
4439 err_detach:
4440         /* detach rule on error */
4441         if (err)
4442                 mlx4_cmd(dev, vhcr->out_param, 0, 0,
4443                          MLX4_QP_FLOW_STEERING_DETACH, MLX4_CMD_TIME_CLASS_A,
4444                          MLX4_CMD_NATIVE);
4445 err_put_qp:
4446         put_res(dev, slave, qpn, RES_QP);
4447         return err;
4448 }
4449
4450 static int mlx4_undo_mirror_rule(struct mlx4_dev *dev, struct res_fs_rule *fs_rule)
4451 {
4452         int err;
4453
4454         err = rem_res_range(dev, fs_rule->com.owner, fs_rule->com.res_id, 1, RES_FS_RULE, 0);
4455         if (err) {
4456                 mlx4_err(dev, "Fail to remove flow steering resources\n");
4457                 return err;
4458         }
4459
4460         mlx4_cmd(dev, fs_rule->com.res_id, 0, 0, MLX4_QP_FLOW_STEERING_DETACH,
4461                  MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE);
4462         return 0;
4463 }
4464
4465 int mlx4_QP_FLOW_STEERING_DETACH_wrapper(struct mlx4_dev *dev, int slave,
4466                                          struct mlx4_vhcr *vhcr,
4467                                          struct mlx4_cmd_mailbox *inbox,
4468                                          struct mlx4_cmd_mailbox *outbox,
4469                                          struct mlx4_cmd_info *cmd)
4470 {
4471         int err;
4472         struct res_qp *rqp;
4473         struct res_fs_rule *rrule;
4474         u64 mirr_reg_id;
4475
4476         if (dev->caps.steering_mode !=
4477             MLX4_STEERING_MODE_DEVICE_MANAGED)
4478                 return -EOPNOTSUPP;
4479
4480         err = get_res(dev, slave, vhcr->in_param, RES_FS_RULE, &rrule);
4481         if (err)
4482                 return err;
4483
4484         if (!rrule->mirr_mbox) {
4485                 mlx4_err(dev, "Mirror rules cannot be removed explicitly\n");
4486                 put_res(dev, slave, vhcr->in_param, RES_FS_RULE);
4487                 return -EINVAL;
4488         }
4489         mirr_reg_id = rrule->mirr_rule_id;
4490         kfree(rrule->mirr_mbox);
4491
4492         /* Release the rule form busy state before removal */
4493         put_res(dev, slave, vhcr->in_param, RES_FS_RULE);
4494         err = get_res(dev, slave, rrule->qpn, RES_QP, &rqp);
4495         if (err)
4496                 return err;
4497
4498         if (mirr_reg_id && mlx4_is_bonded(dev)) {
4499                 err = get_res(dev, slave, mirr_reg_id, RES_FS_RULE, &rrule);
4500                 if (err) {
4501                         mlx4_err(dev, "Fail to get resource of mirror rule\n");
4502                 } else {
4503                         put_res(dev, slave, mirr_reg_id, RES_FS_RULE);
4504                         mlx4_undo_mirror_rule(dev, rrule);
4505                 }
4506         }
4507         err = rem_res_range(dev, slave, vhcr->in_param, 1, RES_FS_RULE, 0);
4508         if (err) {
4509                 mlx4_err(dev, "Fail to remove flow steering resources\n");
4510                 goto out;
4511         }
4512
4513         err = mlx4_cmd(dev, vhcr->in_param, 0, 0,
4514                        MLX4_QP_FLOW_STEERING_DETACH, MLX4_CMD_TIME_CLASS_A,
4515                        MLX4_CMD_NATIVE);
4516         if (!err)
4517                 atomic_dec(&rqp->ref_count);
4518 out:
4519         put_res(dev, slave, rrule->qpn, RES_QP);
4520         return err;
4521 }
4522
4523 enum {
4524         BUSY_MAX_RETRIES = 10
4525 };
4526
4527 int mlx4_QUERY_IF_STAT_wrapper(struct mlx4_dev *dev, int slave,
4528                                struct mlx4_vhcr *vhcr,
4529                                struct mlx4_cmd_mailbox *inbox,
4530                                struct mlx4_cmd_mailbox *outbox,
4531                                struct mlx4_cmd_info *cmd)
4532 {
4533         int err;
4534         int index = vhcr->in_modifier & 0xffff;
4535
4536         err = get_res(dev, slave, index, RES_COUNTER, NULL);
4537         if (err)
4538                 return err;
4539
4540         err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd);
4541         put_res(dev, slave, index, RES_COUNTER);
4542         return err;
4543 }
4544
4545 static void detach_qp(struct mlx4_dev *dev, int slave, struct res_qp *rqp)
4546 {
4547         struct res_gid *rgid;
4548         struct res_gid *tmp;
4549         struct mlx4_qp qp; /* dummy for calling attach/detach */
4550
4551         list_for_each_entry_safe(rgid, tmp, &rqp->mcg_list, list) {
4552                 switch (dev->caps.steering_mode) {
4553                 case MLX4_STEERING_MODE_DEVICE_MANAGED:
4554                         mlx4_flow_detach(dev, rgid->reg_id);
4555                         break;
4556                 case MLX4_STEERING_MODE_B0:
4557                         qp.qpn = rqp->local_qpn;
4558                         (void) mlx4_qp_detach_common(dev, &qp, rgid->gid,
4559                                                      rgid->prot, rgid->steer);
4560                         break;
4561                 }
4562                 list_del(&rgid->list);
4563                 kfree(rgid);
4564         }
4565 }
4566
4567 static int _move_all_busy(struct mlx4_dev *dev, int slave,
4568                           enum mlx4_resource type, int print)
4569 {
4570         struct mlx4_priv *priv = mlx4_priv(dev);
4571         struct mlx4_resource_tracker *tracker =
4572                 &priv->mfunc.master.res_tracker;
4573         struct list_head *rlist = &tracker->slave_list[slave].res_list[type];
4574         struct res_common *r;
4575         struct res_common *tmp;
4576         int busy;
4577
4578         busy = 0;
4579         spin_lock_irq(mlx4_tlock(dev));
4580         list_for_each_entry_safe(r, tmp, rlist, list) {
4581                 if (r->owner == slave) {
4582                         if (!r->removing) {
4583                                 if (r->state == RES_ANY_BUSY) {
4584                                         if (print)
4585                                                 mlx4_dbg(dev,
4586                                                          "%s id 0x%llx is busy\n",
4587                                                           resource_str(type),
4588                                                           r->res_id);
4589                                         ++busy;
4590                                 } else {
4591                                         r->from_state = r->state;
4592                                         r->state = RES_ANY_BUSY;
4593                                         r->removing = 1;
4594                                 }
4595                         }
4596                 }
4597         }
4598         spin_unlock_irq(mlx4_tlock(dev));
4599
4600         return busy;
4601 }
4602
4603 static int move_all_busy(struct mlx4_dev *dev, int slave,
4604                          enum mlx4_resource type)
4605 {
4606         unsigned long begin;
4607         int busy;
4608
4609         begin = jiffies;
4610         do {
4611                 busy = _move_all_busy(dev, slave, type, 0);
4612                 if (time_after(jiffies, begin + 5 * HZ))
4613                         break;
4614                 if (busy)
4615                         cond_resched();
4616         } while (busy);
4617
4618         if (busy)
4619                 busy = _move_all_busy(dev, slave, type, 1);
4620
4621         return busy;
4622 }
4623 static void rem_slave_qps(struct mlx4_dev *dev, int slave)
4624 {
4625         struct mlx4_priv *priv = mlx4_priv(dev);
4626         struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker;
4627         struct list_head *qp_list =
4628                 &tracker->slave_list[slave].res_list[RES_QP];
4629         struct res_qp *qp;
4630         struct res_qp *tmp;
4631         int state;
4632         u64 in_param;
4633         int qpn;
4634         int err;
4635
4636         err = move_all_busy(dev, slave, RES_QP);
4637         if (err)
4638                 mlx4_warn(dev, "rem_slave_qps: Could not move all qps to busy for slave %d\n",
4639                           slave);
4640
4641         spin_lock_irq(mlx4_tlock(dev));
4642         list_for_each_entry_safe(qp, tmp, qp_list, com.list) {
4643                 spin_unlock_irq(mlx4_tlock(dev));
4644                 if (qp->com.owner == slave) {
4645                         qpn = qp->com.res_id;
4646                         detach_qp(dev, slave, qp);
4647                         state = qp->com.from_state;
4648                         while (state != 0) {
4649                                 switch (state) {
4650                                 case RES_QP_RESERVED:
4651                                         spin_lock_irq(mlx4_tlock(dev));
4652                                         rb_erase(&qp->com.node,
4653                                                  &tracker->res_tree[RES_QP]);
4654                                         list_del(&qp->com.list);
4655                                         spin_unlock_irq(mlx4_tlock(dev));
4656                                         if (!valid_reserved(dev, slave, qpn)) {
4657                                                 __mlx4_qp_release_range(dev, qpn, 1);
4658                                                 mlx4_release_resource(dev, slave,
4659                                                                       RES_QP, 1, 0);
4660                                         }
4661                                         kfree(qp);
4662                                         state = 0;
4663                                         break;
4664                                 case RES_QP_MAPPED:
4665                                         if (!valid_reserved(dev, slave, qpn))
4666                                                 __mlx4_qp_free_icm(dev, qpn);
4667                                         state = RES_QP_RESERVED;
4668                                         break;
4669                                 case RES_QP_HW:
4670                                         in_param = slave;
4671                                         err = mlx4_cmd(dev, in_param,
4672                                                        qp->local_qpn, 2,
4673                                                        MLX4_CMD_2RST_QP,
4674                                                        MLX4_CMD_TIME_CLASS_A,
4675                                                        MLX4_CMD_NATIVE);
4676                                         if (err)
4677                                                 mlx4_dbg(dev, "rem_slave_qps: failed to move slave %d qpn %d to reset\n",
4678                                                          slave, qp->local_qpn);
4679                                         atomic_dec(&qp->rcq->ref_count);
4680                                         atomic_dec(&qp->scq->ref_count);
4681                                         atomic_dec(&qp->mtt->ref_count);
4682                                         if (qp->srq)
4683                                                 atomic_dec(&qp->srq->ref_count);
4684                                         state = RES_QP_MAPPED;
4685                                         break;
4686                                 default:
4687                                         state = 0;
4688                                 }
4689                         }
4690                 }
4691                 spin_lock_irq(mlx4_tlock(dev));
4692         }
4693         spin_unlock_irq(mlx4_tlock(dev));
4694 }
4695
4696 static void rem_slave_srqs(struct mlx4_dev *dev, int slave)
4697 {
4698         struct mlx4_priv *priv = mlx4_priv(dev);
4699         struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker;
4700         struct list_head *srq_list =
4701                 &tracker->slave_list[slave].res_list[RES_SRQ];
4702         struct res_srq *srq;
4703         struct res_srq *tmp;
4704         int state;
4705         u64 in_param;
4706         LIST_HEAD(tlist);
4707         int srqn;
4708         int err;
4709
4710         err = move_all_busy(dev, slave, RES_SRQ);
4711         if (err)
4712                 mlx4_warn(dev, "rem_slave_srqs: Could not move all srqs - too busy for slave %d\n",
4713                           slave);
4714
4715         spin_lock_irq(mlx4_tlock(dev));
4716         list_for_each_entry_safe(srq, tmp, srq_list, com.list) {
4717                 spin_unlock_irq(mlx4_tlock(dev));
4718                 if (srq->com.owner == slave) {
4719                         srqn = srq->com.res_id;
4720                         state = srq->com.from_state;
4721                         while (state != 0) {
4722                                 switch (state) {
4723                                 case RES_SRQ_ALLOCATED:
4724                                         __mlx4_srq_free_icm(dev, srqn);
4725                                         spin_lock_irq(mlx4_tlock(dev));
4726                                         rb_erase(&srq->com.node,
4727                                                  &tracker->res_tree[RES_SRQ]);
4728                                         list_del(&srq->com.list);
4729                                         spin_unlock_irq(mlx4_tlock(dev));
4730                                         mlx4_release_resource(dev, slave,
4731                                                               RES_SRQ, 1, 0);
4732                                         kfree(srq);
4733                                         state = 0;
4734                                         break;
4735
4736                                 case RES_SRQ_HW:
4737                                         in_param = slave;
4738                                         err = mlx4_cmd(dev, in_param, srqn, 1,
4739                                                        MLX4_CMD_HW2SW_SRQ,
4740                                                        MLX4_CMD_TIME_CLASS_A,
4741                                                        MLX4_CMD_NATIVE);
4742                                         if (err)
4743                                                 mlx4_dbg(dev, "rem_slave_srqs: failed to move slave %d srq %d to SW ownership\n",
4744                                                          slave, srqn);
4745
4746                                         atomic_dec(&srq->mtt->ref_count);
4747                                         if (srq->cq)
4748                                                 atomic_dec(&srq->cq->ref_count);
4749                                         state = RES_SRQ_ALLOCATED;
4750                                         break;
4751
4752                                 default:
4753                                         state = 0;
4754                                 }
4755                         }
4756                 }
4757                 spin_lock_irq(mlx4_tlock(dev));
4758         }
4759         spin_unlock_irq(mlx4_tlock(dev));
4760 }
4761
4762 static void rem_slave_cqs(struct mlx4_dev *dev, int slave)
4763 {
4764         struct mlx4_priv *priv = mlx4_priv(dev);
4765         struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker;
4766         struct list_head *cq_list =
4767                 &tracker->slave_list[slave].res_list[RES_CQ];
4768         struct res_cq *cq;
4769         struct res_cq *tmp;
4770         int state;
4771         u64 in_param;
4772         LIST_HEAD(tlist);
4773         int cqn;
4774         int err;
4775
4776         err = move_all_busy(dev, slave, RES_CQ);
4777         if (err)
4778                 mlx4_warn(dev, "rem_slave_cqs: Could not move all cqs - too busy for slave %d\n",
4779                           slave);
4780
4781         spin_lock_irq(mlx4_tlock(dev));
4782         list_for_each_entry_safe(cq, tmp, cq_list, com.list) {
4783                 spin_unlock_irq(mlx4_tlock(dev));
4784                 if (cq->com.owner == slave && !atomic_read(&cq->ref_count)) {
4785                         cqn = cq->com.res_id;
4786                         state = cq->com.from_state;
4787                         while (state != 0) {
4788                                 switch (state) {
4789                                 case RES_CQ_ALLOCATED:
4790                                         __mlx4_cq_free_icm(dev, cqn);
4791                                         spin_lock_irq(mlx4_tlock(dev));
4792                                         rb_erase(&cq->com.node,
4793                                                  &tracker->res_tree[RES_CQ]);
4794                                         list_del(&cq->com.list);
4795                                         spin_unlock_irq(mlx4_tlock(dev));
4796                                         mlx4_release_resource(dev, slave,
4797                                                               RES_CQ, 1, 0);
4798                                         kfree(cq);
4799                                         state = 0;
4800                                         break;
4801
4802                                 case RES_CQ_HW:
4803                                         in_param = slave;
4804                                         err = mlx4_cmd(dev, in_param, cqn, 1,
4805                                                        MLX4_CMD_HW2SW_CQ,
4806                                                        MLX4_CMD_TIME_CLASS_A,
4807                                                        MLX4_CMD_NATIVE);
4808                                         if (err)
4809                                                 mlx4_dbg(dev, "rem_slave_cqs: failed to move slave %d cq %d to SW ownership\n",
4810                                                          slave, cqn);
4811                                         atomic_dec(&cq->mtt->ref_count);
4812                                         state = RES_CQ_ALLOCATED;
4813                                         break;
4814
4815                                 default:
4816                                         state = 0;
4817                                 }
4818                         }
4819                 }
4820                 spin_lock_irq(mlx4_tlock(dev));
4821         }
4822         spin_unlock_irq(mlx4_tlock(dev));
4823 }
4824
4825 static void rem_slave_mrs(struct mlx4_dev *dev, int slave)
4826 {
4827         struct mlx4_priv *priv = mlx4_priv(dev);
4828         struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker;
4829         struct list_head *mpt_list =
4830                 &tracker->slave_list[slave].res_list[RES_MPT];
4831         struct res_mpt *mpt;
4832         struct res_mpt *tmp;
4833         int state;
4834         u64 in_param;
4835         LIST_HEAD(tlist);
4836         int mptn;
4837         int err;
4838
4839         err = move_all_busy(dev, slave, RES_MPT);
4840         if (err)
4841                 mlx4_warn(dev, "rem_slave_mrs: Could not move all mpts - too busy for slave %d\n",
4842                           slave);
4843
4844         spin_lock_irq(mlx4_tlock(dev));
4845         list_for_each_entry_safe(mpt, tmp, mpt_list, com.list) {
4846                 spin_unlock_irq(mlx4_tlock(dev));
4847                 if (mpt->com.owner == slave) {
4848                         mptn = mpt->com.res_id;
4849                         state = mpt->com.from_state;
4850                         while (state != 0) {
4851                                 switch (state) {
4852                                 case RES_MPT_RESERVED:
4853                                         __mlx4_mpt_release(dev, mpt->key);
4854                                         spin_lock_irq(mlx4_tlock(dev));
4855                                         rb_erase(&mpt->com.node,
4856                                                  &tracker->res_tree[RES_MPT]);
4857                                         list_del(&mpt->com.list);
4858                                         spin_unlock_irq(mlx4_tlock(dev));
4859                                         mlx4_release_resource(dev, slave,
4860                                                               RES_MPT, 1, 0);
4861                                         kfree(mpt);
4862                                         state = 0;
4863                                         break;
4864
4865                                 case RES_MPT_MAPPED:
4866                                         __mlx4_mpt_free_icm(dev, mpt->key);
4867                                         state = RES_MPT_RESERVED;
4868                                         break;
4869
4870                                 case RES_MPT_HW:
4871                                         in_param = slave;
4872                                         err = mlx4_cmd(dev, in_param, mptn, 0,
4873                                                      MLX4_CMD_HW2SW_MPT,
4874                                                      MLX4_CMD_TIME_CLASS_A,
4875                                                      MLX4_CMD_NATIVE);
4876                                         if (err)
4877                                                 mlx4_dbg(dev, "rem_slave_mrs: failed to move slave %d mpt %d to SW ownership\n",
4878                                                          slave, mptn);
4879                                         if (mpt->mtt)
4880                                                 atomic_dec(&mpt->mtt->ref_count);
4881                                         state = RES_MPT_MAPPED;
4882                                         break;
4883                                 default:
4884                                         state = 0;
4885                                 }
4886                         }
4887                 }
4888                 spin_lock_irq(mlx4_tlock(dev));
4889         }
4890         spin_unlock_irq(mlx4_tlock(dev));
4891 }
4892
4893 static void rem_slave_mtts(struct mlx4_dev *dev, int slave)
4894 {
4895         struct mlx4_priv *priv = mlx4_priv(dev);
4896         struct mlx4_resource_tracker *tracker =
4897                 &priv->mfunc.master.res_tracker;
4898         struct list_head *mtt_list =
4899                 &tracker->slave_list[slave].res_list[RES_MTT];
4900         struct res_mtt *mtt;
4901         struct res_mtt *tmp;
4902         int state;
4903         LIST_HEAD(tlist);
4904         int base;
4905         int err;
4906
4907         err = move_all_busy(dev, slave, RES_MTT);
4908         if (err)
4909                 mlx4_warn(dev, "rem_slave_mtts: Could not move all mtts  - too busy for slave %d\n",
4910                           slave);
4911
4912         spin_lock_irq(mlx4_tlock(dev));
4913         list_for_each_entry_safe(mtt, tmp, mtt_list, com.list) {
4914                 spin_unlock_irq(mlx4_tlock(dev));
4915                 if (mtt->com.owner == slave) {
4916                         base = mtt->com.res_id;
4917                         state = mtt->com.from_state;
4918                         while (state != 0) {
4919                                 switch (state) {
4920                                 case RES_MTT_ALLOCATED:
4921                                         __mlx4_free_mtt_range(dev, base,
4922                                                               mtt->order);
4923                                         spin_lock_irq(mlx4_tlock(dev));
4924                                         rb_erase(&mtt->com.node,
4925                                                  &tracker->res_tree[RES_MTT]);
4926                                         list_del(&mtt->com.list);
4927                                         spin_unlock_irq(mlx4_tlock(dev));
4928                                         mlx4_release_resource(dev, slave, RES_MTT,
4929                                                               1 << mtt->order, 0);
4930                                         kfree(mtt);
4931                                         state = 0;
4932                                         break;
4933
4934                                 default:
4935                                         state = 0;
4936                                 }
4937                         }
4938                 }
4939                 spin_lock_irq(mlx4_tlock(dev));
4940         }
4941         spin_unlock_irq(mlx4_tlock(dev));
4942 }
4943
4944 static int mlx4_do_mirror_rule(struct mlx4_dev *dev, struct res_fs_rule *fs_rule)
4945 {
4946         struct mlx4_cmd_mailbox *mailbox;
4947         int err;
4948         struct res_fs_rule *mirr_rule;
4949         u64 reg_id;
4950
4951         mailbox = mlx4_alloc_cmd_mailbox(dev);
4952         if (IS_ERR(mailbox))
4953                 return PTR_ERR(mailbox);
4954
4955         if (!fs_rule->mirr_mbox) {
4956                 mlx4_err(dev, "rule mirroring mailbox is null\n");
4957                 return -EINVAL;
4958         }
4959         memcpy(mailbox->buf, fs_rule->mirr_mbox, fs_rule->mirr_mbox_size);
4960         err = mlx4_cmd_imm(dev, mailbox->dma, &reg_id, fs_rule->mirr_mbox_size >> 2, 0,
4961                            MLX4_QP_FLOW_STEERING_ATTACH, MLX4_CMD_TIME_CLASS_A,
4962                            MLX4_CMD_NATIVE);
4963         mlx4_free_cmd_mailbox(dev, mailbox);
4964
4965         if (err)
4966                 goto err;
4967
4968         err = add_res_range(dev, fs_rule->com.owner, reg_id, 1, RES_FS_RULE, fs_rule->qpn);
4969         if (err)
4970                 goto err_detach;
4971
4972         err = get_res(dev, fs_rule->com.owner, reg_id, RES_FS_RULE, &mirr_rule);
4973         if (err)
4974                 goto err_rem;
4975
4976         fs_rule->mirr_rule_id = reg_id;
4977         mirr_rule->mirr_rule_id = 0;
4978         mirr_rule->mirr_mbox_size = 0;
4979         mirr_rule->mirr_mbox = NULL;
4980         put_res(dev, fs_rule->com.owner, reg_id, RES_FS_RULE);
4981
4982         return 0;
4983 err_rem:
4984         rem_res_range(dev, fs_rule->com.owner, reg_id, 1, RES_FS_RULE, 0);
4985 err_detach:
4986         mlx4_cmd(dev, reg_id, 0, 0, MLX4_QP_FLOW_STEERING_DETACH,
4987                  MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE);
4988 err:
4989         return err;
4990 }
4991
4992 static int mlx4_mirror_fs_rules(struct mlx4_dev *dev, bool bond)
4993 {
4994         struct mlx4_priv *priv = mlx4_priv(dev);
4995         struct mlx4_resource_tracker *tracker =
4996                 &priv->mfunc.master.res_tracker;
4997         struct rb_root *root = &tracker->res_tree[RES_FS_RULE];
4998         struct rb_node *p;
4999         struct res_fs_rule *fs_rule;
5000         int err = 0;
5001         LIST_HEAD(mirr_list);
5002
5003         for (p = rb_first(root); p; p = rb_next(p)) {
5004                 fs_rule = rb_entry(p, struct res_fs_rule, com.node);
5005                 if ((bond && fs_rule->mirr_mbox_size) ||
5006                     (!bond && !fs_rule->mirr_mbox_size))
5007                         list_add_tail(&fs_rule->mirr_list, &mirr_list);
5008         }
5009
5010         list_for_each_entry(fs_rule, &mirr_list, mirr_list) {
5011                 if (bond)
5012                         err += mlx4_do_mirror_rule(dev, fs_rule);
5013                 else
5014                         err += mlx4_undo_mirror_rule(dev, fs_rule);
5015         }
5016         return err;
5017 }
5018
5019 int mlx4_bond_fs_rules(struct mlx4_dev *dev)
5020 {
5021         return mlx4_mirror_fs_rules(dev, true);
5022 }
5023
5024 int mlx4_unbond_fs_rules(struct mlx4_dev *dev)
5025 {
5026         return mlx4_mirror_fs_rules(dev, false);
5027 }
5028
5029 static void rem_slave_fs_rule(struct mlx4_dev *dev, int slave)
5030 {
5031         struct mlx4_priv *priv = mlx4_priv(dev);
5032         struct mlx4_resource_tracker *tracker =
5033                 &priv->mfunc.master.res_tracker;
5034         struct list_head *fs_rule_list =
5035                 &tracker->slave_list[slave].res_list[RES_FS_RULE];
5036         struct res_fs_rule *fs_rule;
5037         struct res_fs_rule *tmp;
5038         int state;
5039         u64 base;
5040         int err;
5041
5042         err = move_all_busy(dev, slave, RES_FS_RULE);
5043         if (err)
5044                 mlx4_warn(dev, "rem_slave_fs_rule: Could not move all mtts to busy for slave %d\n",
5045                           slave);
5046
5047         spin_lock_irq(mlx4_tlock(dev));
5048         list_for_each_entry_safe(fs_rule, tmp, fs_rule_list, com.list) {
5049                 spin_unlock_irq(mlx4_tlock(dev));
5050                 if (fs_rule->com.owner == slave) {
5051                         base = fs_rule->com.res_id;
5052                         state = fs_rule->com.from_state;
5053                         while (state != 0) {
5054                                 switch (state) {
5055                                 case RES_FS_RULE_ALLOCATED:
5056                                         /* detach rule */
5057                                         err = mlx4_cmd(dev, base, 0, 0,
5058                                                        MLX4_QP_FLOW_STEERING_DETACH,
5059                                                        MLX4_CMD_TIME_CLASS_A,
5060                                                        MLX4_CMD_NATIVE);
5061
5062                                         spin_lock_irq(mlx4_tlock(dev));
5063                                         rb_erase(&fs_rule->com.node,
5064                                                  &tracker->res_tree[RES_FS_RULE]);
5065                                         list_del(&fs_rule->com.list);
5066                                         spin_unlock_irq(mlx4_tlock(dev));
5067                                         kfree(fs_rule);
5068                                         state = 0;
5069                                         break;
5070
5071                                 default:
5072                                         state = 0;
5073                                 }
5074                         }
5075                 }
5076                 spin_lock_irq(mlx4_tlock(dev));
5077         }
5078         spin_unlock_irq(mlx4_tlock(dev));
5079 }
5080
5081 static void rem_slave_eqs(struct mlx4_dev *dev, int slave)
5082 {
5083         struct mlx4_priv *priv = mlx4_priv(dev);
5084         struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker;
5085         struct list_head *eq_list =
5086                 &tracker->slave_list[slave].res_list[RES_EQ];
5087         struct res_eq *eq;
5088         struct res_eq *tmp;
5089         int err;
5090         int state;
5091         LIST_HEAD(tlist);
5092         int eqn;
5093
5094         err = move_all_busy(dev, slave, RES_EQ);
5095         if (err)
5096                 mlx4_warn(dev, "rem_slave_eqs: Could not move all eqs - too busy for slave %d\n",
5097                           slave);
5098
5099         spin_lock_irq(mlx4_tlock(dev));
5100         list_for_each_entry_safe(eq, tmp, eq_list, com.list) {
5101                 spin_unlock_irq(mlx4_tlock(dev));
5102                 if (eq->com.owner == slave) {
5103                         eqn = eq->com.res_id;
5104                         state = eq->com.from_state;
5105                         while (state != 0) {
5106                                 switch (state) {
5107                                 case RES_EQ_RESERVED:
5108                                         spin_lock_irq(mlx4_tlock(dev));
5109                                         rb_erase(&eq->com.node,
5110                                                  &tracker->res_tree[RES_EQ]);
5111                                         list_del(&eq->com.list);
5112                                         spin_unlock_irq(mlx4_tlock(dev));
5113                                         kfree(eq);
5114                                         state = 0;
5115                                         break;
5116
5117                                 case RES_EQ_HW:
5118                                         err = mlx4_cmd(dev, slave, eqn & 0x3ff,
5119                                                        1, MLX4_CMD_HW2SW_EQ,
5120                                                        MLX4_CMD_TIME_CLASS_A,
5121                                                        MLX4_CMD_NATIVE);
5122                                         if (err)
5123                                                 mlx4_dbg(dev, "rem_slave_eqs: failed to move slave %d eqs %d to SW ownership\n",
5124                                                          slave, eqn & 0x3ff);
5125                                         atomic_dec(&eq->mtt->ref_count);
5126                                         state = RES_EQ_RESERVED;
5127                                         break;
5128
5129                                 default:
5130                                         state = 0;
5131                                 }
5132                         }
5133                 }
5134                 spin_lock_irq(mlx4_tlock(dev));
5135         }
5136         spin_unlock_irq(mlx4_tlock(dev));
5137 }
5138
5139 static void rem_slave_counters(struct mlx4_dev *dev, int slave)
5140 {
5141         struct mlx4_priv *priv = mlx4_priv(dev);
5142         struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker;
5143         struct list_head *counter_list =
5144                 &tracker->slave_list[slave].res_list[RES_COUNTER];
5145         struct res_counter *counter;
5146         struct res_counter *tmp;
5147         int err;
5148         int *counters_arr = NULL;
5149         int i, j;
5150
5151         err = move_all_busy(dev, slave, RES_COUNTER);
5152         if (err)
5153                 mlx4_warn(dev, "rem_slave_counters: Could not move all counters - too busy for slave %d\n",
5154                           slave);
5155
5156         counters_arr = kmalloc_array(dev->caps.max_counters,
5157                                      sizeof(*counters_arr), GFP_KERNEL);
5158         if (!counters_arr)
5159                 return;
5160
5161         do {
5162                 i = 0;
5163                 j = 0;
5164                 spin_lock_irq(mlx4_tlock(dev));
5165                 list_for_each_entry_safe(counter, tmp, counter_list, com.list) {
5166                         if (counter->com.owner == slave) {
5167                                 counters_arr[i++] = counter->com.res_id;
5168                                 rb_erase(&counter->com.node,
5169                                          &tracker->res_tree[RES_COUNTER]);
5170                                 list_del(&counter->com.list);
5171                                 kfree(counter);
5172                         }
5173                 }
5174                 spin_unlock_irq(mlx4_tlock(dev));
5175
5176                 while (j < i) {
5177                         __mlx4_counter_free(dev, counters_arr[j++]);
5178                         mlx4_release_resource(dev, slave, RES_COUNTER, 1, 0);
5179                 }
5180         } while (i);
5181
5182         kfree(counters_arr);
5183 }
5184
5185 static void rem_slave_xrcdns(struct mlx4_dev *dev, int slave)
5186 {
5187         struct mlx4_priv *priv = mlx4_priv(dev);
5188         struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker;
5189         struct list_head *xrcdn_list =
5190                 &tracker->slave_list[slave].res_list[RES_XRCD];
5191         struct res_xrcdn *xrcd;
5192         struct res_xrcdn *tmp;
5193         int err;
5194         int xrcdn;
5195
5196         err = move_all_busy(dev, slave, RES_XRCD);
5197         if (err)
5198                 mlx4_warn(dev, "rem_slave_xrcdns: Could not move all xrcdns - too busy for slave %d\n",
5199                           slave);
5200
5201         spin_lock_irq(mlx4_tlock(dev));
5202         list_for_each_entry_safe(xrcd, tmp, xrcdn_list, com.list) {
5203                 if (xrcd->com.owner == slave) {
5204                         xrcdn = xrcd->com.res_id;
5205                         rb_erase(&xrcd->com.node, &tracker->res_tree[RES_XRCD]);
5206                         list_del(&xrcd->com.list);
5207                         kfree(xrcd);
5208                         __mlx4_xrcd_free(dev, xrcdn);
5209                 }
5210         }
5211         spin_unlock_irq(mlx4_tlock(dev));
5212 }
5213
5214 void mlx4_delete_all_resources_for_slave(struct mlx4_dev *dev, int slave)
5215 {
5216         struct mlx4_priv *priv = mlx4_priv(dev);
5217         mlx4_reset_roce_gids(dev, slave);
5218         mutex_lock(&priv->mfunc.master.res_tracker.slave_list[slave].mutex);
5219         rem_slave_vlans(dev, slave);
5220         rem_slave_macs(dev, slave);
5221         rem_slave_fs_rule(dev, slave);
5222         rem_slave_qps(dev, slave);
5223         rem_slave_srqs(dev, slave);
5224         rem_slave_cqs(dev, slave);
5225         rem_slave_mrs(dev, slave);
5226         rem_slave_eqs(dev, slave);
5227         rem_slave_mtts(dev, slave);
5228         rem_slave_counters(dev, slave);
5229         rem_slave_xrcdns(dev, slave);
5230         mutex_unlock(&priv->mfunc.master.res_tracker.slave_list[slave].mutex);
5231 }
5232
5233 void mlx4_vf_immed_vlan_work_handler(struct work_struct *_work)
5234 {
5235         struct mlx4_vf_immed_vlan_work *work =
5236                 container_of(_work, struct mlx4_vf_immed_vlan_work, work);
5237         struct mlx4_cmd_mailbox *mailbox;
5238         struct mlx4_update_qp_context *upd_context;
5239         struct mlx4_dev *dev = &work->priv->dev;
5240         struct mlx4_resource_tracker *tracker =
5241                 &work->priv->mfunc.master.res_tracker;
5242         struct list_head *qp_list =
5243                 &tracker->slave_list[work->slave].res_list[RES_QP];
5244         struct res_qp *qp;
5245         struct res_qp *tmp;
5246         u64 qp_path_mask_vlan_ctrl =
5247                        ((1ULL << MLX4_UPD_QP_PATH_MASK_ETH_TX_BLOCK_UNTAGGED) |
5248                        (1ULL << MLX4_UPD_QP_PATH_MASK_ETH_TX_BLOCK_1P) |
5249                        (1ULL << MLX4_UPD_QP_PATH_MASK_ETH_TX_BLOCK_TAGGED) |
5250                        (1ULL << MLX4_UPD_QP_PATH_MASK_ETH_RX_BLOCK_UNTAGGED) |
5251                        (1ULL << MLX4_UPD_QP_PATH_MASK_ETH_RX_BLOCK_1P) |
5252                        (1ULL << MLX4_UPD_QP_PATH_MASK_ETH_RX_BLOCK_TAGGED));
5253
5254         u64 qp_path_mask = ((1ULL << MLX4_UPD_QP_PATH_MASK_VLAN_INDEX) |
5255                        (1ULL << MLX4_UPD_QP_PATH_MASK_FVL) |
5256                        (1ULL << MLX4_UPD_QP_PATH_MASK_CV) |
5257                        (1ULL << MLX4_UPD_QP_PATH_MASK_SV) |
5258                        (1ULL << MLX4_UPD_QP_PATH_MASK_ETH_HIDE_CQE_VLAN) |
5259                        (1ULL << MLX4_UPD_QP_PATH_MASK_FEUP) |
5260                        (1ULL << MLX4_UPD_QP_PATH_MASK_FVL_RX) |
5261                        (1ULL << MLX4_UPD_QP_PATH_MASK_SCHED_QUEUE));
5262
5263         int err;
5264         int port, errors = 0;
5265         u8 vlan_control;
5266
5267         if (mlx4_is_slave(dev)) {
5268                 mlx4_warn(dev, "Trying to update-qp in slave %d\n",
5269                           work->slave);
5270                 goto out;
5271         }
5272
5273         mailbox = mlx4_alloc_cmd_mailbox(dev);
5274         if (IS_ERR(mailbox))
5275                 goto out;
5276         if (work->flags & MLX4_VF_IMMED_VLAN_FLAG_LINK_DISABLE) /* block all */
5277                 vlan_control = MLX4_VLAN_CTRL_ETH_TX_BLOCK_TAGGED |
5278                         MLX4_VLAN_CTRL_ETH_TX_BLOCK_PRIO_TAGGED |
5279                         MLX4_VLAN_CTRL_ETH_TX_BLOCK_UNTAGGED |
5280                         MLX4_VLAN_CTRL_ETH_RX_BLOCK_PRIO_TAGGED |
5281                         MLX4_VLAN_CTRL_ETH_RX_BLOCK_UNTAGGED |
5282                         MLX4_VLAN_CTRL_ETH_RX_BLOCK_TAGGED;
5283         else if (!work->vlan_id)
5284                 vlan_control = MLX4_VLAN_CTRL_ETH_TX_BLOCK_TAGGED |
5285                         MLX4_VLAN_CTRL_ETH_RX_BLOCK_TAGGED;
5286         else if (work->vlan_proto == htons(ETH_P_8021AD))
5287                 vlan_control = MLX4_VLAN_CTRL_ETH_TX_BLOCK_PRIO_TAGGED |
5288                         MLX4_VLAN_CTRL_ETH_TX_BLOCK_TAGGED |
5289                         MLX4_VLAN_CTRL_ETH_RX_BLOCK_PRIO_TAGGED |
5290                         MLX4_VLAN_CTRL_ETH_RX_BLOCK_UNTAGGED;
5291         else  /* vst 802.1Q */
5292                 vlan_control = MLX4_VLAN_CTRL_ETH_TX_BLOCK_TAGGED |
5293                         MLX4_VLAN_CTRL_ETH_RX_BLOCK_PRIO_TAGGED |
5294                         MLX4_VLAN_CTRL_ETH_RX_BLOCK_UNTAGGED;
5295
5296         upd_context = mailbox->buf;
5297         upd_context->qp_mask = cpu_to_be64(1ULL << MLX4_UPD_QP_MASK_VSD);
5298
5299         spin_lock_irq(mlx4_tlock(dev));
5300         list_for_each_entry_safe(qp, tmp, qp_list, com.list) {
5301                 spin_unlock_irq(mlx4_tlock(dev));
5302                 if (qp->com.owner == work->slave) {
5303                         if (qp->com.from_state != RES_QP_HW ||
5304                             !qp->sched_queue ||  /* no INIT2RTR trans yet */
5305                             mlx4_is_qp_reserved(dev, qp->local_qpn) ||
5306                             qp->qpc_flags & (1 << MLX4_RSS_QPC_FLAG_OFFSET)) {
5307                                 spin_lock_irq(mlx4_tlock(dev));
5308                                 continue;
5309                         }
5310                         port = (qp->sched_queue >> 6 & 1) + 1;
5311                         if (port != work->port) {
5312                                 spin_lock_irq(mlx4_tlock(dev));
5313                                 continue;
5314                         }
5315                         if (MLX4_QP_ST_RC == ((qp->qpc_flags >> 16) & 0xff))
5316                                 upd_context->primary_addr_path_mask = cpu_to_be64(qp_path_mask);
5317                         else
5318                                 upd_context->primary_addr_path_mask =
5319                                         cpu_to_be64(qp_path_mask | qp_path_mask_vlan_ctrl);
5320                         if (work->vlan_id == MLX4_VGT) {
5321                                 upd_context->qp_context.param3 = qp->param3;
5322                                 upd_context->qp_context.pri_path.vlan_control = qp->vlan_control;
5323                                 upd_context->qp_context.pri_path.fvl_rx = qp->fvl_rx;
5324                                 upd_context->qp_context.pri_path.vlan_index = qp->vlan_index;
5325                                 upd_context->qp_context.pri_path.fl = qp->pri_path_fl;
5326                                 upd_context->qp_context.pri_path.feup = qp->feup;
5327                                 upd_context->qp_context.pri_path.sched_queue =
5328                                         qp->sched_queue;
5329                         } else {
5330                                 upd_context->qp_context.param3 = qp->param3 & ~cpu_to_be32(MLX4_STRIP_VLAN);
5331                                 upd_context->qp_context.pri_path.vlan_control = vlan_control;
5332                                 upd_context->qp_context.pri_path.vlan_index = work->vlan_ix;
5333                                 upd_context->qp_context.pri_path.fvl_rx =
5334                                         qp->fvl_rx | MLX4_FVL_RX_FORCE_ETH_VLAN;
5335                                 upd_context->qp_context.pri_path.fl =
5336                                         qp->pri_path_fl | MLX4_FL_ETH_HIDE_CQE_VLAN;
5337                                 if (work->vlan_proto == htons(ETH_P_8021AD))
5338                                         upd_context->qp_context.pri_path.fl |= MLX4_FL_SV;
5339                                 else
5340                                         upd_context->qp_context.pri_path.fl |= MLX4_FL_CV;
5341                                 upd_context->qp_context.pri_path.feup =
5342                                         qp->feup | MLX4_FEUP_FORCE_ETH_UP | MLX4_FVL_FORCE_ETH_VLAN;
5343                                 upd_context->qp_context.pri_path.sched_queue =
5344                                         qp->sched_queue & 0xC7;
5345                                 upd_context->qp_context.pri_path.sched_queue |=
5346                                         ((work->qos & 0x7) << 3);
5347                                 upd_context->qp_mask |=
5348                                         cpu_to_be64(1ULL <<
5349                                                     MLX4_UPD_QP_MASK_QOS_VPP);
5350                                 upd_context->qp_context.qos_vport =
5351                                         work->qos_vport;
5352                         }
5353
5354                         err = mlx4_cmd(dev, mailbox->dma,
5355                                        qp->local_qpn & 0xffffff,
5356                                        0, MLX4_CMD_UPDATE_QP,
5357                                        MLX4_CMD_TIME_CLASS_C, MLX4_CMD_NATIVE);
5358                         if (err) {
5359                                 mlx4_info(dev, "UPDATE_QP failed for slave %d, port %d, qpn %d (%d)\n",
5360                                           work->slave, port, qp->local_qpn, err);
5361                                 errors++;
5362                         }
5363                 }
5364                 spin_lock_irq(mlx4_tlock(dev));
5365         }
5366         spin_unlock_irq(mlx4_tlock(dev));
5367         mlx4_free_cmd_mailbox(dev, mailbox);
5368
5369         if (errors)
5370                 mlx4_err(dev, "%d UPDATE_QP failures for slave %d, port %d\n",
5371                          errors, work->slave, work->port);
5372
5373         /* unregister previous vlan_id if needed and we had no errors
5374          * while updating the QPs
5375          */
5376         if (work->flags & MLX4_VF_IMMED_VLAN_FLAG_VLAN && !errors &&
5377             NO_INDX != work->orig_vlan_ix)
5378                 __mlx4_unregister_vlan(&work->priv->dev, work->port,
5379                                        work->orig_vlan_id);
5380 out:
5381         kfree(work);
5382         return;
5383 }