drivers/net/ethernet/mellanox/mlx5/core/health.c

   1 /*
   2  * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32
  33 #include <linux/kernel.h>
  34 #include <linux/module.h>
  35 #include <linux/random.h>
  36 #include <linux/vmalloc.h>
  37 #include <linux/hardirq.h>
  38 #include <linux/mlx5/driver.h>
  39 #include <linux/mlx5/cmd.h>
  40 #include "mlx5_core.h"
  41
  42 enum {
  43         MLX5_HEALTH_POLL_INTERVAL       = 2 * HZ,
  44         MAX_MISSES                      = 3,
  45 };
  46
  47 enum {
  48         MLX5_HEALTH_SYNDR_FW_ERR                = 0x1,
  49         MLX5_HEALTH_SYNDR_IRISC_ERR             = 0x7,
  50         MLX5_HEALTH_SYNDR_HW_UNRECOVERABLE_ERR  = 0x8,
  51         MLX5_HEALTH_SYNDR_CRC_ERR               = 0x9,
  52         MLX5_HEALTH_SYNDR_FETCH_PCI_ERR         = 0xa,
  53         MLX5_HEALTH_SYNDR_HW_FTL_ERR            = 0xb,
  54         MLX5_HEALTH_SYNDR_ASYNC_EQ_OVERRUN_ERR  = 0xc,
  55         MLX5_HEALTH_SYNDR_EQ_ERR                = 0xd,
  56         MLX5_HEALTH_SYNDR_EQ_INV                = 0xe,
  57         MLX5_HEALTH_SYNDR_FFSER_ERR             = 0xf,
  58         MLX5_HEALTH_SYNDR_HIGH_TEMP             = 0x10
  59 };
  60
  61 enum {
  62         MLX5_NIC_IFC_FULL               = 0,
  63         MLX5_NIC_IFC_DISABLED           = 1,
  64         MLX5_NIC_IFC_NO_DRAM_NIC        = 2
  65 };
  66
  67 enum {
  68         MLX5_DROP_NEW_HEALTH_WORK,
  69 };
  70
  71 static u8 get_nic_interface(struct mlx5_core_dev *dev)
  72 {
  73         return (ioread32be(&dev->iseg->cmdq_addr_l_sz) >> 8) & 3;
  74 }
  75
  76 static void trigger_cmd_completions(struct mlx5_core_dev *dev)
  77 {
  78         unsigned long flags;
  79         u64 vector;
  80
  81         /* wait for pending handlers to complete */
  82         synchronize_irq(dev->priv.msix_arr[MLX5_EQ_VEC_CMD].vector);
  83         spin_lock_irqsave(&dev->cmd.alloc_lock, flags);
  84         vector = ~dev->cmd.bitmask & ((1ul << (1 << dev->cmd.log_sz)) - 1);
  85         if (!vector)
  86                 goto no_trig;
  87
  88         vector |= MLX5_TRIGGERED_CMD_COMP;
  89         spin_unlock_irqrestore(&dev->cmd.alloc_lock, flags);
  90
  91         mlx5_core_dbg(dev, "vector 0x%llx\n", vector);
  92         mlx5_cmd_comp_handler(dev, vector);
  93         return;
  94
  95 no_trig:
  96         spin_unlock_irqrestore(&dev->cmd.alloc_lock, flags);
  97 }
  98
  99 static int in_fatal(struct mlx5_core_dev *dev)
 100 {
 101         struct mlx5_core_health *health = &dev->priv.health;
 102         struct health_buffer __iomem *h = health->health;
 103
 104         if (get_nic_interface(dev) == MLX5_NIC_IFC_DISABLED)
 105                 return 1;
 106
 107         if (ioread32be(&h->fw_ver) == 0xffffffff)
 108                 return 1;
 109
 110         return 0;
 111 }
 112
 113 void mlx5_enter_error_state(struct mlx5_core_dev *dev)
 114 {
 115         mutex_lock(&dev->intf_state_mutex);
 116         if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR)
 117                 goto unlock;
 118
 119         mlx5_core_err(dev, "start\n");
 120         if (pci_channel_offline(dev->pdev) || in_fatal(dev)) {
 121                 dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR;
 122                 trigger_cmd_completions(dev);
 123         }
 124
 125         mlx5_core_event(dev, MLX5_DEV_EVENT_SYS_ERROR, 0);
 126         mlx5_core_err(dev, "end\n");
 127
 128 unlock:
 129         mutex_unlock(&dev->intf_state_mutex);
 130 }
 131
 132 static void mlx5_handle_bad_state(struct mlx5_core_dev *dev)
 133 {
 134         u8 nic_interface = get_nic_interface(dev);
 135
 136         switch (nic_interface) {
 137         case MLX5_NIC_IFC_FULL:
 138                 mlx5_core_warn(dev, "Expected to see disabled NIC but it is full driver\n");
 139                 break;
 140
 141         case MLX5_NIC_IFC_DISABLED:
 142                 mlx5_core_warn(dev, "starting teardown\n");
 143                 break;
 144
 145         case MLX5_NIC_IFC_NO_DRAM_NIC:
 146                 mlx5_core_warn(dev, "Expected to see disabled NIC but it is no dram nic\n");
 147                 break;
 148         default:
 149                 mlx5_core_warn(dev, "Expected to see disabled NIC but it is has invalid value %d\n",
 150                                nic_interface);
 151         }
 152
 153         mlx5_disable_device(dev);
 154 }
 155
 156 static void health_care(struct work_struct *work)
 157 {
 158         struct mlx5_core_health *health;
 159         struct mlx5_core_dev *dev;
 160         struct mlx5_priv *priv;
 161
 162         health = container_of(work, struct mlx5_core_health, work);
 163         priv = container_of(health, struct mlx5_priv, health);
 164         dev = container_of(priv, struct mlx5_core_dev, priv);
 165         mlx5_core_warn(dev, "handling bad device here\n");
 166         mlx5_handle_bad_state(dev);
 167 }
 168
 169 static const char *hsynd_str(u8 synd)
 170 {
 171         switch (synd) {
 172         case MLX5_HEALTH_SYNDR_FW_ERR:
 173                 return "firmware internal error";
 174         case MLX5_HEALTH_SYNDR_IRISC_ERR:
 175                 return "irisc not responding";
 176         case MLX5_HEALTH_SYNDR_HW_UNRECOVERABLE_ERR:
 177                 return "unrecoverable hardware error";
 178         case MLX5_HEALTH_SYNDR_CRC_ERR:
 179                 return "firmware CRC error";
 180         case MLX5_HEALTH_SYNDR_FETCH_PCI_ERR:
 181                 return "ICM fetch PCI error";
 182         case MLX5_HEALTH_SYNDR_HW_FTL_ERR:
 183                 return "HW fatal error\n";
 184         case MLX5_HEALTH_SYNDR_ASYNC_EQ_OVERRUN_ERR:
 185                 return "async EQ buffer overrun";
 186         case MLX5_HEALTH_SYNDR_EQ_ERR:
 187                 return "EQ error";
 188         case MLX5_HEALTH_SYNDR_EQ_INV:
 189                 return "Invalid EQ referenced";
 190         case MLX5_HEALTH_SYNDR_FFSER_ERR:
 191                 return "FFSER error";
 192         case MLX5_HEALTH_SYNDR_HIGH_TEMP:
 193                 return "High temperature";
 194         default:
 195                 return "unrecognized error";
 196         }
 197 }
 198
 199 static u16 get_maj(u32 fw)
 200 {
 201         return fw >> 28;
 202 }
 203
 204 static u16 get_min(u32 fw)
 205 {
 206         return fw >> 16 & 0xfff;
 207 }
 208
 209 static u16 get_sub(u32 fw)
 210 {
 211         return fw & 0xffff;
 212 }
 213
 214 static void print_health_info(struct mlx5_core_dev *dev)
 215 {
 216         struct mlx5_core_health *health = &dev->priv.health;
 217         struct health_buffer __iomem *h = health->health;
 218         char fw_str[18];
 219         u32 fw;
 220         int i;
 221
 222         /* If the syndrom is 0, the device is OK and no need to print buffer */
 223         if (!ioread8(&h->synd))
 224                 return;
 225
 226         for (i = 0; i < ARRAY_SIZE(h->assert_var); i++)
 227                 dev_err(&dev->pdev->dev, "assert_var[%d] 0x%08x\n", i, ioread32be(h->assert_var + i));
 228
 229         dev_err(&dev->pdev->dev, "assert_exit_ptr 0x%08x\n", ioread32be(&h->assert_exit_ptr));
 230         dev_err(&dev->pdev->dev, "assert_callra 0x%08x\n", ioread32be(&h->assert_callra));
 231         fw = ioread32be(&h->fw_ver);
 232         sprintf(fw_str, "%d.%d.%d", get_maj(fw), get_min(fw), get_sub(fw));
 233         dev_err(&dev->pdev->dev, "fw_ver %s\n", fw_str);
 234         dev_err(&dev->pdev->dev, "hw_id 0x%08x\n", ioread32be(&h->hw_id));
 235         dev_err(&dev->pdev->dev, "irisc_index %d\n", ioread8(&h->irisc_index));
 236         dev_err(&dev->pdev->dev, "synd 0x%x: %s\n", ioread8(&h->synd), hsynd_str(ioread8(&h->synd)));
 237         dev_err(&dev->pdev->dev, "ext_synd 0x%04x\n", ioread16be(&h->ext_synd));
 238 }
 239
 240 static unsigned long get_next_poll_jiffies(void)
 241 {
 242         unsigned long next;
 243
 244         get_random_bytes(&next, sizeof(next));
 245         next %= HZ;
 246         next += jiffies + MLX5_HEALTH_POLL_INTERVAL;
 247
 248         return next;
 249 }
 250
 251 static void poll_health(unsigned long data)
 252 {
 253         struct mlx5_core_dev *dev = (struct mlx5_core_dev *)data;
 254         struct mlx5_core_health *health = &dev->priv.health;
 255         u32 count;
 256
 257         if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) {
 258                 mod_timer(&health->timer, get_next_poll_jiffies());
 259                 return;
 260         }
 261
 262         count = ioread32be(health->health_counter);
 263         if (count == health->prev)
 264                 ++health->miss_counter;
 265         else
 266                 health->miss_counter = 0;
 267
 268         health->prev = count;
 269         if (health->miss_counter == MAX_MISSES) {
 270                 dev_err(&dev->pdev->dev, "device's health compromised - reached miss count\n");
 271                 print_health_info(dev);
 272         } else {
 273                 mod_timer(&health->timer, get_next_poll_jiffies());
 274         }
 275
 276         if (in_fatal(dev) && !health->sick) {
 277                 health->sick = true;
 278                 print_health_info(dev);
 279                 spin_lock(&health->wq_lock);
 280                 if (!test_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags))
 281                         queue_work(health->wq, &health->work);
 282                 else
 283                         dev_err(&dev->pdev->dev,
 284                                 "new health works are not permitted at this stage\n");
 285                 spin_unlock(&health->wq_lock);
 286         }
 287 }
 288
 289 void mlx5_start_health_poll(struct mlx5_core_dev *dev)
 290 {
 291         struct mlx5_core_health *health = &dev->priv.health;
 292
 293         init_timer(&health->timer);
 294         health->sick = 0;
 295         clear_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags);
 296         health->health = &dev->iseg->health;
 297         health->health_counter = &dev->iseg->health_counter;
 298
 299         health->timer.data = (unsigned long)dev;
 300         health->timer.function = poll_health;
 301         health->timer.expires = round_jiffies(jiffies + MLX5_HEALTH_POLL_INTERVAL);
 302         add_timer(&health->timer);
 303 }
 304
 305 void mlx5_stop_health_poll(struct mlx5_core_dev *dev)
 306 {
 307         struct mlx5_core_health *health = &dev->priv.health;
 308
 309         del_timer_sync(&health->timer);
 310 }
 311
 312 void mlx5_drain_health_wq(struct mlx5_core_dev *dev)
 313 {
 314         struct mlx5_core_health *health = &dev->priv.health;
 315
 316         spin_lock(&health->wq_lock);
 317         set_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags);
 318         spin_unlock(&health->wq_lock);
 319         cancel_work_sync(&health->work);
 320 }
 321
 322 void mlx5_health_cleanup(struct mlx5_core_dev *dev)
 323 {
 324         struct mlx5_core_health *health = &dev->priv.health;
 325
 326         destroy_workqueue(health->wq);
 327 }
 328
 329 int mlx5_health_init(struct mlx5_core_dev *dev)
 330 {
 331         struct mlx5_core_health *health;
 332         char *name;
 333
 334         health = &dev->priv.health;
 335         name = kmalloc(64, GFP_KERNEL);
 336         if (!name)
 337                 return -ENOMEM;
 338
 339         strcpy(name, "mlx5_health");
 340         strcat(name, dev_name(&dev->pdev->dev));
 341         health->wq = create_singlethread_workqueue(name);
 342         kfree(name);
 343         if (!health->wq)
 344                 return -ENOMEM;
 345         spin_lock_init(&health->wq_lock);
 346         INIT_WORK(&health->work, health_care);
 347
 348         return 0;
 349 }