drivers/block/mtip32xx/mtip32xx.c

   1 /*
   2  * Driver for the Micron P320 SSD
   3  *   Copyright (C) 2011 Micron Technology, Inc.
   4  *
   5  * Portions of this code were derived from works subjected to the
   6  * following copyright:
   7  *    Copyright (C) 2009 Integrated Device Technology, Inc.
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License as published by
  11  * the Free Software Foundation; either version 2 of the License, or
  12  * (at your option) any later version.
  13  *
  14  * This program is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17  * GNU General Public License for more details.
  18  *
  19  */
  20
  21 #include <linux/pci.h>
  22 #include <linux/interrupt.h>
  23 #include <linux/ata.h>
  24 #include <linux/delay.h>
  25 #include <linux/hdreg.h>
  26 #include <linux/uaccess.h>
  27 #include <linux/random.h>
  28 #include <linux/smp.h>
  29 #include <linux/compat.h>
  30 #include <linux/fs.h>
  31 #include <linux/genhd.h>
  32 #include <linux/blkdev.h>
  33 #include <linux/bio.h>
  34 #include <linux/dma-mapping.h>
  35 #include <linux/idr.h>
  36 #include <../drivers/ata/ahci.h>
  37 #include "mtip32xx.h"
  38
  39 #define HW_CMD_SLOT_SZ          (MTIP_MAX_COMMAND_SLOTS * 32)
  40 #define HW_CMD_TBL_SZ           (AHCI_CMD_TBL_HDR_SZ + (MTIP_MAX_SG * 16))
  41 #define HW_CMD_TBL_AR_SZ        (HW_CMD_TBL_SZ * MTIP_MAX_COMMAND_SLOTS)
  42 #define HW_PORT_PRIV_DMA_SZ \
  43                 (HW_CMD_SLOT_SZ + HW_CMD_TBL_AR_SZ + AHCI_RX_FIS_SZ)
  44
  45 #define HOST_HSORG              0xFC
  46 #define HSORG_DISABLE_SLOTGRP_INTR (1<<24)
  47 #define HSORG_DISABLE_SLOTGRP_PXIS (1<<16)
  48 #define HSORG_HWREV             0xFF00
  49 #define HSORG_STYLE             0x8
  50 #define HSORG_SLOTGROUPS        0x7
  51
  52 #define PORT_COMMAND_ISSUE      0x38
  53 #define PORT_SDBV               0x7C
  54
  55 #define PORT_OFFSET             0x100
  56 #define PORT_MEM_SIZE           0x80
  57
  58 #define PORT_IRQ_ERR \
  59         (PORT_IRQ_HBUS_ERR | PORT_IRQ_IF_ERR | PORT_IRQ_CONNECT | \
  60          PORT_IRQ_PHYRDY | PORT_IRQ_UNK_FIS | PORT_IRQ_BAD_PMP | \
  61          PORT_IRQ_TF_ERR | PORT_IRQ_HBUS_DATA_ERR | PORT_IRQ_IF_NONFATAL | \
  62          PORT_IRQ_OVERFLOW)
  63 #define PORT_IRQ_LEGACY \
  64         (PORT_IRQ_PIOS_FIS | PORT_IRQ_D2H_REG_FIS)
  65 #define PORT_IRQ_HANDLED \
  66         (PORT_IRQ_SDB_FIS | PORT_IRQ_LEGACY | \
  67          PORT_IRQ_TF_ERR | PORT_IRQ_IF_ERR | \
  68          PORT_IRQ_CONNECT | PORT_IRQ_PHYRDY)
  69 #define DEF_PORT_IRQ \
  70         (PORT_IRQ_ERR | PORT_IRQ_LEGACY | PORT_IRQ_SDB_FIS)
  71
  72 /* product numbers */
  73 #define MTIP_PRODUCT_UNKNOWN    0x00
  74 #define MTIP_PRODUCT_ASICFPGA   0x11
  75
  76 /* Device instance number, incremented each time a device is probed. */
  77 static int instance;
  78
  79 /*
  80  * Global variable used to hold the major block device number
  81  * allocated in mtip_init().
  82  */
  83 int mtip_major;
  84
  85 static DEFINE_SPINLOCK(rssd_index_lock);
  86 static DEFINE_IDA(rssd_index_ida);
  87
  88 #ifdef CONFIG_COMPAT
  89 struct mtip_compat_ide_task_request_s {
  90         __u8            io_ports[8];
  91         __u8            hob_ports[8];
  92         ide_reg_valid_t out_flags;
  93         ide_reg_valid_t in_flags;
  94         int             data_phase;
  95         int             req_cmd;
  96         compat_ulong_t  out_size;
  97         compat_ulong_t  in_size;
  98 };
  99 #endif
 100
 101 static int mtip_exec_internal_command(struct mtip_port *port,
 102                                 void *fis,
 103                                 int fisLen,
 104                                 dma_addr_t buffer,
 105                                 int bufLen,
 106                                 u32 opts,
 107                                 gfp_t atomic,
 108                                 unsigned long timeout);
 109
 110 /*
 111  * Obtain an empty command slot.
 112  *
 113  * This function needs to be reentrant since it could be called
 114  * at the same time on multiple CPUs. The allocation of the
 115  * command slot must be atomic.
 116  *
 117  * @port Pointer to the port data structure.
 118  *
 119  * return value
 120  *      >= 0    Index of command slot obtained.
 121  *      -1      No command slots available.
 122  */
 123 static int get_slot(struct mtip_port *port)
 124 {
 125         int slot, i;
 126         unsigned int num_command_slots = port->dd->slot_groups * 32;
 127
 128         /*
 129          * Try 10 times, because there is a small race here.
 130          *  that's ok, because it's still cheaper than a lock.
 131          *
 132          * Race: Since this section is not protected by lock, same bit
 133          * could be chosen by different process contexts running in
 134          * different processor. So instead of costly lock, we are going
 135          * with loop.
 136          */
 137         for (i = 0; i < 10; i++) {
 138                 slot = find_next_zero_bit(port->allocated,
 139                                          num_command_slots, 1);
 140                 if ((slot < num_command_slots) &&
 141                     (!test_and_set_bit(slot, port->allocated)))
 142                         return slot;
 143         }
 144         dev_warn(&port->dd->pdev->dev, "Failed to get a tag.\n");
 145
 146         if (mtip_check_surprise_removal(port->dd->pdev)) {
 147                 /* Device not present, clean outstanding commands */
 148                 mtip_command_cleanup(port->dd);
 149         }
 150         return -1;
 151 }
 152
 153 /*
 154  * Release a command slot.
 155  *
 156  * @port Pointer to the port data structure.
 157  * @tag  Tag of command to release
 158  *
 159  * return value
 160  *      None
 161  */
 162 static inline void release_slot(struct mtip_port *port, int tag)
 163 {
 164         smp_mb__before_clear_bit();
 165         clear_bit(tag, port->allocated);
 166         smp_mb__after_clear_bit();
 167 }
 168
 169 /*
 170  * Issue a command to the hardware.
 171  *
 172  * Set the appropriate bit in the s_active and Command Issue hardware
 173  * registers, causing hardware command processing to begin.
 174  *
 175  * @port Pointer to the port structure.
 176  * @tag  The tag of the command to be issued.
 177  *
 178  * return value
 179  *      None
 180  */
 181 static inline void mtip_issue_ncq_command(struct mtip_port *port, int tag)
 182 {
 183         unsigned long flags = 0;
 184
 185         atomic_set(&port->commands[tag].active, 1);
 186
 187         spin_lock_irqsave(&port->cmd_issue_lock, flags);
 188
 189         writel((1 << MTIP_TAG_BIT(tag)),
 190                         port->s_active[MTIP_TAG_INDEX(tag)]);
 191         writel((1 << MTIP_TAG_BIT(tag)),
 192                         port->cmd_issue[MTIP_TAG_INDEX(tag)]);
 193
 194         spin_unlock_irqrestore(&port->cmd_issue_lock, flags);
 195 }
 196
 197 /*
 198  * Called periodically to see if any read/write commands are
 199  * taking too long to complete.
 200  *
 201  * @data Pointer to the PORT data structure.
 202  *
 203  * return value
 204  *      None
 205  */
 206 void mtip_timeout_function(unsigned long int data)
 207 {
 208         struct mtip_port *port = (struct mtip_port *) data;
 209         struct host_to_dev_fis *fis;
 210         struct mtip_cmd *command;
 211         int tag, cmdto_cnt = 0;
 212         unsigned int bit, group;
 213         unsigned int num_command_slots = port->dd->slot_groups * 32;
 214
 215         if (unlikely(!port))
 216                 return;
 217
 218         if (atomic_read(&port->dd->resumeflag) == true) {
 219                 mod_timer(&port->cmd_timer,
 220                         jiffies + msecs_to_jiffies(30000));
 221                 return;
 222         }
 223
 224         for (tag = 0; tag < num_command_slots; tag++) {
 225                 /*
 226                  * Skip internal command slot as it has
 227                  * its own timeout mechanism
 228                  */
 229                 if (tag == MTIP_TAG_INTERNAL)
 230                         continue;
 231
 232                 if (atomic_read(&port->commands[tag].active) &&
 233                    (time_after(jiffies, port->commands[tag].comp_time))) {
 234                         group = tag >> 5;
 235                         bit = tag & 0x1f;
 236
 237                         command = &port->commands[tag];
 238                         fis = (struct host_to_dev_fis *) command->command;
 239
 240                         dev_warn(&port->dd->pdev->dev,
 241                                 "Timeout for command tag %d\n", tag);
 242
 243                         cmdto_cnt++;
 244                         if (cmdto_cnt == 1)
 245                                 atomic_inc(&port->dd->eh_active);
 246
 247                         /*
 248                          * Clear the completed bit. This should prevent
 249                          *  any interrupt handlers from trying to retire
 250                          *  the command.
 251                          */
 252                         writel(1 << bit, port->completed[group]);
 253
 254                         /* Call the async completion callback. */
 255                         if (likely(command->async_callback))
 256                                 command->async_callback(command->async_data,
 257                                                          -EIO);
 258                         command->async_callback = NULL;
 259                         command->comp_func = NULL;
 260
 261                         /* Unmap the DMA scatter list entries */
 262                         dma_unmap_sg(&port->dd->pdev->dev,
 263                                         command->sg,
 264                                         command->scatter_ents,
 265                                         command->direction);
 266
 267                         /*
 268                          * Clear the allocated bit and active tag for the
 269                          * command.
 270                          */
 271                         atomic_set(&port->commands[tag].active, 0);
 272                         release_slot(port, tag);
 273
 274                         up(&port->cmd_slot);
 275                 }
 276         }
 277
 278         if (cmdto_cnt) {
 279                 dev_warn(&port->dd->pdev->dev,
 280                         "%d commands timed out: restarting port",
 281                         cmdto_cnt);
 282                 mtip_restart_port(port);
 283                 atomic_dec(&port->dd->eh_active);
 284         }
 285
 286         /* Restart the timer */
 287         mod_timer(&port->cmd_timer,
 288                 jiffies + msecs_to_jiffies(MTIP_TIMEOUT_CHECK_PERIOD));
 289 }
 290
 291 /*
 292  * IO completion function.
 293  *
 294  * This completion function is called by the driver ISR when a
 295  * command that was issued by the kernel completes. It first calls the
 296  * asynchronous completion function which normally calls back into the block
 297  * layer passing the asynchronous callback data, then unmaps the
 298  * scatter list associated with the completed command, and finally
 299  * clears the allocated bit associated with the completed command.
 300  *
 301  * @port   Pointer to the port data structure.
 302  * @tag    Tag of the command.
 303  * @data   Pointer to driver_data.
 304  * @status Completion status.
 305  *
 306  * return value
 307  *      None
 308  */
 309 static void mtip_async_complete(struct mtip_port *port,
 310                                 int tag,
 311                                 void *data,
 312                                 int status)
 313 {
 314         struct mtip_cmd *command;
 315         struct driver_data *dd = data;
 316         int cb_status = status ? -EIO : 0;
 317
 318         if (unlikely(!dd) || unlikely(!port))
 319                 return;
 320
 321         command = &port->commands[tag];
 322
 323         if (unlikely(status == PORT_IRQ_TF_ERR)) {
 324                 dev_warn(&port->dd->pdev->dev,
 325                         "Command tag %d failed due to TFE\n", tag);
 326         }
 327
 328         /* Upper layer callback */
 329         if (likely(command->async_callback))
 330                 command->async_callback(command->async_data, cb_status);
 331
 332         command->async_callback = NULL;
 333         command->comp_func = NULL;
 334
 335         /* Unmap the DMA scatter list entries */
 336         dma_unmap_sg(&dd->pdev->dev,
 337                 command->sg,
 338                 command->scatter_ents,
 339                 command->direction);
 340
 341         /* Clear the allocated and active bits for the command */
 342         atomic_set(&port->commands[tag].active, 0);
 343         release_slot(port, tag);
 344
 345         up(&port->cmd_slot);
 346 }
 347
 348 /*
 349  * Internal command completion callback function.
 350  *
 351  * This function is normally called by the driver ISR when an internal
 352  * command completed. This function signals the command completion by
 353  * calling complete().
 354  *
 355  * @port   Pointer to the port data structure.
 356  * @tag    Tag of the command that has completed.
 357  * @data   Pointer to a completion structure.
 358  * @status Completion status.
 359  *
 360  * return value
 361  *      None
 362  */
 363 static void mtip_completion(struct mtip_port *port,
 364                             int tag,
 365                             void *data,
 366                             int status)
 367 {
 368         struct mtip_cmd *command = &port->commands[tag];
 369         struct completion *waiting = data;
 370         if (unlikely(status == PORT_IRQ_TF_ERR))
 371                 dev_warn(&port->dd->pdev->dev,
 372                         "Internal command %d completed with TFE\n", tag);
 373
 374         command->async_callback = NULL;
 375         command->comp_func = NULL;
 376
 377         complete(waiting);
 378 }
 379
 380 /*
 381  * Enable/disable the reception of FIS
 382  *
 383  * @port   Pointer to the port data structure
 384  * @enable 1 to enable, 0 to disable
 385  *
 386  * return value
 387  *      Previous state: 1 enabled, 0 disabled
 388  */
 389 static int mtip_enable_fis(struct mtip_port *port, int enable)
 390 {
 391         u32 tmp;
 392
 393         /* enable FIS reception */
 394         tmp = readl(port->mmio + PORT_CMD);
 395         if (enable)
 396                 writel(tmp | PORT_CMD_FIS_RX, port->mmio + PORT_CMD);
 397         else
 398                 writel(tmp & ~PORT_CMD_FIS_RX, port->mmio + PORT_CMD);
 399
 400         /* Flush */
 401         readl(port->mmio + PORT_CMD);
 402
 403         return (((tmp & PORT_CMD_FIS_RX) == PORT_CMD_FIS_RX));
 404 }
 405
 406 /*
 407  * Enable/disable the DMA engine
 408  *
 409  * @port   Pointer to the port data structure
 410  * @enable 1 to enable, 0 to disable
 411  *
 412  * return value
 413  *      Previous state: 1 enabled, 0 disabled.
 414  */
 415 static int mtip_enable_engine(struct mtip_port *port, int enable)
 416 {
 417         u32 tmp;
 418
 419         /* enable FIS reception */
 420         tmp = readl(port->mmio + PORT_CMD);
 421         if (enable)
 422                 writel(tmp | PORT_CMD_START, port->mmio + PORT_CMD);
 423         else
 424                 writel(tmp & ~PORT_CMD_START, port->mmio + PORT_CMD);
 425
 426         readl(port->mmio + PORT_CMD);
 427         return (((tmp & PORT_CMD_START) == PORT_CMD_START));
 428 }
 429
 430 /*
 431  * Enables the port DMA engine and FIS reception.
 432  *
 433  * return value
 434  *      None
 435  */
 436 static inline void mtip_start_port(struct mtip_port *port)
 437 {
 438         /* Enable FIS reception */
 439         mtip_enable_fis(port, 1);
 440
 441         /* Enable the DMA engine */
 442         mtip_enable_engine(port, 1);
 443 }
 444
 445 /*
 446  * Deinitialize a port by disabling port interrupts, the DMA engine,
 447  * and FIS reception.
 448  *
 449  * @port Pointer to the port structure
 450  *
 451  * return value
 452  *      None
 453  */
 454 static inline void mtip_deinit_port(struct mtip_port *port)
 455 {
 456         /* Disable interrupts on this port */
 457         writel(0, port->mmio + PORT_IRQ_MASK);
 458
 459         /* Disable the DMA engine */
 460         mtip_enable_engine(port, 0);
 461
 462         /* Disable FIS reception */
 463         mtip_enable_fis(port, 0);
 464 }
 465
 466 /*
 467  * Initialize a port.
 468  *
 469  * This function deinitializes the port by calling mtip_deinit_port() and
 470  * then initializes it by setting the command header and RX FIS addresses,
 471  * clearing the SError register and any pending port interrupts before
 472  * re-enabling the default set of port interrupts.
 473  *
 474  * @port Pointer to the port structure.
 475  *
 476  * return value
 477  *      None
 478  */
 479 static void mtip_init_port(struct mtip_port *port)
 480 {
 481         int i;
 482         mtip_deinit_port(port);
 483
 484         /* Program the command list base and FIS base addresses */
 485         if (readl(port->dd->mmio + HOST_CAP) & HOST_CAP_64) {
 486                 writel((port->command_list_dma >> 16) >> 16,
 487                          port->mmio + PORT_LST_ADDR_HI);
 488                 writel((port->rxfis_dma >> 16) >> 16,
 489                          port->mmio + PORT_FIS_ADDR_HI);
 490         }
 491
 492         writel(port->command_list_dma & 0xffffffff,
 493                         port->mmio + PORT_LST_ADDR);
 494         writel(port->rxfis_dma & 0xffffffff, port->mmio + PORT_FIS_ADDR);
 495
 496         /* Clear SError */
 497         writel(readl(port->mmio + PORT_SCR_ERR), port->mmio + PORT_SCR_ERR);
 498
 499         /* reset the completed registers.*/
 500         for (i = 0; i < port->dd->slot_groups; i++)
 501                 writel(0xFFFFFFFF, port->completed[i]);
 502
 503         /* Clear any pending interrupts for this port */
 504         writel(readl(port->mmio + PORT_IRQ_STAT), port->mmio + PORT_IRQ_STAT);
 505
 506         /* Enable port interrupts */
 507         writel(DEF_PORT_IRQ, port->mmio + PORT_IRQ_MASK);
 508 }
 509
 510 /*
 511  * Reset the HBA (without sleeping)
 512  *
 513  * Just like hba_reset, except does not call sleep, so can be
 514  * run from interrupt/tasklet context.
 515  *
 516  * @dd Pointer to the driver data structure.
 517  *
 518  * return value
 519  *      0       The reset was successful.
 520  *      -1      The HBA Reset bit did not clear.
 521  */
 522 int hba_reset_nosleep(struct driver_data *dd)
 523 {
 524         unsigned long timeout;
 525
 526         /* Chip quirk: quiesce any chip function */
 527         mdelay(10);
 528
 529         /* Set the reset bit */
 530         writel(HOST_RESET, dd->mmio + HOST_CTL);
 531
 532         /* Flush */
 533         readl(dd->mmio + HOST_CTL);
 534
 535         /*
 536          * Wait 10ms then spin for up to 1 second
 537          * waiting for reset acknowledgement
 538          */
 539         timeout = jiffies + msecs_to_jiffies(1000);
 540         mdelay(10);
 541         while ((readl(dd->mmio + HOST_CTL) & HOST_RESET)
 542                  && time_before(jiffies, timeout))
 543                 mdelay(1);
 544
 545         if (readl(dd->mmio + HOST_CTL) & HOST_RESET)
 546                 return -1;
 547
 548         return 0;
 549 }
 550
 551 /*
 552  * Restart a port
 553  *
 554  * @port Pointer to the port data structure.
 555  *
 556  * return value
 557  *      None
 558  */
 559 void mtip_restart_port(struct mtip_port *port)
 560 {
 561         unsigned long timeout;
 562
 563         /* Disable the DMA engine */
 564         mtip_enable_engine(port, 0);
 565
 566         /* Chip quirk: wait up to 500ms for PxCMD.CR == 0 */
 567         timeout = jiffies + msecs_to_jiffies(500);
 568         while ((readl(port->mmio + PORT_CMD) & PORT_CMD_LIST_ON)
 569                  && time_before(jiffies, timeout))
 570                 ;
 571
 572         /*
 573          * Chip quirk: escalate to hba reset if
 574          * PxCMD.CR not clear after 500 ms
 575          */
 576         if (readl(port->mmio + PORT_CMD) & PORT_CMD_LIST_ON) {
 577                 dev_warn(&port->dd->pdev->dev,
 578                         "PxCMD.CR not clear, escalating reset\n");
 579
 580                 if (hba_reset_nosleep(port->dd))
 581                         dev_err(&port->dd->pdev->dev,
 582                                 "HBA reset escalation failed.\n");
 583
 584                 /* 30 ms delay before com reset to quiesce chip */
 585                 mdelay(30);
 586         }
 587
 588         dev_warn(&port->dd->pdev->dev, "Issuing COM reset\n");
 589
 590         /* Set PxSCTL.DET */
 591         writel(readl(port->mmio + PORT_SCR_CTL) |
 592                          1, port->mmio + PORT_SCR_CTL);
 593         readl(port->mmio + PORT_SCR_CTL);
 594
 595         /* Wait 1 ms to quiesce chip function */
 596         timeout = jiffies + msecs_to_jiffies(1);
 597         while (time_before(jiffies, timeout))
 598                 ;
 599
 600         /* Clear PxSCTL.DET */
 601         writel(readl(port->mmio + PORT_SCR_CTL) & ~1,
 602                          port->mmio + PORT_SCR_CTL);
 603         readl(port->mmio + PORT_SCR_CTL);
 604
 605         /* Wait 500 ms for bit 0 of PORT_SCR_STS to be set */
 606         timeout = jiffies + msecs_to_jiffies(500);
 607         while (((readl(port->mmio + PORT_SCR_STAT) & 0x01) == 0)
 608                          && time_before(jiffies, timeout))
 609                 ;
 610
 611         if ((readl(port->mmio + PORT_SCR_STAT) & 0x01) == 0)
 612                 dev_warn(&port->dd->pdev->dev,
 613                         "COM reset failed\n");
 614
 615         /* Clear SError, the PxSERR.DIAG.x should be set so clear it */
 616         writel(readl(port->mmio + PORT_SCR_ERR), port->mmio + PORT_SCR_ERR);
 617
 618         /* Enable the DMA engine */
 619         mtip_enable_engine(port, 1);
 620 }
 621
 622 /*
 623  * Helper function for tag logging
 624  */
 625 static void print_tags(struct driver_data *dd,
 626                         char *msg,
 627                         unsigned long *tagbits)
 628 {
 629         unsigned int tag, count = 0;
 630
 631         for (tag = 0; tag < (dd->slot_groups) * 32; tag++) {
 632                 if (test_bit(tag, tagbits))
 633                         count++;
 634         }
 635         if (count)
 636                 dev_info(&dd->pdev->dev, "%s [%i tags]\n", msg, count);
 637 }
 638
 639 /*
 640  * Handle an error.
 641  *
 642  * @dd Pointer to the DRIVER_DATA structure.
 643  *
 644  * return value
 645  *      None
 646  */
 647 static void mtip_handle_tfe(struct driver_data *dd)
 648 {
 649         int group, tag, bit, reissue;
 650         struct mtip_port *port;
 651         struct mtip_cmd  *command;
 652         u32 completed;
 653         struct host_to_dev_fis *fis;
 654         unsigned long tagaccum[SLOTBITS_IN_LONGS];
 655
 656         dev_warn(&dd->pdev->dev, "Taskfile error\n");
 657
 658         port = dd->port;
 659
 660         /* Stop the timer to prevent command timeouts. */
 661         del_timer(&port->cmd_timer);
 662
 663         /* Set eh_active */
 664         atomic_inc(&dd->eh_active);
 665
 666         /* Loop through all the groups */
 667         for (group = 0; group < dd->slot_groups; group++) {
 668                 completed = readl(port->completed[group]);
 669
 670                 /* clear completed status register in the hardware.*/
 671                 writel(completed, port->completed[group]);
 672
 673                 /* clear the tag accumulator */
 674                 memset(tagaccum, 0, SLOTBITS_IN_LONGS * sizeof(long));
 675
 676                 /* Process successfully completed commands */
 677                 for (bit = 0; bit < 32 && completed; bit++) {
 678                         if (!(completed & (1<<bit)))
 679                                 continue;
 680                         tag = (group << 5) + bit;
 681
 682                         /* Skip the internal command slot */
 683                         if (tag == MTIP_TAG_INTERNAL)
 684                                 continue;
 685
 686                         command = &port->commands[tag];
 687                         if (likely(command->comp_func)) {
 688                                 set_bit(tag, tagaccum);
 689                                 atomic_set(&port->commands[tag].active, 0);
 690                                 command->comp_func(port,
 691                                          tag,
 692                                          command->comp_data,
 693                                          0);
 694                         } else {
 695                                 dev_err(&port->dd->pdev->dev,
 696                                         "Missing completion func for tag %d",
 697                                         tag);
 698                                 if (mtip_check_surprise_removal(dd->pdev)) {
 699                                         mtip_command_cleanup(dd);
 700                                         /* don't proceed further */
 701                                         return;
 702                                 }
 703                         }
 704                 }
 705         }
 706         print_tags(dd, "TFE tags completed:", tagaccum);
 707
 708         /* Restart the port */
 709         mdelay(20);
 710         mtip_restart_port(port);
 711
 712         /* clear the tag accumulator */
 713         memset(tagaccum, 0, SLOTBITS_IN_LONGS * sizeof(long));
 714
 715         /* Loop through all the groups */
 716         for (group = 0; group < dd->slot_groups; group++) {
 717                 for (bit = 0; bit < 32; bit++) {
 718                         reissue = 1;
 719                         tag = (group << 5) + bit;
 720
 721                         /* If the active bit is set re-issue the command */
 722                         if (atomic_read(&port->commands[tag].active) == 0)
 723                                 continue;
 724
 725                         fis = (struct host_to_dev_fis *)
 726                                 port->commands[tag].command;
 727
 728                         /* Should re-issue? */
 729                         if (tag == MTIP_TAG_INTERNAL ||
 730                             fis->command == ATA_CMD_SET_FEATURES)
 731                                 reissue = 0;
 732
 733                         /*
 734                          * First check if this command has
 735                          *  exceeded its retries.
 736                          */
 737                         if (reissue &&
 738                             (port->commands[tag].retries-- > 0)) {
 739
 740                                 set_bit(tag, tagaccum);
 741
 742                                 /* Update the timeout value. */
 743                                 port->commands[tag].comp_time =
 744                                         jiffies + msecs_to_jiffies(
 745                                         MTIP_NCQ_COMMAND_TIMEOUT_MS);
 746                                 /* Re-issue the command. */
 747                                 mtip_issue_ncq_command(port, tag);
 748
 749                                 continue;
 750                         }
 751
 752                         /* Retire a command that will not be reissued */
 753                         dev_warn(&port->dd->pdev->dev,
 754                                 "retiring tag %d\n", tag);
 755                         atomic_set(&port->commands[tag].active, 0);
 756
 757                         if (port->commands[tag].comp_func)
 758                                 port->commands[tag].comp_func(
 759                                         port,
 760                                         tag,
 761                                         port->commands[tag].comp_data,
 762                                         PORT_IRQ_TF_ERR);
 763                         else
 764                                 dev_warn(&port->dd->pdev->dev,
 765                                         "Bad completion for tag %d\n",
 766                                         tag);
 767                 }
 768         }
 769         print_tags(dd, "TFE tags reissued:", tagaccum);
 770
 771         /* Decrement eh_active */
 772         atomic_dec(&dd->eh_active);
 773
 774         mod_timer(&port->cmd_timer,
 775                  jiffies + msecs_to_jiffies(MTIP_TIMEOUT_CHECK_PERIOD));
 776 }
 777
 778 /*
 779  * Handle a set device bits interrupt
 780  */
 781 static inline void mtip_process_sdbf(struct driver_data *dd)
 782 {
 783         struct mtip_port  *port = dd->port;
 784         int group, tag, bit;
 785         u32 completed;
 786         struct mtip_cmd *command;
 787
 788         /* walk all bits in all slot groups */
 789         for (group = 0; group < dd->slot_groups; group++) {
 790                 completed = readl(port->completed[group]);
 791
 792                 /* clear completed status register in the hardware.*/
 793                 writel(completed, port->completed[group]);
 794
 795                 /* Process completed commands. */
 796                 for (bit = 0;
 797                      (bit < 32) && completed;
 798                      bit++, completed >>= 1) {
 799                         if (completed & 0x01) {
 800                                 tag = (group << 5) | bit;
 801
 802                                 /* skip internal command slot. */
 803                                 if (unlikely(tag == MTIP_TAG_INTERNAL))
 804                                         continue;
 805
 806                                 command = &port->commands[tag];
 807
 808                                 /* make internal callback */
 809                                 if (likely(command->comp_func)) {
 810                                         command->comp_func(
 811                                                 port,
 812                                                 tag,
 813                                                 command->comp_data,
 814                                                 0);
 815                                 } else {
 816                                         dev_warn(&dd->pdev->dev,
 817                                                 "Null completion "
 818                                                 "for tag %d",
 819                                                 tag);
 820
 821                                         if (mtip_check_surprise_removal(
 822                                                 dd->pdev)) {
 823                                                 mtip_command_cleanup(dd);
 824                                                 return;
 825                                         }
 826                                 }
 827                         }
 828                 }
 829         }
 830 }
 831
 832 /*
 833  * Process legacy pio and d2h interrupts
 834  */
 835 static inline void mtip_process_legacy(struct driver_data *dd, u32 port_stat)
 836 {
 837         struct mtip_port *port = dd->port;
 838         struct mtip_cmd *cmd = &port->commands[MTIP_TAG_INTERNAL];
 839
 840         if (port->internal_cmd_in_progress &&
 841             cmd != NULL &&
 842             !(readl(port->cmd_issue[MTIP_TAG_INTERNAL])
 843                 & (1 << MTIP_TAG_INTERNAL))) {
 844                 if (cmd->comp_func) {
 845                         cmd->comp_func(port,
 846                                 MTIP_TAG_INTERNAL,
 847                                 cmd->comp_data,
 848                                 0);
 849                         return;
 850                 }
 851         }
 852
 853         dev_warn(&dd->pdev->dev, "IRQ status 0x%x ignored.\n", port_stat);
 854
 855         return;
 856 }
 857
 858 /*
 859  * Demux and handle errors
 860  */
 861 static inline void mtip_process_errors(struct driver_data *dd, u32 port_stat)
 862 {
 863         if (likely(port_stat & (PORT_IRQ_TF_ERR | PORT_IRQ_IF_ERR)))
 864                 mtip_handle_tfe(dd);
 865
 866         if (unlikely(port_stat & PORT_IRQ_CONNECT)) {
 867                 dev_warn(&dd->pdev->dev,
 868                         "Clearing PxSERR.DIAG.x\n");
 869                 writel((1 << 26), dd->port->mmio + PORT_SCR_ERR);
 870         }
 871
 872         if (unlikely(port_stat & PORT_IRQ_PHYRDY)) {
 873                 dev_warn(&dd->pdev->dev,
 874                         "Clearing PxSERR.DIAG.n\n");
 875                 writel((1 << 16), dd->port->mmio + PORT_SCR_ERR);
 876         }
 877
 878         if (unlikely(port_stat & ~PORT_IRQ_HANDLED)) {
 879                 dev_warn(&dd->pdev->dev,
 880                         "Port stat errors %x unhandled\n",
 881                         (port_stat & ~PORT_IRQ_HANDLED));
 882         }
 883 }
 884
 885 static inline irqreturn_t mtip_handle_irq(struct driver_data *data)
 886 {
 887         struct driver_data *dd = (struct driver_data *) data;
 888         struct mtip_port *port = dd->port;
 889         u32 hba_stat, port_stat;
 890         int rv = IRQ_NONE;
 891
 892         hba_stat = readl(dd->mmio + HOST_IRQ_STAT);
 893         if (hba_stat) {
 894                 rv = IRQ_HANDLED;
 895
 896                 /* Acknowledge the interrupt status on the port.*/
 897                 port_stat = readl(port->mmio + PORT_IRQ_STAT);
 898                 writel(port_stat, port->mmio + PORT_IRQ_STAT);
 899
 900                 /* Demux port status */
 901                 if (likely(port_stat & PORT_IRQ_SDB_FIS))
 902                         mtip_process_sdbf(dd);
 903
 904                 if (unlikely(port_stat & PORT_IRQ_ERR)) {
 905                         if (unlikely(mtip_check_surprise_removal(dd->pdev))) {
 906                                 mtip_command_cleanup(dd);
 907                                 /* don't proceed further */
 908                                 return IRQ_HANDLED;
 909                         }
 910
 911                         mtip_process_errors(dd, port_stat & PORT_IRQ_ERR);
 912                 }
 913
 914                 if (unlikely(port_stat & PORT_IRQ_LEGACY))
 915                         mtip_process_legacy(dd, port_stat & PORT_IRQ_LEGACY);
 916         }
 917
 918         /* acknowledge interrupt */
 919         writel(hba_stat, dd->mmio + HOST_IRQ_STAT);
 920
 921         return rv;
 922 }
 923
 924 /*
 925  * Wrapper for mtip_handle_irq
 926  * (ignores return code)
 927  */
 928 static void mtip_tasklet(unsigned long data)
 929 {
 930         mtip_handle_irq((struct driver_data *) data);
 931 }
 932
 933 /*
 934  * HBA interrupt subroutine.
 935  *
 936  * @irq         IRQ number.
 937  * @instance    Pointer to the driver data structure.
 938  *
 939  * return value
 940  *      IRQ_HANDLED     A HBA interrupt was pending and handled.
 941  *      IRQ_NONE        This interrupt was not for the HBA.
 942  */
 943 static irqreturn_t mtip_irq_handler(int irq, void *instance)
 944 {
 945         struct driver_data *dd = instance;
 946         tasklet_schedule(&dd->tasklet);
 947         return IRQ_HANDLED;
 948 }
 949
 950 static void mtip_issue_non_ncq_command(struct mtip_port *port, int tag)
 951 {
 952         atomic_set(&port->commands[tag].active, 1);
 953         writel(1 << MTIP_TAG_BIT(tag),
 954                 port->cmd_issue[MTIP_TAG_INDEX(tag)]);
 955 }
 956
 957 /*
 958  * Wait for port to quiesce
 959  *
 960  * @port    Pointer to port data structure
 961  * @timeout Max duration to wait (ms)
 962  *
 963  * return value
 964  *      0       Success
 965  *      -EBUSY  Commands still active
 966  */
 967 static int mtip_quiesce_io(struct mtip_port *port, unsigned long timeout)
 968 {
 969         unsigned long to;
 970         unsigned int n, active;
 971
 972         to = jiffies + msecs_to_jiffies(timeout);
 973         do {
 974                 /*
 975                  * Ignore s_active bit 0 of array element 0.
 976                  * This bit will always be set
 977                  */
 978                 active = readl(port->s_active[0]) & 0xfffffffe;
 979                 for (n = 1; n < port->dd->slot_groups; n++)
 980                         active |= readl(port->s_active[n]);
 981
 982                 if (!active)
 983                         break;
 984
 985                 msleep(20);
 986         } while (time_before(jiffies, to));
 987
 988         return active ? -EBUSY : 0;
 989 }
 990
 991 /*
 992  * Execute an internal command and wait for the completion.
 993  *
 994  * @port    Pointer to the port data structure.
 995  * @fis     Pointer to the FIS that describes the command.
 996  * @fisLen  Length in WORDS of the FIS.
 997  * @buffer  DMA accessible for command data.
 998  * @bufLen  Length, in bytes, of the data buffer.
 999  * @opts    Command header options, excluding the FIS length
1000  *             and the number of PRD entries.
1001  * @timeout Time in ms to wait for the command to complete.
1002  *
1003  * return value
1004  *      0        Command completed successfully.
1005  *      -EFAULT  The buffer address is not correctly aligned.
1006  *      -EBUSY   Internal command or other IO in progress.
1007  *      -EAGAIN  Time out waiting for command to complete.
1008  */
1009 static int mtip_exec_internal_command(struct mtip_port *port,
1010                                         void *fis,
1011                                         int fisLen,
1012                                         dma_addr_t buffer,
1013                                         int bufLen,
1014                                         u32 opts,
1015                                         gfp_t atomic,
1016                                         unsigned long timeout)
1017 {
1018         struct mtip_cmd_sg *command_sg;
1019         DECLARE_COMPLETION_ONSTACK(wait);
1020         int rv = 0;
1021         struct mtip_cmd *int_cmd = &port->commands[MTIP_TAG_INTERNAL];
1022
1023         /* Make sure the buffer is 8 byte aligned. This is asic specific. */
1024         if (buffer & 0x00000007) {
1025                 dev_err(&port->dd->pdev->dev,
1026                         "SG buffer is not 8 byte aligned\n");
1027                 return -EFAULT;
1028         }
1029
1030         /* Only one internal command should be running at a time */
1031         if (test_and_set_bit(MTIP_TAG_INTERNAL, port->allocated)) {
1032                 dev_warn(&port->dd->pdev->dev,
1033                         "Internal command already active\n");
1034                 return -EBUSY;
1035         }
1036         port->internal_cmd_in_progress = 1;
1037
1038         if (atomic == GFP_KERNEL) {
1039                 /* wait for io to complete if non atomic */
1040                 if (mtip_quiesce_io(port, 5000) < 0) {
1041                         dev_warn(&port->dd->pdev->dev,
1042                                 "Failed to quiesce IO\n");
1043                         release_slot(port, MTIP_TAG_INTERNAL);
1044                         port->internal_cmd_in_progress = 0;
1045                         return -EBUSY;
1046                 }
1047
1048                 /* Set the completion function and data for the command. */
1049                 int_cmd->comp_data = &wait;
1050                 int_cmd->comp_func = mtip_completion;
1051
1052         } else {
1053                 /* Clear completion - we're going to poll */
1054                 int_cmd->comp_data = NULL;
1055                 int_cmd->comp_func = NULL;
1056         }
1057
1058         /* Copy the command to the command table */
1059         memcpy(int_cmd->command, fis, fisLen*4);
1060
1061         /* Populate the SG list */
1062         int_cmd->command_header->opts =
1063                  cpu_to_le32(opts | fisLen);
1064         if (bufLen) {
1065                 command_sg = int_cmd->command + AHCI_CMD_TBL_HDR_SZ;
1066
1067                 command_sg->info = cpu_to_le32((bufLen-1) & 0x3fffff);
1068                 command_sg->dba = cpu_to_le32(buffer & 0xffffffff);
1069                 command_sg->dba_upper = cpu_to_le32((buffer >> 16) >> 16);
1070
1071                 int_cmd->command_header->opts |= cpu_to_le32((1 << 16));
1072         }
1073
1074         /* Populate the command header */
1075         int_cmd->command_header->byte_count = 0;
1076
1077         /* Issue the command to the hardware */
1078         mtip_issue_non_ncq_command(port, MTIP_TAG_INTERNAL);
1079
1080         /* Poll if atomic, wait_for_completion otherwise */
1081         if (atomic == GFP_KERNEL) {
1082                 /* Wait for the command to complete or timeout. */
1083                 if (wait_for_completion_timeout(
1084                                 &wait,
1085                                 msecs_to_jiffies(timeout)) == 0) {
1086                         dev_err(&port->dd->pdev->dev,
1087                                 "Internal command did not complete [%d]\n",
1088                                 atomic);
1089                         rv = -EAGAIN;
1090                 }
1091
1092                 if (readl(port->cmd_issue[MTIP_TAG_INTERNAL])
1093                         & (1 << MTIP_TAG_INTERNAL)) {
1094                         dev_warn(&port->dd->pdev->dev,
1095                                 "Retiring internal command but CI is 1.\n");
1096                 }
1097
1098         } else {
1099                 /* Spin for <timeout> checking if command still outstanding */
1100                 timeout = jiffies + msecs_to_jiffies(timeout);
1101
1102                 while ((readl(
1103                         port->cmd_issue[MTIP_TAG_INTERNAL])
1104                         & (1 << MTIP_TAG_INTERNAL))
1105                         && time_before(jiffies, timeout))
1106                         ;
1107
1108                 if (readl(port->cmd_issue[MTIP_TAG_INTERNAL])
1109                         & (1 << MTIP_TAG_INTERNAL)) {
1110                         dev_err(&port->dd->pdev->dev,
1111                                 "Internal command did not complete [%d]\n",
1112                                 atomic);
1113                         rv = -EAGAIN;
1114                 }
1115         }
1116
1117         /* Clear the allocated and active bits for the internal command. */
1118         atomic_set(&int_cmd->active, 0);
1119         release_slot(port, MTIP_TAG_INTERNAL);
1120         port->internal_cmd_in_progress = 0;
1121
1122         return rv;
1123 }
1124
1125 /*
1126  * Byte-swap ATA ID strings.
1127  *
1128  * ATA identify data contains strings in byte-swapped 16-bit words.
1129  * They must be swapped (on all architectures) to be usable as C strings.
1130  * This function swaps bytes in-place.
1131  *
1132  * @buf The buffer location of the string
1133  * @len The number of bytes to swap
1134  *
1135  * return value
1136  *      None
1137  */
1138 static inline void ata_swap_string(u16 *buf, unsigned int len)
1139 {
1140         int i;
1141         for (i = 0; i < (len/2); i++)
1142                 be16_to_cpus(&buf[i]);
1143 }
1144
1145 /*
1146  * Request the device identity information.
1147  *
1148  * If a user space buffer is not specified, i.e. is NULL, the
1149  * identify information is still read from the drive and placed
1150  * into the identify data buffer (@e port->identify) in the
1151  * port data structure.
1152  * When the identify buffer contains valid identify information @e
1153  * port->identify_valid is non-zero.
1154  *
1155  * @port         Pointer to the port structure.
1156  * @user_buffer  A user space buffer where the identify data should be
1157  *                    copied.
1158  *
1159  * return value
1160  *      0       Command completed successfully.
1161  *      -EFAULT An error occurred while coping data to the user buffer.
1162  *      -1      Command failed.
1163  */
1164 static int mtip_get_identify(struct mtip_port *port, void __user *user_buffer)
1165 {
1166         int rv = 0;
1167         struct host_to_dev_fis fis;
1168
1169         down_write(&port->dd->internal_sem);
1170
1171         /* Build the FIS. */
1172         memset(&fis, 0, sizeof(struct host_to_dev_fis));
1173         fis.type        = 0x27;
1174         fis.opts        = 1 << 7;
1175         fis.command     = ATA_CMD_ID_ATA;
1176
1177         /* Set the identify information as invalid. */
1178         port->identify_valid = 0;
1179
1180         /* Clear the identify information. */
1181         memset(port->identify, 0, sizeof(u16) * ATA_ID_WORDS);
1182
1183         /* Execute the command. */
1184         if (mtip_exec_internal_command(port,
1185                                 &fis,
1186                                 5,
1187                                 port->identify_dma,
1188                                 sizeof(u16) * ATA_ID_WORDS,
1189                                 0,
1190                                 GFP_KERNEL,
1191                                 MTIP_INTERNAL_COMMAND_TIMEOUT_MS)
1192                                 < 0) {
1193                 rv = -1;
1194                 goto out;
1195         }
1196
1197         /*
1198          * Perform any necessary byte-swapping.  Yes, the kernel does in fact
1199          * perform field-sensitive swapping on the string fields.
1200          * See the kernel use of ata_id_string() for proof of this.
1201          */
1202 #ifdef __LITTLE_ENDIAN
1203         ata_swap_string(port->identify + 27, 40);  /* model string*/
1204         ata_swap_string(port->identify + 23, 8);   /* firmware string*/
1205         ata_swap_string(port->identify + 10, 20);  /* serial# string*/
1206 #else
1207         {
1208                 int i;
1209                 for (i = 0; i < ATA_ID_WORDS; i++)
1210                         port->identify[i] = le16_to_cpu(port->identify[i]);
1211         }
1212 #endif
1213
1214         /* Set the identify buffer as valid. */
1215         port->identify_valid = 1;
1216
1217         if (user_buffer) {
1218                 if (copy_to_user(
1219                         user_buffer,
1220                         port->identify,
1221                         ATA_ID_WORDS * sizeof(u16))) {
1222                         rv = -EFAULT;
1223                         goto out;
1224                 }
1225         }
1226
1227 out:
1228         up_write(&port->dd->internal_sem);
1229         return rv;
1230 }
1231
1232 /*
1233  * Issue a standby immediate command to the device.
1234  *
1235  * @port Pointer to the port structure.
1236  *
1237  * return value
1238  *      0       Command was executed successfully.
1239  *      -1      An error occurred while executing the command.
1240  */
1241 static int mtip_standby_immediate(struct mtip_port *port)
1242 {
1243         int rv;
1244         struct host_to_dev_fis  fis;
1245
1246         down_write(&port->dd->internal_sem);
1247
1248         /* Build the FIS. */
1249         memset(&fis, 0, sizeof(struct host_to_dev_fis));
1250         fis.type        = 0x27;
1251         fis.opts        = 1 << 7;
1252         fis.command     = ATA_CMD_STANDBYNOW1;
1253
1254         /* Execute the command.  Use a 15-second timeout for large drives. */
1255         rv = mtip_exec_internal_command(port,
1256                                         &fis,
1257                                         5,
1258                                         0,
1259                                         0,
1260                                         0,
1261                                         GFP_KERNEL,
1262                                         15000);
1263
1264         up_write(&port->dd->internal_sem);
1265
1266         return rv;
1267 }
1268
1269 /*
1270  * Get the drive capacity.
1271  *
1272  * @dd      Pointer to the device data structure.
1273  * @sectors Pointer to the variable that will receive the sector count.
1274  *
1275  * return value
1276  *      1 Capacity was returned successfully.
1277  *      0 The identify information is invalid.
1278  */
1279 bool mtip_hw_get_capacity(struct driver_data *dd, sector_t *sectors)
1280 {
1281         struct mtip_port *port = dd->port;
1282         u64 total, raw0, raw1, raw2, raw3;
1283         raw0 = port->identify[100];
1284         raw1 = port->identify[101];
1285         raw2 = port->identify[102];
1286         raw3 = port->identify[103];
1287         total = raw0 | raw1<<16 | raw2<<32 | raw3<<48;
1288         *sectors = total;
1289         return (bool) !!port->identify_valid;
1290 }
1291
1292 /*
1293  * Reset the HBA.
1294  *
1295  * Resets the HBA by setting the HBA Reset bit in the Global
1296  * HBA Control register. After setting the HBA Reset bit the
1297  * function waits for 1 second before reading the HBA Reset
1298  * bit to make sure it has cleared. If HBA Reset is not clear
1299  * an error is returned. Cannot be used in non-blockable
1300  * context.
1301  *
1302  * @dd Pointer to the driver data structure.
1303  *
1304  * return value
1305  *      0  The reset was successful.
1306  *      -1 The HBA Reset bit did not clear.
1307  */
1308 static int mtip_hba_reset(struct driver_data *dd)
1309 {
1310         mtip_deinit_port(dd->port);
1311
1312         /* Set the reset bit */
1313         writel(HOST_RESET, dd->mmio + HOST_CTL);
1314
1315         /* Flush */
1316         readl(dd->mmio + HOST_CTL);
1317
1318         /* Wait for reset to clear */
1319         ssleep(1);
1320
1321         /* Check the bit has cleared */
1322         if (readl(dd->mmio + HOST_CTL) & HOST_RESET) {
1323                 dev_err(&dd->pdev->dev,
1324                         "Reset bit did not clear.\n");
1325                 return -1;
1326         }
1327
1328         return 0;
1329 }
1330
1331 /*
1332  * Display the identify command data.
1333  *
1334  * @port Pointer to the port data structure.
1335  *
1336  * return value
1337  *      None
1338  */
1339 static void mtip_dump_identify(struct mtip_port *port)
1340 {
1341         sector_t sectors;
1342         unsigned short revid;
1343         char cbuf[42];
1344
1345         if (!port->identify_valid)
1346                 return;
1347
1348         strlcpy(cbuf, (char *)(port->identify+10), 21);
1349         dev_info(&port->dd->pdev->dev,
1350                 "Serial No.: %s\n", cbuf);
1351
1352         strlcpy(cbuf, (char *)(port->identify+23), 9);
1353         dev_info(&port->dd->pdev->dev,
1354                 "Firmware Ver.: %s\n", cbuf);
1355
1356         strlcpy(cbuf, (char *)(port->identify+27), 41);
1357         dev_info(&port->dd->pdev->dev, "Model: %s\n", cbuf);
1358
1359         if (mtip_hw_get_capacity(port->dd, &sectors))
1360                 dev_info(&port->dd->pdev->dev,
1361                         "Capacity: %llu sectors (%llu MB)\n",
1362                          (u64)sectors,
1363                          ((u64)sectors) * ATA_SECT_SIZE >> 20);
1364
1365         pci_read_config_word(port->dd->pdev, PCI_REVISION_ID, &revid);
1366         switch (revid & 0xff) {
1367         case 0x1:
1368                 strlcpy(cbuf, "A0", 3);
1369                 break;
1370         case 0x3:
1371                 strlcpy(cbuf, "A2", 3);
1372                 break;
1373         default:
1374                 strlcpy(cbuf, "?", 2);
1375                 break;
1376         }
1377         dev_info(&port->dd->pdev->dev,
1378                 "Card Type: %s\n", cbuf);
1379 }
1380
1381 /*
1382  * Map the commands scatter list into the command table.
1383  *
1384  * @command Pointer to the command.
1385  * @nents Number of scatter list entries.
1386  *
1387  * return value
1388  *      None
1389  */
1390 static inline void fill_command_sg(struct driver_data *dd,
1391                                 struct mtip_cmd *command,
1392                                 int nents)
1393 {
1394         int n;
1395         unsigned int dma_len;
1396         struct mtip_cmd_sg *command_sg;
1397         struct scatterlist *sg = command->sg;
1398
1399         command_sg = command->command + AHCI_CMD_TBL_HDR_SZ;
1400
1401         for (n = 0; n < nents; n++) {
1402                 dma_len = sg_dma_len(sg);
1403                 if (dma_len > 0x400000)
1404                         dev_err(&dd->pdev->dev,
1405                                 "DMA segment length truncated\n");
1406                 command_sg->info = cpu_to_le32((dma_len-1) & 0x3fffff);
1407 #if (BITS_PER_LONG == 64)
1408                 *((unsigned long *) &command_sg->dba) =
1409                          cpu_to_le64(sg_dma_address(sg));
1410 #else
1411                 command_sg->dba = cpu_to_le32(sg_dma_address(sg));
1412                 command_sg->dba_upper   =
1413                          cpu_to_le32((sg_dma_address(sg) >> 16) >> 16);
1414 #endif
1415                 command_sg++;
1416                 sg++;
1417         }
1418 }
1419
1420 /*
1421  * @brief Execute a drive command.
1422  *
1423  * return value 0 The command completed successfully.
1424  * return value -1 An error occurred while executing the command.
1425  */
1426 int exec_drive_task(struct mtip_port *port, u8 *command)
1427 {
1428         struct host_to_dev_fis  fis;
1429         struct host_to_dev_fis *reply = (port->rxfis + RX_FIS_D2H_REG);
1430
1431         /* Lock the internal command semaphore. */
1432         down_write(&port->dd->internal_sem);
1433
1434         /* Build the FIS. */
1435         memset(&fis, 0, sizeof(struct host_to_dev_fis));
1436         fis.type        = 0x27;
1437         fis.opts        = 1 << 7;
1438         fis.command     = command[0];
1439         fis.features    = command[1];
1440         fis.sect_count  = command[2];
1441         fis.sector      = command[3];
1442         fis.cyl_low     = command[4];
1443         fis.cyl_hi      = command[5];
1444         fis.device      = command[6] & ~0x10; /* Clear the dev bit*/
1445
1446
1447         dbg_printk(MTIP_DRV_NAME "%s: User Command: cmd %x, feat %x, "
1448                 "nsect %x, sect %x, lcyl %x, "
1449                 "hcyl %x, sel %x\n",
1450                 __func__,
1451                 command[0],
1452                 command[1],
1453                 command[2],
1454                 command[3],
1455                 command[4],
1456                 command[5],
1457                 command[6]);
1458
1459         /* Execute the command. */
1460         if (mtip_exec_internal_command(port,
1461                                  &fis,
1462                                  5,
1463                                  0,
1464                                  0,
1465                                  0,
1466                                  GFP_KERNEL,
1467                                  MTIP_IOCTL_COMMAND_TIMEOUT_MS) < 0) {
1468                 up_write(&port->dd->internal_sem);
1469                 return -1;
1470         }
1471
1472         command[0] = reply->command; /* Status*/
1473         command[1] = reply->features; /* Error*/
1474         command[4] = reply->cyl_low;
1475         command[5] = reply->cyl_hi;
1476
1477         dbg_printk(MTIP_DRV_NAME "%s: Completion Status: stat %x, "
1478                 "err %x , cyl_lo %x cyl_hi %x\n",
1479                 __func__,
1480                 command[0],
1481                 command[1],
1482                 command[4],
1483                 command[5]);
1484
1485         up_write(&port->dd->internal_sem);
1486         return 0;
1487 }
1488
1489 /*
1490  * @brief Execute a drive command.
1491  *
1492  * @param port Pointer to the port data structure.
1493  * @param command Pointer to the user specified command parameters.
1494  * @param user_buffer Pointer to the user space buffer where read sector
1495  *                   data should be copied.
1496  *
1497  * return value 0 The command completed successfully.
1498  * return value -EFAULT An error occurred while copying the completion
1499  *                 data to the user space buffer.
1500  * return value -1 An error occurred while executing the command.
1501  */
1502 int exec_drive_command(struct mtip_port *port, u8 *command,
1503                         void __user *user_buffer)
1504 {
1505         struct host_to_dev_fis  fis;
1506         struct host_to_dev_fis *reply = (port->rxfis + RX_FIS_D2H_REG);
1507
1508         /* Lock the internal command semaphore. */
1509         down_write(&port->dd->internal_sem);
1510
1511         /* Build the FIS. */
1512         memset(&fis, 0, sizeof(struct host_to_dev_fis));
1513         fis.type                = 0x27;
1514         fis.opts                = 1 << 7;
1515         fis.command             = command[0];
1516         fis.features    = command[2];
1517         fis.sect_count  = command[3];
1518         if (fis.command == ATA_CMD_SMART) {
1519                 fis.sector      = command[1];
1520                 fis.cyl_low     = 0x4f;
1521                 fis.cyl_hi      = 0xc2;
1522         }
1523
1524         dbg_printk(MTIP_DRV_NAME
1525                 "%s: User Command: cmd %x, sect %x, "
1526                 "feat %x, sectcnt %x\n",
1527                 __func__,
1528                 command[0],
1529                 command[1],
1530                 command[2],
1531                 command[3]);
1532
1533         memset(port->sector_buffer, 0x00, ATA_SECT_SIZE);
1534
1535         /* Execute the command. */
1536         if (mtip_exec_internal_command(port,
1537                                 &fis,
1538                                  5,
1539                                  port->sector_buffer_dma,
1540                                  (command[3] != 0) ? ATA_SECT_SIZE : 0,
1541                                  0,
1542                                  GFP_KERNEL,
1543                                  MTIP_IOCTL_COMMAND_TIMEOUT_MS)
1544                                  < 0) {
1545                 up_write(&port->dd->internal_sem);
1546                 return -1;
1547         }
1548
1549         /* Collect the completion status. */
1550         command[0] = reply->command; /* Status*/
1551         command[1] = reply->features; /* Error*/
1552         command[2] = command[3];
1553
1554         dbg_printk(MTIP_DRV_NAME
1555                 "%s: Completion Status: stat %x, "
1556                 "err %x, cmd %x\n",
1557                 __func__,
1558                 command[0],
1559                 command[1],
1560                 command[2]);
1561
1562         if (user_buffer && command[3]) {
1563                 if (copy_to_user(user_buffer,
1564                                  port->sector_buffer,
1565                                  ATA_SECT_SIZE * command[3])) {
1566                         up_write(&port->dd->internal_sem);
1567                         return -EFAULT;
1568                 }
1569         }
1570
1571         up_write(&port->dd->internal_sem);
1572         return 0;
1573 }
1574
1575 /*
1576  *  Indicates whether a command has a single sector payload.
1577  *
1578  *  @command passed to the device to perform the certain event.
1579  *  @features passed to the device to perform the certain event.
1580  *
1581  *  return value
1582  *      1       command is one that always has a single sector payload,
1583  *              regardless of the value in the Sector Count field.
1584  *      0       otherwise
1585  *
1586  */
1587 static unsigned int implicit_sector(unsigned char command,
1588                                     unsigned char features)
1589 {
1590         unsigned int rv = 0;
1591
1592         /* list of commands that have an implicit sector count of 1 */
1593         switch (command) {
1594         case 0xF1:
1595         case 0xF2:
1596         case 0xF3:
1597         case 0xF4:
1598         case 0xF5:
1599         case 0xF6:
1600         case 0xE4:
1601         case 0xE8:
1602                 rv = 1;
1603                 break;
1604         case 0xF9:
1605                 if (features == 0x03)
1606                         rv = 1;
1607                 break;
1608         case 0xB0:
1609                 if ((features == 0xD0) || (features == 0xD1))
1610                         rv = 1;
1611                 break;
1612         case 0xB1:
1613                 if ((features == 0xC2) || (features == 0xC3))
1614                         rv = 1;
1615                 break;
1616         }
1617         return rv;
1618 }
1619
1620 /*
1621  * Executes a taskfile
1622  * See ide_taskfile_ioctl() for derivation
1623  */
1624 static int exec_drive_taskfile(struct driver_data *dd,
1625                                 unsigned long arg,
1626                                 unsigned char compat)
1627 {
1628         struct host_to_dev_fis  fis;
1629         struct host_to_dev_fis *reply;
1630         ide_task_request_t *req_task;
1631         u8 *outbuf = NULL;
1632         u8 *inbuf = NULL;
1633         dma_addr_t outbuf_dma = 0;
1634         dma_addr_t inbuf_dma = 0;
1635         dma_addr_t dma_buffer = 0;
1636         int err = 0;
1637         int tasksize = sizeof(struct ide_task_request_s);
1638         unsigned int taskin = 0;
1639         unsigned int taskout = 0;
1640         u8 nsect = 0;
1641         char __user *buf = (char __user *)arg;
1642         unsigned int timeout = MTIP_IOCTL_COMMAND_TIMEOUT_MS;
1643         unsigned int force_single_sector;
1644         unsigned int transfer_size;
1645         unsigned long task_file_data;
1646         int intotal, outtotal;
1647 #ifdef CONFIG_COMPAT
1648         struct mtip_compat_ide_task_request_s *compat_req_task = NULL;
1649         int compat_tasksize = sizeof(struct mtip_compat_ide_task_request_s);
1650 #endif
1651
1652
1653         req_task = kzalloc(tasksize, GFP_KERNEL);
1654         if (req_task == NULL)
1655                 return -ENOMEM;
1656
1657         if (compat == 1) {
1658 #ifdef CONFIG_COMPAT
1659                 compat_req_task =
1660                         (struct mtip_compat_ide_task_request_s __user *) arg;
1661
1662                 if (copy_from_user(req_task, buf,
1663                                 compat_tasksize -
1664                                 (2 * sizeof(compat_long_t)))) {
1665                         err = -EFAULT;
1666                         goto abort;
1667                 }
1668
1669                 if (get_user(req_task->out_size, &compat_req_task->out_size)) {
1670                         err = -EFAULT;
1671                         goto abort;
1672                 }
1673
1674                 if (get_user(req_task->in_size, &compat_req_task->in_size)) {
1675                         err = -EFAULT;
1676                         goto abort;
1677                 }
1678
1679                 outtotal = compat_tasksize;
1680                 intotal = compat_tasksize + req_task->out_size;
1681 #else
1682                 outtotal = 0;
1683                 intotal = 0;
1684 #endif
1685         } else {
1686                 if (copy_from_user(req_task, buf, tasksize)) {
1687                         kfree(req_task);
1688                         err = -EFAULT;
1689                         goto abort;
1690                 }
1691
1692                 outtotal = tasksize;
1693                 intotal = tasksize + req_task->out_size;
1694         }
1695
1696         taskout = req_task->out_size;
1697         taskin = req_task->in_size;
1698         /* 130560 = 512 * 0xFF*/
1699         if (taskin > 130560 || taskout > 130560) {
1700                 err = -EINVAL;
1701                 goto abort;
1702         }
1703
1704         if (taskout) {
1705                 outbuf = kzalloc(taskout, GFP_KERNEL);
1706                 if (outbuf == NULL) {
1707                         err = -ENOMEM;
1708                         goto abort;
1709                 }
1710                 if (copy_from_user(outbuf, buf + outtotal, taskout)) {
1711                         err = -EFAULT;
1712                         goto abort;
1713                 }
1714                 outbuf_dma = pci_map_single(dd->pdev,
1715                                          outbuf,
1716                                          taskout,
1717                                          DMA_TO_DEVICE);
1718                 if (outbuf_dma == 0) {
1719                         err = -ENOMEM;
1720                         goto abort;
1721                 }
1722                 dma_buffer = outbuf_dma;
1723         }
1724
1725         if (taskin) {
1726                 inbuf = kzalloc(taskin, GFP_KERNEL);
1727                 if (inbuf == NULL) {
1728                         err = -ENOMEM;
1729                         goto abort;
1730                 }
1731
1732                 if (copy_from_user(inbuf, buf + intotal, taskin)) {
1733                         err = -EFAULT;
1734                         goto abort;
1735                 }
1736                 inbuf_dma = pci_map_single(dd->pdev,
1737                                          inbuf,
1738                                          taskin, DMA_FROM_DEVICE);
1739                 if (inbuf_dma == 0) {
1740                         err = -ENOMEM;
1741                         goto abort;
1742                 }
1743                 dma_buffer = inbuf_dma;
1744         }
1745
1746         /* only supports PIO and non-data commands from this ioctl. */
1747         switch (req_task->data_phase) {
1748         case TASKFILE_OUT:
1749                 nsect = taskout / ATA_SECT_SIZE;
1750                 reply = (dd->port->rxfis + RX_FIS_PIO_SETUP);
1751                 break;
1752         case TASKFILE_IN:
1753                 reply = (dd->port->rxfis + RX_FIS_PIO_SETUP);
1754                 break;
1755         case TASKFILE_NO_DATA:
1756                 reply = (dd->port->rxfis + RX_FIS_D2H_REG);
1757                 break;
1758         default:
1759                 err = -EINVAL;
1760                 goto abort;
1761         }
1762
1763         /* Lock the internal command semaphore. */
1764         down_write(&dd->internal_sem);
1765
1766         /* Build the FIS. */
1767         memset(&fis, 0, sizeof(struct host_to_dev_fis));
1768
1769         fis.type        = 0x27;
1770         fis.opts        = 1 << 7;
1771         fis.command     = req_task->io_ports[7];
1772         fis.features    = req_task->io_ports[1];
1773         fis.sect_count  = req_task->io_ports[2];
1774         fis.lba_low     = req_task->io_ports[3];
1775         fis.lba_mid     = req_task->io_ports[4];
1776         fis.lba_hi      = req_task->io_ports[5];
1777          /* Clear the dev bit*/
1778         fis.device      = req_task->io_ports[6] & ~0x10;
1779
1780         if ((req_task->in_flags.all == 0) && (req_task->out_flags.all & 1)) {
1781                 req_task->in_flags.all  =
1782                         IDE_TASKFILE_STD_IN_FLAGS |
1783                         (IDE_HOB_STD_IN_FLAGS << 8);
1784                 fis.lba_low_ex          = req_task->hob_ports[3];
1785                 fis.lba_mid_ex          = req_task->hob_ports[4];
1786                 fis.lba_hi_ex           = req_task->hob_ports[5];
1787                 fis.features_ex         = req_task->hob_ports[1];
1788                 fis.sect_cnt_ex         = req_task->hob_ports[2];
1789
1790         } else {
1791                 req_task->in_flags.all = IDE_TASKFILE_STD_IN_FLAGS;
1792         }
1793
1794         force_single_sector = implicit_sector(fis.command, fis.features);
1795
1796         if ((taskin || taskout) && (!fis.sect_count)) {
1797                 if (nsect)
1798                         fis.sect_count = nsect;
1799                 else {
1800                         if (!force_single_sector) {
1801                                 dev_warn(&dd->pdev->dev,
1802                                         "data movement but "
1803                                         "sect_count is 0\n");
1804                                         up_write(&dd->internal_sem);
1805                                         err = -EINVAL;
1806                                         goto abort;
1807                         }
1808                 }
1809         }
1810
1811         dbg_printk(MTIP_DRV_NAME
1812                 "taskfile: cmd %x, feat %x, nsect %x,"
1813                 " sect/lbal %x, lcyl/lbam %x, hcyl/lbah %x,"
1814                 " head/dev %x\n",
1815                 fis.command,
1816                 fis.features,
1817                 fis.sect_count,
1818                 fis.lba_low,
1819                 fis.lba_mid,
1820                 fis.lba_hi,
1821                 fis.device);
1822
1823         switch (fis.command) {
1824         case 0x92: /* Change timeout for Download Microcode to 60 seconds.*/
1825                 timeout = 60000;
1826                 break;
1827         case 0xf4: /* Change timeout for Security Erase Unit to 4 minutes.*/
1828                 timeout = 240000;
1829                 break;
1830         case 0xe0: /* Change timeout for standby immediate to 10 seconds.*/
1831                 timeout = 10000;
1832                 break;
1833         case 0xf7: /* Change timeout for vendor unique command to 10 secs */
1834                 timeout = 10000;
1835                 break;
1836         case 0xfa: /* Change timeout for vendor unique command to 10 secs */
1837                 timeout = 10000;
1838                 break;
1839         default:
1840                 timeout = MTIP_IOCTL_COMMAND_TIMEOUT_MS;
1841                 break;
1842         }
1843
1844         /* Determine the correct transfer size.*/
1845         if (force_single_sector)
1846                 transfer_size = ATA_SECT_SIZE;
1847         else
1848                 transfer_size = ATA_SECT_SIZE * fis.sect_count;
1849
1850         /* Execute the command.*/
1851         if (mtip_exec_internal_command(dd->port,
1852                                  &fis,
1853                                  5,
1854                                  dma_buffer,
1855                                  transfer_size,
1856                                  0,
1857                                  GFP_KERNEL,
1858                                  timeout) < 0) {
1859                 up_write(&dd->internal_sem);
1860                 err = -EIO;
1861                 goto abort;
1862         }
1863
1864         task_file_data = readl(dd->port->mmio+PORT_TFDATA);
1865
1866         if ((req_task->data_phase == TASKFILE_IN) && !(task_file_data & 1)) {
1867                 reply = dd->port->rxfis + RX_FIS_PIO_SETUP;
1868                 req_task->io_ports[7] = reply->control;
1869         } else {
1870                 reply = dd->port->rxfis + RX_FIS_D2H_REG;
1871                 req_task->io_ports[7] = reply->command;
1872         }
1873
1874         /* reclaim the DMA buffers.*/
1875         if (inbuf_dma)
1876                 pci_unmap_single(dd->pdev, inbuf_dma,
1877                         taskin, DMA_FROM_DEVICE);
1878         if (outbuf_dma)
1879                 pci_unmap_single(dd->pdev, outbuf_dma,
1880                         taskout, DMA_TO_DEVICE);
1881         inbuf_dma  = 0;
1882         outbuf_dma = 0;
1883
1884         /* return the ATA registers to the caller.*/
1885         req_task->io_ports[1] = reply->features;
1886         req_task->io_ports[2] = reply->sect_count;
1887         req_task->io_ports[3] = reply->lba_low;
1888         req_task->io_ports[4] = reply->lba_mid;
1889         req_task->io_ports[5] = reply->lba_hi;
1890         req_task->io_ports[6] = reply->device;
1891
1892         if (req_task->out_flags.all & 1)  {
1893
1894                 req_task->hob_ports[3] = reply->lba_low_ex;
1895                 req_task->hob_ports[4] = reply->lba_mid_ex;
1896                 req_task->hob_ports[5] = reply->lba_hi_ex;
1897                 req_task->hob_ports[1] = reply->features_ex;
1898                 req_task->hob_ports[2] = reply->sect_cnt_ex;
1899         }
1900
1901         /* Com rest after secure erase or lowlevel format */
1902         if (((fis.command == 0xF4) ||
1903                 ((fis.command == 0xFC) &&
1904                         (fis.features == 0x27 || fis.features == 0x72 ||
1905                          fis.features == 0x62 || fis.features == 0x26))) &&
1906                          !(reply->command & 1)) {
1907                 mtip_restart_port(dd->port);
1908         }
1909
1910         dbg_printk(MTIP_DRV_NAME
1911                 "%s: Completion: stat %x,"
1912                 "err %x, sect_cnt %x, lbalo %x,"
1913                 "lbamid %x, lbahi %x, dev %x\n",
1914                 __func__,
1915                 req_task->io_ports[7],
1916                 req_task->io_ports[1],
1917                 req_task->io_ports[2],
1918                 req_task->io_ports[3],
1919                 req_task->io_ports[4],
1920                 req_task->io_ports[5],
1921                 req_task->io_ports[6]);
1922
1923         up_write(&dd->internal_sem);
1924
1925         if (compat == 1) {
1926 #ifdef CONFIG_COMPAT
1927                 if (copy_to_user(buf, req_task,
1928                                 compat_tasksize -
1929                                 (2 * sizeof(compat_long_t)))) {
1930                         err = -EFAULT;
1931                         goto abort;
1932                 }
1933                 if (put_user(req_task->out_size,
1934                                 &compat_req_task->out_size)) {
1935                         err = -EFAULT;
1936                         goto abort;
1937                 }
1938                 if (put_user(req_task->in_size, &compat_req_task->in_size)) {
1939                         err = -EFAULT;
1940                         goto abort;
1941                 }
1942 #endif
1943         } else {
1944                 if (copy_to_user(buf, req_task, tasksize)) {
1945                         err = -EFAULT;
1946                         goto abort;
1947                 }
1948         }
1949         if (taskout) {
1950                 if (copy_to_user(buf + outtotal, outbuf, taskout)) {
1951                         err = -EFAULT;
1952                         goto abort;
1953                 }
1954         }
1955         if (taskin) {
1956                 if (copy_to_user(buf + intotal, inbuf, taskin)) {
1957                         err = -EFAULT;
1958                         goto abort;
1959                 }
1960         }
1961 abort:
1962         if (inbuf_dma)
1963                 pci_unmap_single(dd->pdev, inbuf_dma,
1964                                         taskin, DMA_FROM_DEVICE);
1965         if (outbuf_dma)
1966                 pci_unmap_single(dd->pdev, outbuf_dma,
1967                                         taskout, DMA_TO_DEVICE);
1968         kfree(req_task);
1969         kfree(outbuf);
1970         kfree(inbuf);
1971
1972         return err;
1973 }
1974
1975 /*
1976  * Handle IOCTL calls from the Block Layer.
1977  *
1978  * This function is called by the Block Layer when it receives an IOCTL
1979  * command that it does not understand. If the IOCTL command is not supported
1980  * this function returns -ENOTTY.
1981  *
1982  * @dd  Pointer to the driver data structure.
1983  * @cmd IOCTL command passed from the Block Layer.
1984  * @arg IOCTL argument passed from the Block Layer.
1985  *
1986  * return value
1987  *      0       The IOCTL completed successfully.
1988  *      -ENOTTY The specified command is not supported.
1989  *      -EFAULT An error occurred copying data to a user space buffer.
1990  *      -EIO    An error occurred while executing the command.
1991  */
1992 int mtip_hw_ioctl(struct driver_data *dd,
1993                   unsigned int cmd,
1994                   unsigned long arg,
1995                   unsigned char compat)
1996 {
1997         switch (cmd) {
1998         case HDIO_GET_IDENTITY:
1999                 if (mtip_get_identify(dd->port, (void __user *) arg) < 0) {
2000                         dev_warn(&dd->pdev->dev,
2001                                 "Unable to read identity\n");
2002                         return -EIO;
2003                 }
2004
2005                 break;
2006         case HDIO_DRIVE_CMD:
2007         {
2008                 u8 drive_command[4];
2009
2010                 /* Copy the user command info to our buffer. */
2011                 if (copy_from_user(drive_command,
2012                                          (void __user *) arg,
2013                                          sizeof(drive_command)))
2014                         return -EFAULT;
2015
2016                 /* Execute the drive command. */
2017                 if (exec_drive_command(dd->port,
2018                                          drive_command,
2019                                          (void __user *) (arg+4)))
2020                         return -EIO;
2021
2022                 /* Copy the status back to the users buffer. */
2023                 if (copy_to_user((void __user *) arg,
2024                                          drive_command,
2025                                          sizeof(drive_command)))
2026                         return -EFAULT;
2027
2028                 break;
2029         }
2030         case HDIO_DRIVE_TASK:
2031         {
2032                 u8 drive_command[7];
2033
2034                 /* Copy the user command info to our buffer. */
2035                 if (copy_from_user(drive_command,
2036                                          (void __user *) arg,
2037                                          sizeof(drive_command)))
2038                         return -EFAULT;
2039
2040                 /* Execute the drive command. */
2041                 if (exec_drive_task(dd->port, drive_command))
2042                         return -EIO;
2043
2044                 /* Copy the status back to the users buffer. */
2045                 if (copy_to_user((void __user *) arg,
2046                                          drive_command,
2047                                          sizeof(drive_command)))
2048                         return -EFAULT;
2049
2050                 break;
2051         }
2052         case HDIO_DRIVE_TASKFILE:
2053                 return exec_drive_taskfile(dd, arg, compat);
2054
2055         default:
2056                 return -EINVAL;
2057         }
2058         return 0;
2059 }
2060
2061 /*
2062  * Submit an IO to the hw
2063  *
2064  * This function is called by the block layer to issue an io
2065  * to the device. Upon completion, the callback function will
2066  * be called with the data parameter passed as the callback data.
2067  *
2068  * @dd       Pointer to the driver data structure.
2069  * @start    First sector to read.
2070  * @nsect    Number of sectors to read.
2071  * @nents    Number of entries in scatter list for the read command.
2072  * @tag      The tag of this read command.
2073  * @callback Pointer to the function that should be called
2074  *           when the read completes.
2075  * @data     Callback data passed to the callback function
2076  *           when the read completes.
2077  * @barrier  If non-zero, this command must be completed before
2078  *           issuing any other commands.
2079  * @dir      Direction (read or write)
2080  *
2081  * return value
2082  *      None
2083  */
2084 void mtip_hw_submit_io(struct driver_data *dd,
2085                         sector_t start,
2086                         int nsect,
2087                         int nents,
2088                         int tag,
2089                         void *callback,
2090                         void *data,
2091                         int barrier,
2092                         int dir)
2093 {
2094         struct host_to_dev_fis  *fis;
2095         struct mtip_port *port = dd->port;
2096         struct mtip_cmd *command = &port->commands[tag];
2097
2098         /* Map the scatter list for DMA access */
2099         if (dir == READ)
2100                 nents = dma_map_sg(&dd->pdev->dev, command->sg,
2101                                         nents, DMA_FROM_DEVICE);
2102         else
2103                 nents = dma_map_sg(&dd->pdev->dev, command->sg,
2104                                         nents, DMA_TO_DEVICE);
2105
2106         command->scatter_ents = nents;
2107
2108         /*
2109          * The number of retries for this command before it is
2110          * reported as a failure to the upper layers.
2111          */
2112         command->retries = MTIP_MAX_RETRIES;
2113
2114         /* Fill out fis */
2115         fis = command->command;
2116         fis->type        = 0x27;
2117         fis->opts        = 1 << 7;
2118         fis->command     =
2119                 (dir == READ ? ATA_CMD_FPDMA_READ : ATA_CMD_FPDMA_WRITE);
2120         *((unsigned int *) &fis->lba_low) = (start & 0xffffff);
2121         *((unsigned int *) &fis->lba_low_ex) = ((start >> 24) & 0xffffff);
2122         fis->device      = 1 << 6;
2123         if (barrier)
2124                 fis->device |= FUA_BIT;
2125         fis->features    = nsect & 0xff;
2126         fis->features_ex = (nsect >> 8) & 0xff;
2127         fis->sect_count  = ((tag << 3) | (tag >> 5));
2128         fis->sect_cnt_ex = 0;
2129         fis->control     = 0;
2130         fis->res2        = 0;
2131         fis->res3        = 0;
2132         fill_command_sg(dd, command, nents);
2133
2134         /* Populate the command header */
2135         command->command_header->opts = cpu_to_le32(
2136                         (nents << 16) | 5 | AHCI_CMD_PREFETCH);
2137         command->command_header->byte_count = 0;
2138
2139         /*
2140          * Set the completion function and data for the command
2141          * within this layer.
2142          */
2143         command->comp_data = dd;
2144         command->comp_func = mtip_async_complete;
2145         command->direction = (dir == READ ? DMA_FROM_DEVICE : DMA_TO_DEVICE);
2146
2147         /*
2148          * Set the completion function and data for the command passed
2149          * from the upper layer.
2150          */
2151         command->async_data = data;
2152         command->async_callback = callback;
2153
2154         /*
2155          * Lock used to prevent this command from being issued
2156          * if an internal command is in progress.
2157          */
2158         down_read(&port->dd->internal_sem);
2159
2160         /* Issue the command to the hardware */
2161         mtip_issue_ncq_command(port, tag);
2162
2163         /* Set the command's timeout value.*/
2164         port->commands[tag].comp_time = jiffies + msecs_to_jiffies(
2165                                         MTIP_NCQ_COMMAND_TIMEOUT_MS);
2166
2167         up_read(&port->dd->internal_sem);
2168 }
2169
2170 /*
2171  * Release a command slot.
2172  *
2173  * @dd  Pointer to the driver data structure.
2174  * @tag Slot tag
2175  *
2176  * return value
2177  *      None
2178  */
2179 void mtip_hw_release_scatterlist(struct driver_data *dd, int tag)
2180 {
2181         release_slot(dd->port, tag);
2182 }
2183
2184 /*
2185  * Obtain a command slot and return its associated scatter list.
2186  *
2187  * @dd  Pointer to the driver data structure.
2188  * @tag Pointer to an int that will receive the allocated command
2189  *            slot tag.
2190  *
2191  * return value
2192  *      Pointer to the scatter list for the allocated command slot
2193  *      or NULL if no command slots are available.
2194  */
2195 struct scatterlist *mtip_hw_get_scatterlist(struct driver_data *dd,
2196                                                 int *tag)
2197 {
2198         /*
2199          * It is possible that, even with this semaphore, a thread
2200          * may think that no command slots are available. Therefore, we
2201          * need to make an attempt to get_slot().
2202          */
2203         down(&dd->port->cmd_slot);
2204         *tag = get_slot(dd->port);
2205
2206         if (unlikely(*tag < 0))
2207                 return NULL;
2208
2209         return dd->port->commands[*tag].sg;
2210 }
2211
2212 /*
2213  * Sysfs register/status dump.
2214  *
2215  * @dev  Pointer to the device structure, passed by the kernrel.
2216  * @attr Pointer to the device_attribute structure passed by the kernel.
2217  * @buf  Pointer to the char buffer that will receive the stats info.
2218  *
2219  * return value
2220  *      The size, in bytes, of the data copied into buf.
2221  */
2222 static ssize_t hw_show_registers(struct device *dev,
2223                                 struct device_attribute *attr,
2224                                 char *buf)
2225 {
2226         u32 group_allocated;
2227         struct driver_data *dd = dev_to_disk(dev)->private_data;
2228         int size = 0;
2229         int n;
2230
2231         size += sprintf(&buf[size], "%s:\ns_active:\n", __func__);
2232
2233         for (n = 0; n < dd->slot_groups; n++)
2234                 size += sprintf(&buf[size], "0x%08x\n",
2235                                          readl(dd->port->s_active[n]));
2236
2237         size += sprintf(&buf[size], "Command Issue:\n");
2238
2239         for (n = 0; n < dd->slot_groups; n++)
2240                 size += sprintf(&buf[size], "0x%08x\n",
2241                                         readl(dd->port->cmd_issue[n]));
2242
2243         size += sprintf(&buf[size], "Allocated:\n");
2244
2245         for (n = 0; n < dd->slot_groups; n++) {
2246                 if (sizeof(long) > sizeof(u32))
2247                         group_allocated =
2248                                 dd->port->allocated[n/2] >> (32*(n&1));
2249                 else
2250                         group_allocated = dd->port->allocated[n];
2251                 size += sprintf(&buf[size], "0x%08x\n",
2252                                  group_allocated);
2253         }
2254
2255         size += sprintf(&buf[size], "completed:\n");
2256
2257         for (n = 0; n < dd->slot_groups; n++)
2258                 size += sprintf(&buf[size], "0x%08x\n",
2259                                 readl(dd->port->completed[n]));
2260
2261         size += sprintf(&buf[size], "PORT_IRQ_STAT 0x%08x\n",
2262                                 readl(dd->port->mmio + PORT_IRQ_STAT));
2263         size += sprintf(&buf[size], "HOST_IRQ_STAT 0x%08x\n",
2264                                 readl(dd->mmio + HOST_IRQ_STAT));
2265
2266         return size;
2267 }
2268 static DEVICE_ATTR(registers, S_IRUGO, hw_show_registers, NULL);
2269
2270 /*
2271  * Create the sysfs related attributes.
2272  *
2273  * @dd   Pointer to the driver data structure.
2274  * @kobj Pointer to the kobj for the block device.
2275  *
2276  * return value
2277  *      0       Operation completed successfully.
2278  *      -EINVAL Invalid parameter.
2279  */
2280 int mtip_hw_sysfs_init(struct driver_data *dd, struct kobject *kobj)
2281 {
2282         if (!kobj || !dd)
2283                 return -EINVAL;
2284
2285         if (sysfs_create_file(kobj, &dev_attr_registers.attr))
2286                 dev_warn(&dd->pdev->dev,
2287                         "Error creating registers sysfs entry\n");
2288         return 0;
2289 }
2290
2291 /*
2292  * Remove the sysfs related attributes.
2293  *
2294  * @dd   Pointer to the driver data structure.
2295  * @kobj Pointer to the kobj for the block device.
2296  *
2297  * return value
2298  *      0       Operation completed successfully.
2299  *      -EINVAL Invalid parameter.
2300  */
2301 int mtip_hw_sysfs_exit(struct driver_data *dd, struct kobject *kobj)
2302 {
2303         if (!kobj || !dd)
2304                 return -EINVAL;
2305
2306         sysfs_remove_file(kobj, &dev_attr_registers.attr);
2307
2308         return 0;
2309 }
2310
2311 /*
2312  * Perform any init/resume time hardware setup
2313  *
2314  * @dd Pointer to the driver data structure.
2315  *
2316  * return value
2317  *      None
2318  */
2319 static inline void hba_setup(struct driver_data *dd)
2320 {
2321         u32 hwdata;
2322         hwdata = readl(dd->mmio + HOST_HSORG);
2323
2324         /* interrupt bug workaround: use only 1 IS bit.*/
2325         writel(hwdata |
2326                 HSORG_DISABLE_SLOTGRP_INTR |
2327                 HSORG_DISABLE_SLOTGRP_PXIS,
2328                 dd->mmio + HOST_HSORG);
2329 }
2330
2331 /*
2332  * Detect the details of the product, and store anything needed
2333  * into the driver data structure.  This includes product type and
2334  * version and number of slot groups.
2335  *
2336  * @dd Pointer to the driver data structure.
2337  *
2338  * return value
2339  *      None
2340  */
2341 static void mtip_detect_product(struct driver_data *dd)
2342 {
2343         u32 hwdata;
2344         unsigned int rev, slotgroups;
2345
2346         /*
2347          * HBA base + 0xFC [15:0] - vendor-specific hardware interface
2348          * info register:
2349          * [15:8] hardware/software interface rev#
2350          * [   3] asic-style interface
2351          * [ 2:0] number of slot groups, minus 1 (only valid for asic-style).
2352          */
2353         hwdata = readl(dd->mmio + HOST_HSORG);
2354
2355         dd->product_type = MTIP_PRODUCT_UNKNOWN;
2356         dd->slot_groups = 1;
2357
2358         if (hwdata & 0x8) {
2359                 dd->product_type = MTIP_PRODUCT_ASICFPGA;
2360                 rev = (hwdata & HSORG_HWREV) >> 8;
2361                 slotgroups = (hwdata & HSORG_SLOTGROUPS) + 1;
2362                 dev_info(&dd->pdev->dev,
2363                         "ASIC-FPGA design, HS rev 0x%x, "
2364                         "%i slot groups [%i slots]\n",
2365                          rev,
2366                          slotgroups,
2367                          slotgroups * 32);
2368
2369                 if (slotgroups > MTIP_MAX_SLOT_GROUPS) {
2370                         dev_warn(&dd->pdev->dev,
2371                                 "Warning: driver only supports "
2372                                 "%i slot groups.\n", MTIP_MAX_SLOT_GROUPS);
2373                         slotgroups = MTIP_MAX_SLOT_GROUPS;
2374                 }
2375                 dd->slot_groups = slotgroups;
2376                 return;
2377         }
2378
2379         dev_warn(&dd->pdev->dev, "Unrecognized product id\n");
2380 }
2381
2382 /*
2383  * Blocking wait for FTL rebuild to complete
2384  *
2385  * @dd Pointer to the DRIVER_DATA structure.
2386  *
2387  * return value
2388  *      0       FTL rebuild completed successfully
2389  *      -EFAULT FTL rebuild error/timeout/interruption
2390  */
2391 static int mtip_ftl_rebuild_poll(struct driver_data *dd)
2392 {
2393         unsigned long timeout, cnt = 0, start;
2394
2395         dev_warn(&dd->pdev->dev,
2396                 "FTL rebuild in progress. Polling for completion.\n");
2397
2398         start = jiffies;
2399         dd->ftlrebuildflag = 1;
2400         timeout = jiffies + msecs_to_jiffies(MTIP_FTL_REBUILD_TIMEOUT_MS);
2401
2402         do {
2403 #ifdef CONFIG_HOTPLUG
2404                 if (mtip_check_surprise_removal(dd->pdev))
2405                         return -EFAULT;
2406 #endif
2407                 if (mtip_get_identify(dd->port, NULL) < 0)
2408                         return -EFAULT;
2409
2410                 if (*(dd->port->identify + MTIP_FTL_REBUILD_OFFSET) ==
2411                         MTIP_FTL_REBUILD_MAGIC) {
2412                         ssleep(1);
2413                         /* Print message every 3 minutes */
2414                         if (cnt++ >= 180) {
2415                                 dev_warn(&dd->pdev->dev,
2416                                 "FTL rebuild in progress (%d secs).\n",
2417                                 jiffies_to_msecs(jiffies - start) / 1000);
2418                                 cnt = 0;
2419                         }
2420                 } else {
2421                         dev_warn(&dd->pdev->dev,
2422                                 "FTL rebuild complete (%d secs).\n",
2423                         jiffies_to_msecs(jiffies - start) / 1000);
2424                         dd->ftlrebuildflag = 0;
2425                         break;
2426                 }
2427                 ssleep(10);
2428         } while (time_before(jiffies, timeout));
2429
2430         /* Check for timeout */
2431         if (dd->ftlrebuildflag) {
2432                 dev_err(&dd->pdev->dev,
2433                 "Timed out waiting for FTL rebuild to complete (%d secs).\n",
2434                 jiffies_to_msecs(jiffies - start) / 1000);
2435                 return -EFAULT;
2436         }
2437
2438         return 0;
2439 }
2440
2441 /*
2442  * Called once for each card.
2443  *
2444  * @dd Pointer to the driver data structure.
2445  *
2446  * return value
2447  *      0 on success, else an error code.
2448  */
2449 int mtip_hw_init(struct driver_data *dd)
2450 {
2451         int i;
2452         int rv;
2453         unsigned int num_command_slots;
2454
2455         dd->mmio = pcim_iomap_table(dd->pdev)[MTIP_ABAR];
2456
2457         mtip_detect_product(dd);
2458         if (dd->product_type == MTIP_PRODUCT_UNKNOWN) {
2459                 rv = -EIO;
2460                 goto out1;
2461         }
2462         num_command_slots = dd->slot_groups * 32;
2463
2464         hba_setup(dd);
2465
2466         /*
2467          * Initialize the internal semaphore
2468          * Use a rw semaphore to enable prioritization of
2469          * mgmnt ioctl traffic during heavy IO load
2470          */
2471         init_rwsem(&dd->internal_sem);
2472
2473         tasklet_init(&dd->tasklet, mtip_tasklet, (unsigned long)dd);
2474
2475         dd->port = kzalloc(sizeof(struct mtip_port), GFP_KERNEL);
2476         if (!dd->port) {
2477                 dev_err(&dd->pdev->dev,
2478                         "Memory allocation: port structure\n");
2479                 return -ENOMEM;
2480         }
2481
2482         /* Counting semaphore to track command slot usage */
2483         sema_init(&dd->port->cmd_slot, num_command_slots - 1);
2484
2485         /* Spinlock to prevent concurrent issue */
2486         spin_lock_init(&dd->port->cmd_issue_lock);
2487
2488         /* Set the port mmio base address. */
2489         dd->port->mmio  = dd->mmio + PORT_OFFSET;
2490         dd->port->dd    = dd;
2491
2492         /* Allocate memory for the command list. */
2493         dd->port->command_list =
2494                 dmam_alloc_coherent(&dd->pdev->dev,
2495                         HW_PORT_PRIV_DMA_SZ + (ATA_SECT_SIZE * 2),
2496                         &dd->port->command_list_dma,
2497                         GFP_KERNEL);
2498         if (!dd->port->command_list) {
2499                 dev_err(&dd->pdev->dev,
2500                         "Memory allocation: command list\n");
2501                 rv = -ENOMEM;
2502                 goto out1;
2503         }
2504
2505         /* Clear the memory we have allocated. */
2506         memset(dd->port->command_list,
2507                 0,
2508                 HW_PORT_PRIV_DMA_SZ + (ATA_SECT_SIZE * 2));
2509
2510         /* Setup the addresse of the RX FIS. */
2511         dd->port->rxfis     = dd->port->command_list + HW_CMD_SLOT_SZ;
2512         dd->port->rxfis_dma = dd->port->command_list_dma + HW_CMD_SLOT_SZ;
2513
2514         /* Setup the address of the command tables. */
2515         dd->port->command_table   = dd->port->rxfis + AHCI_RX_FIS_SZ;
2516         dd->port->command_tbl_dma = dd->port->rxfis_dma + AHCI_RX_FIS_SZ;
2517
2518         /* Setup the address of the identify data. */
2519         dd->port->identify     = dd->port->command_table +
2520                                         HW_CMD_TBL_AR_SZ;
2521         dd->port->identify_dma = dd->port->command_tbl_dma +
2522                                         HW_CMD_TBL_AR_SZ;
2523
2524         /* Setup the address of the sector buffer. */
2525         dd->port->sector_buffer = (void *) dd->port->identify + ATA_SECT_SIZE;
2526         dd->port->sector_buffer_dma = dd->port->identify_dma + ATA_SECT_SIZE;
2527
2528         /* Point the command headers at the command tables. */
2529         for (i = 0; i < num_command_slots; i++) {
2530                 dd->port->commands[i].command_header =
2531                                         dd->port->command_list +
2532                                         (sizeof(struct mtip_cmd_hdr) * i);
2533                 dd->port->commands[i].command_header_dma =
2534                                         dd->port->command_list_dma +
2535                                         (sizeof(struct mtip_cmd_hdr) * i);
2536
2537                 dd->port->commands[i].command =
2538                         dd->port->command_table + (HW_CMD_TBL_SZ * i);
2539                 dd->port->commands[i].command_dma =
2540                         dd->port->command_tbl_dma + (HW_CMD_TBL_SZ * i);
2541
2542                 if (readl(dd->mmio + HOST_CAP) & HOST_CAP_64)
2543                         dd->port->commands[i].command_header->ctbau =
2544                         cpu_to_le32(
2545                         (dd->port->commands[i].command_dma >> 16) >> 16);
2546                 dd->port->commands[i].command_header->ctba = cpu_to_le32(
2547                         dd->port->commands[i].command_dma & 0xffffffff);
2548
2549                 /*
2550                  * If this is not done, a bug is reported by the stock
2551                  * FC11 i386. Due to the fact that it has lots of kernel
2552                  * debugging enabled.
2553                  */
2554                 sg_init_table(dd->port->commands[i].sg, MTIP_MAX_SG);
2555
2556                 /* Mark all commands as currently inactive.*/
2557                 atomic_set(&dd->port->commands[i].active, 0);
2558         }
2559
2560         /* Setup the pointers to the extended s_active and CI registers. */
2561         for (i = 0; i < dd->slot_groups; i++) {
2562                 dd->port->s_active[i] =
2563                         dd->port->mmio + i*0x80 + PORT_SCR_ACT;
2564                 dd->port->cmd_issue[i] =
2565                         dd->port->mmio + i*0x80 + PORT_COMMAND_ISSUE;
2566                 dd->port->completed[i] =
2567                         dd->port->mmio + i*0x80 + PORT_SDBV;
2568         }
2569
2570         /* Reset the HBA. */
2571         if (mtip_hba_reset(dd) < 0) {
2572                 dev_err(&dd->pdev->dev,
2573                         "Card did not reset within timeout\n");
2574                 rv = -EIO;
2575                 goto out2;
2576         }
2577
2578         mtip_init_port(dd->port);
2579         mtip_start_port(dd->port);
2580
2581         /* Setup the ISR and enable interrupts. */
2582         rv = devm_request_irq(&dd->pdev->dev,
2583                                 dd->pdev->irq,
2584                                 mtip_irq_handler,
2585                                 IRQF_SHARED,
2586                                 dev_driver_string(&dd->pdev->dev),
2587                                 dd);
2588
2589         if (rv) {
2590                 dev_err(&dd->pdev->dev,
2591                         "Unable to allocate IRQ %d\n", dd->pdev->irq);
2592                 goto out2;
2593         }
2594
2595         /* Enable interrupts on the HBA. */
2596         writel(readl(dd->mmio + HOST_CTL) | HOST_IRQ_EN,
2597                                         dd->mmio + HOST_CTL);
2598
2599         init_timer(&dd->port->cmd_timer);
2600         dd->port->cmd_timer.data = (unsigned long int) dd->port;
2601         dd->port->cmd_timer.function = mtip_timeout_function;
2602         mod_timer(&dd->port->cmd_timer,
2603                 jiffies + msecs_to_jiffies(MTIP_TIMEOUT_CHECK_PERIOD));
2604
2605         if (mtip_get_identify(dd->port, NULL) < 0) {
2606                 rv = -EFAULT;
2607                 goto out3;
2608         }
2609         mtip_dump_identify(dd->port);
2610
2611         if (*(dd->port->identify + MTIP_FTL_REBUILD_OFFSET) ==
2612                 MTIP_FTL_REBUILD_MAGIC) {
2613                 return mtip_ftl_rebuild_poll(dd);
2614         }
2615         return rv;
2616
2617 out3:
2618         del_timer_sync(&dd->port->cmd_timer);
2619
2620         /* Disable interrupts on the HBA. */
2621         writel(readl(dd->mmio + HOST_CTL) & ~HOST_IRQ_EN,
2622                         dd->mmio + HOST_CTL);
2623
2624         /*Release the IRQ. */
2625         devm_free_irq(&dd->pdev->dev, dd->pdev->irq, dd);
2626
2627 out2:
2628         mtip_deinit_port(dd->port);
2629
2630         /* Free the command/command header memory. */
2631         dmam_free_coherent(&dd->pdev->dev,
2632                                 HW_PORT_PRIV_DMA_SZ + (ATA_SECT_SIZE * 2),
2633                                 dd->port->command_list,
2634                                 dd->port->command_list_dma);
2635 out1:
2636         /* Free the memory allocated for the for structure. */
2637         kfree(dd->port);
2638
2639         return rv;
2640 }
2641
2642 /*
2643  * Called to deinitialize an interface.
2644  *
2645  * @dd Pointer to the driver data structure.
2646  *
2647  * return value
2648  *      0
2649  */
2650 int mtip_hw_exit(struct driver_data *dd)
2651 {
2652         /*
2653          * Send standby immediate (E0h) to the drive so that it
2654          * saves its state.
2655          */
2656         if (atomic_read(&dd->drv_cleanup_done) != true) {
2657
2658                 mtip_standby_immediate(dd->port);
2659
2660                 /* de-initialize the port. */
2661                 mtip_deinit_port(dd->port);
2662
2663                 /* Disable interrupts on the HBA. */
2664                 writel(readl(dd->mmio + HOST_CTL) & ~HOST_IRQ_EN,
2665                                 dd->mmio + HOST_CTL);
2666         }
2667
2668         del_timer_sync(&dd->port->cmd_timer);
2669
2670         /* Stop the bottom half tasklet. */
2671         tasklet_kill(&dd->tasklet);
2672
2673         /* Release the IRQ. */
2674         devm_free_irq(&dd->pdev->dev, dd->pdev->irq, dd);
2675
2676         /* Free the command/command header memory. */
2677         dmam_free_coherent(&dd->pdev->dev,
2678                         HW_PORT_PRIV_DMA_SZ + (ATA_SECT_SIZE * 2),
2679                         dd->port->command_list,
2680                         dd->port->command_list_dma);
2681         /* Free the memory allocated for the for structure. */
2682         kfree(dd->port);
2683
2684         return 0;
2685 }
2686
2687 /*
2688  * Issue a Standby Immediate command to the device.
2689  *
2690  * This function is called by the Block Layer just before the
2691  * system powers off during a shutdown.
2692  *
2693  * @dd Pointer to the driver data structure.
2694  *
2695  * return value
2696  *      0
2697  */
2698 int mtip_hw_shutdown(struct driver_data *dd)
2699 {
2700         /*
2701          * Send standby immediate (E0h) to the drive so that it
2702          * saves its state.
2703          */
2704         mtip_standby_immediate(dd->port);
2705
2706         return 0;
2707 }
2708
2709 /*
2710  * Suspend function
2711  *
2712  * This function is called by the Block Layer just before the
2713  * system hibernates.
2714  *
2715  * @dd Pointer to the driver data structure.
2716  *
2717  * return value
2718  *      0       Suspend was successful
2719  *      -EFAULT Suspend was not successful
2720  */
2721 int mtip_hw_suspend(struct driver_data *dd)
2722 {
2723         /*
2724          * Send standby immediate (E0h) to the drive
2725          * so that it saves its state.
2726          */
2727         if (mtip_standby_immediate(dd->port) != 0) {
2728                 dev_err(&dd->pdev->dev,
2729                         "Failed standby-immediate command\n");
2730                 return -EFAULT;
2731         }
2732
2733         /* Disable interrupts on the HBA.*/
2734         writel(readl(dd->mmio + HOST_CTL) & ~HOST_IRQ_EN,
2735                         dd->mmio + HOST_CTL);
2736         mtip_deinit_port(dd->port);
2737
2738         return 0;
2739 }
2740
2741 /*
2742  * Resume function
2743  *
2744  * This function is called by the Block Layer as the
2745  * system resumes.
2746  *
2747  * @dd Pointer to the driver data structure.
2748  *
2749  * return value
2750  *      0       Resume was successful
2751  *      -EFAULT Resume was not successful
2752  */
2753 int mtip_hw_resume(struct driver_data *dd)
2754 {
2755         /* Perform any needed hardware setup steps */
2756         hba_setup(dd);
2757
2758         /* Reset the HBA */
2759         if (mtip_hba_reset(dd) != 0) {
2760                 dev_err(&dd->pdev->dev,
2761                         "Unable to reset the HBA\n");
2762                 return -EFAULT;
2763         }
2764
2765         /*
2766          * Enable the port, DMA engine, and FIS reception specific
2767          * h/w in controller.
2768          */
2769         mtip_init_port(dd->port);
2770         mtip_start_port(dd->port);
2771
2772         /* Enable interrupts on the HBA.*/
2773         writel(readl(dd->mmio + HOST_CTL) | HOST_IRQ_EN,
2774                         dd->mmio + HOST_CTL);
2775
2776         return 0;
2777 }
2778
2779 /*
2780  * This function is called for clean the pending command in the
2781  * command slot during the surprise removal of device and return
2782  * error to the upper layer.
2783  *
2784  * @dd Pointer to the DRIVER_DATA structure.
2785  *
2786  * return value
2787  *      None
2788  */
2789 void mtip_command_cleanup(struct driver_data *dd)
2790 {
2791         int group = 0, commandslot = 0, commandindex = 0;
2792         struct mtip_cmd *command;
2793         struct mtip_port *port = dd->port;
2794
2795         for (group = 0; group < 4; group++) {
2796                 for (commandslot = 0; commandslot < 32; commandslot++) {
2797                         if (!(port->allocated[group] & (1 << commandslot)))
2798                                 continue;
2799
2800                         commandindex = group << 5 | commandslot;
2801                         command = &port->commands[commandindex];
2802
2803                         if (atomic_read(&command->active)
2804                             && (command->async_callback)) {
2805                                 command->async_callback(command->async_data,
2806                                         -ENODEV);
2807                                 command->async_callback = NULL;
2808                                 command->async_data = NULL;
2809                         }
2810
2811                         dma_unmap_sg(&port->dd->pdev->dev,
2812                                 command->sg,
2813                                 command->scatter_ents,
2814                                 command->direction);
2815                 }
2816         }
2817
2818         up(&port->cmd_slot);
2819
2820         atomic_set(&dd->drv_cleanup_done, true);
2821 }
2822
2823 /*
2824  * Helper function for reusing disk name
2825  * upon hot insertion.
2826  */
2827 static int rssd_disk_name_format(char *prefix,
2828                                  int index,
2829                                  char *buf,
2830                                  int buflen)
2831 {
2832         const int base = 'z' - 'a' + 1;
2833         char *begin = buf + strlen(prefix);
2834         char *end = buf + buflen;
2835         char *p;
2836         int unit;
2837
2838         p = end - 1;
2839         *p = '\0';
2840         unit = base;
2841         do {
2842                 if (p == begin)
2843                         return -EINVAL;
2844                 *--p = 'a' + (index % unit);
2845                 index = (index / unit) - 1;
2846         } while (index >= 0);
2847
2848         memmove(begin, p, end - p);
2849         memcpy(buf, prefix, strlen(prefix));
2850
2851         return 0;
2852 }
2853
2854 /*
2855  * Block layer IOCTL handler.
2856  *
2857  * @dev Pointer to the block_device structure.
2858  * @mode ignored
2859  * @cmd IOCTL command passed from the user application.
2860  * @arg Argument passed from the user application.
2861  *
2862  * return value
2863  *      0        IOCTL completed successfully.
2864  *      -ENOTTY  IOCTL not supported or invalid driver data
2865  *                 structure pointer.
2866  */
2867 static int mtip_block_ioctl(struct block_device *dev,
2868                             fmode_t mode,
2869                             unsigned cmd,
2870                             unsigned long arg)
2871 {
2872         struct driver_data *dd = dev->bd_disk->private_data;
2873
2874         if (!capable(CAP_SYS_ADMIN))
2875                 return -EACCES;
2876
2877         if (!dd)
2878                 return -ENOTTY;
2879
2880         switch (cmd) {
2881         case BLKFLSBUF:
2882                 return 0;
2883         default:
2884                 return mtip_hw_ioctl(dd, cmd, arg, 0);
2885         }
2886 }
2887
2888 #ifdef CONFIG_COMPAT
2889 /*
2890  * Block layer compat IOCTL handler.
2891  *
2892  * @dev Pointer to the block_device structure.
2893  * @mode ignored
2894  * @cmd IOCTL command passed from the user application.
2895  * @arg Argument passed from the user application.
2896  *
2897  * return value
2898  *      0        IOCTL completed successfully.
2899  *      -ENOTTY  IOCTL not supported or invalid driver data
2900  *                 structure pointer.
2901  */
2902 static int mtip_block_compat_ioctl(struct block_device *dev,
2903                             fmode_t mode,
2904                             unsigned cmd,
2905                             unsigned long arg)
2906 {
2907         struct driver_data *dd = dev->bd_disk->private_data;
2908
2909         if (!capable(CAP_SYS_ADMIN))
2910                 return -EACCES;
2911
2912         if (!dd)
2913                 return -ENOTTY;
2914
2915         switch (cmd) {
2916         case BLKFLSBUF:
2917                 return 0;
2918         default:
2919                 return mtip_hw_ioctl(dd, cmd, arg, 1);
2920         }
2921 }
2922 #endif
2923
2924 /*
2925  * Obtain the geometry of the device.
2926  *
2927  * You may think that this function is obsolete, but some applications,
2928  * fdisk for example still used CHS values. This function describes the
2929  * device as having 224 heads and 56 sectors per cylinder. These values are
2930  * chosen so that each cylinder is aligned on a 4KB boundary. Since a
2931  * partition is described in terms of a start and end cylinder this means
2932  * that each partition is also 4KB aligned. Non-aligned partitions adversely
2933  * affects performance.
2934  *
2935  * @dev Pointer to the block_device strucutre.
2936  * @geo Pointer to a hd_geometry structure.
2937  *
2938  * return value
2939  *      0       Operation completed successfully.
2940  *      -ENOTTY An error occurred while reading the drive capacity.
2941  */
2942 static int mtip_block_getgeo(struct block_device *dev,
2943                                 struct hd_geometry *geo)
2944 {
2945         struct driver_data *dd = dev->bd_disk->private_data;
2946         sector_t capacity;
2947
2948         if (!dd)
2949                 return -ENOTTY;
2950
2951         if (!(mtip_hw_get_capacity(dd, &capacity))) {
2952                 dev_warn(&dd->pdev->dev,
2953                         "Could not get drive capacity.\n");
2954                 return -ENOTTY;
2955         }
2956
2957         geo->heads = 224;
2958         geo->sectors = 56;
2959 #if BITS_PER_LONG == 64
2960         geo->cylinders = capacity / (geo->heads * geo->sectors);
2961 #else
2962         do_div(capacity, (geo->heads * geo->sectors));
2963         geo->cylinders = capacity;
2964 #endif
2965         return 0;
2966 }
2967
2968 /*
2969  * Block device operation function.
2970  *
2971  * This structure contains pointers to the functions required by the block
2972  * layer.
2973  */
2974 static const struct block_device_operations mtip_block_ops = {
2975         .ioctl          = mtip_block_ioctl,
2976 #ifdef CONFIG_COMPAT
2977         .compat_ioctl   = mtip_block_compat_ioctl,
2978 #endif
2979         .getgeo         = mtip_block_getgeo,
2980         .owner          = THIS_MODULE
2981 };
2982
2983 /*
2984  * Block layer make request function.
2985  *
2986  * This function is called by the kernel to process a BIO for
2987  * the P320 device.
2988  *
2989  * @queue Pointer to the request queue. Unused other than to obtain
2990  *              the driver data structure.
2991  * @bio   Pointer to the BIO.
2992  *
2993  * return value
2994  *      0
2995  */
2996 static int mtip_make_request(struct request_queue *queue, struct bio *bio)
2997 {
2998         struct driver_data *dd = queue->queuedata;
2999         struct scatterlist *sg;
3000         struct bio_vec *bvec;
3001         int nents = 0;
3002         int tag = 0;
3003
3004         if (unlikely(!bio_has_data(bio))) {
3005                 blk_queue_flush(queue, 0);
3006                 bio_endio(bio, 0);
3007                 return 0;
3008         }
3009
3010         if (unlikely(atomic_read(&dd->eh_active))) {
3011                 bio_endio(bio, -EBUSY);
3012                 return 0;
3013         }
3014
3015         sg = mtip_hw_get_scatterlist(dd, &tag);
3016         if (likely(sg != NULL)) {
3017                 blk_queue_bounce(queue, &bio);
3018
3019                 if (unlikely((bio)->bi_vcnt > MTIP_MAX_SG)) {
3020                         dev_warn(&dd->pdev->dev,
3021                                 "Maximum number of SGL entries exceeded");
3022                         bio_io_error(bio);
3023                         mtip_hw_release_scatterlist(dd, tag);
3024                         return 0;
3025                 }
3026
3027                 /* Create the scatter list for this bio. */
3028                 bio_for_each_segment(bvec, bio, nents) {
3029                         sg_set_page(&sg[nents],
3030                                         bvec->bv_page,
3031                                         bvec->bv_len,
3032                                         bvec->bv_offset);
3033                 }
3034
3035                 /* Issue the read/write. */
3036                 mtip_hw_submit_io(dd,
3037                                 bio->bi_sector,
3038                                 bio_sectors(bio),
3039                                 nents,
3040                                 tag,
3041                                 bio_endio,
3042                                 bio,
3043                                 bio->bi_rw & REQ_FLUSH,
3044                                 bio_data_dir(bio));
3045         } else {
3046                 bio_io_error(bio);
3047         }
3048
3049         return 0;
3050 }
3051
3052 /*
3053  * Block layer initialization function.
3054  *
3055  * This function is called once by the PCI layer for each P320
3056  * device that is connected to the system.
3057  *
3058  * @dd Pointer to the driver data structure.
3059  *
3060  * return value
3061  *      0 on success else an error code.
3062  */
3063 int mtip_block_initialize(struct driver_data *dd)
3064 {
3065         int rv = 0;
3066         sector_t capacity;
3067         unsigned int index = 0;
3068         struct kobject *kobj;
3069
3070         /* Initialize the protocol layer. */
3071         rv = mtip_hw_init(dd);
3072         if (rv < 0) {
3073                 dev_err(&dd->pdev->dev,
3074                         "Protocol layer initialization failed\n");
3075                 rv = -EINVAL;
3076                 goto protocol_init_error;
3077         }
3078
3079         /* Allocate the request queue. */
3080         dd->queue = blk_alloc_queue(GFP_KERNEL);
3081         if (dd->queue == NULL) {
3082                 dev_err(&dd->pdev->dev,
3083                         "Unable to allocate request queue\n");
3084                 rv = -ENOMEM;
3085                 goto block_queue_alloc_init_error;
3086         }
3087
3088         /* Attach our request function to the request queue. */
3089         blk_queue_make_request(dd->queue, mtip_make_request);
3090
3091         /* Set device limits. */
3092         set_bit(QUEUE_FLAG_NONROT, &dd->queue->queue_flags);
3093         blk_queue_max_segments(dd->queue, MTIP_MAX_SG);
3094         blk_queue_physical_block_size(dd->queue, 4096);
3095         blk_queue_io_min(dd->queue, 4096);
3096
3097         dd->disk = alloc_disk(MTIP_MAX_MINORS);
3098         if (dd->disk  == NULL) {
3099                 dev_err(&dd->pdev->dev,
3100                         "Unable to allocate gendisk structure\n");
3101                 rv = -EINVAL;
3102                 goto alloc_disk_error;
3103         }
3104
3105         /* Generate the disk name, implemented same as in sd.c */
3106         do {
3107                 if (!ida_pre_get(&rssd_index_ida, GFP_KERNEL))
3108                         goto ida_get_error;
3109
3110                 spin_lock(&rssd_index_lock);
3111                 rv = ida_get_new(&rssd_index_ida, &index);
3112                 spin_unlock(&rssd_index_lock);
3113         } while (rv == -EAGAIN);
3114
3115         if (rv)
3116                 goto ida_get_error;
3117
3118         rv = rssd_disk_name_format("rssd",
3119                                 index,
3120                                 dd->disk->disk_name,
3121                                 DISK_NAME_LEN);
3122         if (rv)
3123                 goto disk_index_error;
3124
3125         dd->disk->driverfs_dev  = &dd->pdev->dev;
3126         dd->disk->major         = dd->major;
3127         dd->disk->first_minor   = dd->instance * MTIP_MAX_MINORS;
3128         dd->disk->fops          = &mtip_block_ops;
3129         dd->disk->queue         = dd->queue;
3130         dd->disk->private_data  = dd;
3131         dd->queue->queuedata    = dd;
3132         dd->index               = index;
3133
3134         /* Set the capacity of the device in 512 byte sectors. */
3135         if (!(mtip_hw_get_capacity(dd, &capacity))) {
3136                 dev_warn(&dd->pdev->dev,
3137                         "Could not read drive capacity\n");
3138                 rv = -EIO;
3139                 goto read_capacity_error;
3140         }
3141         set_capacity(dd->disk, capacity);
3142
3143         /* Enable the block device and add it to /dev */
3144         add_disk(dd->disk);
3145
3146         /*
3147          * Now that the disk is active, initialize any sysfs attributes
3148          * managed by the protocol layer.
3149          */
3150         kobj = kobject_get(&disk_to_dev(dd->disk)->kobj);
3151         if (kobj) {
3152                 mtip_hw_sysfs_init(dd, kobj);
3153                 kobject_put(kobj);
3154         }
3155
3156         return rv;
3157
3158 read_capacity_error:
3159         /*
3160          * Delete our gendisk structure. This also removes the device
3161          * from /dev
3162          */
3163         del_gendisk(dd->disk);
3164
3165 disk_index_error:
3166         spin_lock(&rssd_index_lock);
3167         ida_remove(&rssd_index_ida, index);
3168         spin_unlock(&rssd_index_lock);
3169
3170 ida_get_error:
3171         put_disk(dd->disk);
3172
3173 alloc_disk_error:
3174         blk_cleanup_queue(dd->queue);
3175
3176 block_queue_alloc_init_error:
3177         /* De-initialize the protocol layer. */
3178         mtip_hw_exit(dd);
3179
3180 protocol_init_error:
3181         return rv;
3182 }
3183
3184 /*
3185  * Block layer deinitialization function.
3186  *
3187  * Called by the PCI layer as each P320 device is removed.
3188  *
3189  * @dd Pointer to the driver data structure.
3190  *
3191  * return value
3192  *      0
3193  */
3194 int mtip_block_remove(struct driver_data *dd)
3195 {
3196         struct kobject *kobj;
3197         /* Clean up the sysfs attributes managed by the protocol layer. */
3198         kobj = kobject_get(&disk_to_dev(dd->disk)->kobj);
3199         if (kobj) {
3200                 mtip_hw_sysfs_exit(dd, kobj);
3201                 kobject_put(kobj);
3202         }
3203
3204         /*
3205          * Delete our gendisk structure. This also removes the device
3206          * from /dev
3207          */
3208         del_gendisk(dd->disk);
3209         blk_cleanup_queue(dd->queue);
3210         dd->disk  = NULL;
3211         dd->queue = NULL;
3212
3213         /* De-initialize the protocol layer. */
3214         mtip_hw_exit(dd);
3215
3216         return 0;
3217 }
3218
3219 /*
3220  * Function called by the PCI layer when just before the
3221  * machine shuts down.
3222  *
3223  * If a protocol layer shutdown function is present it will be called
3224  * by this function.
3225  *
3226  * @dd Pointer to the driver data structure.
3227  *
3228  * return value
3229  *      0
3230  */
3231 int mtip_block_shutdown(struct driver_data *dd)
3232 {
3233         dev_info(&dd->pdev->dev,
3234                 "Shutting down %s ...\n", dd->disk->disk_name);
3235
3236         /* Delete our gendisk structure, and cleanup the blk queue. */
3237         del_gendisk(dd->disk);
3238         blk_cleanup_queue(dd->queue);
3239         dd->disk  = NULL;
3240         dd->queue = NULL;
3241
3242         mtip_hw_shutdown(dd);
3243         return 0;
3244 }
3245
3246 int mtip_block_suspend(struct driver_data *dd)
3247 {
3248         dev_info(&dd->pdev->dev,
3249                 "Suspending %s ...\n", dd->disk->disk_name);
3250         mtip_hw_suspend(dd);
3251         return 0;
3252 }
3253
3254 int mtip_block_resume(struct driver_data *dd)
3255 {
3256         dev_info(&dd->pdev->dev, "Resuming %s ...\n",
3257                 dd->disk->disk_name);
3258         mtip_hw_resume(dd);
3259         return 0;
3260 }
3261
3262 /*
3263  * Called for each supported PCI device detected.
3264  *
3265  * This function allocates the private data structure, enables the
3266  * PCI device and then calls the block layer initialization function.
3267  *
3268  * return value
3269  *      0 on success else an error code.
3270  */
3271 static int mtip_pci_probe(struct pci_dev *pdev,
3272                         const struct pci_device_id *ent)
3273 {
3274         int rv = 0;
3275         struct driver_data *dd = NULL;
3276
3277         /* Allocate memory for this devices private data. */
3278         dd = kzalloc(sizeof(struct driver_data), GFP_KERNEL);
3279         if (dd == NULL) {
3280                 dev_err(&pdev->dev,
3281                         "Unable to allocate memory for driver data\n");
3282                 return -ENOMEM;
3283         }
3284
3285         /* Set the atomic variable as 1 in case of SRSI */
3286         atomic_set(&dd->drv_cleanup_done, true);
3287
3288         atomic_set(&dd->resumeflag, false);
3289         atomic_set(&dd->eh_active, 0);
3290
3291         /* Attach the private data to this PCI device.  */
3292         pci_set_drvdata(pdev, dd);
3293
3294         rv = pcim_enable_device(pdev);
3295         if (rv < 0) {
3296                 dev_err(&pdev->dev, "Unable to enable device\n");
3297                 goto iomap_err;
3298         }
3299
3300         /* Map BAR5 to memory. */
3301         rv = pcim_iomap_regions(pdev, 1 << MTIP_ABAR, MTIP_DRV_NAME);
3302         if (rv < 0) {
3303                 dev_err(&pdev->dev, "Unable to map regions\n");
3304                 goto iomap_err;
3305         }
3306
3307         if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
3308                 rv = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
3309
3310                 if (rv) {
3311                         rv = pci_set_consistent_dma_mask(pdev,
3312                                                 DMA_BIT_MASK(32));
3313                         if (rv) {
3314                                 dev_warn(&pdev->dev,
3315                                         "64-bit DMA enable failed\n");
3316                                 goto setmask_err;
3317                         }
3318                 }
3319         }
3320
3321         pci_set_master(pdev);
3322
3323         if (pci_enable_msi(pdev)) {
3324                 dev_warn(&pdev->dev,
3325                         "Unable to enable MSI interrupt.\n");
3326                 goto block_initialize_err;
3327         }
3328
3329         /* Copy the info we may need later into the private data structure. */
3330         dd->major       = mtip_major;
3331         dd->protocol    = ent->driver_data;
3332         dd->instance    = instance;
3333         dd->pdev        = pdev;
3334
3335         /* Initialize the block layer. */
3336         rv = mtip_block_initialize(dd);
3337         if (rv < 0) {
3338                 dev_err(&pdev->dev,
3339                         "Unable to initialize block layer\n");
3340                 goto block_initialize_err;
3341         }
3342
3343         /*
3344          * Increment the instance count so that each device has a unique
3345          * instance number.
3346          */
3347         instance++;
3348
3349         goto done;
3350
3351 block_initialize_err:
3352         pci_disable_msi(pdev);
3353
3354 setmask_err:
3355         pcim_iounmap_regions(pdev, 1 << MTIP_ABAR);
3356
3357 iomap_err:
3358         kfree(dd);
3359         pci_set_drvdata(pdev, NULL);
3360         return rv;
3361 done:
3362         /* Set the atomic variable as 0 in case of SRSI */
3363         atomic_set(&dd->drv_cleanup_done, true);
3364
3365         return rv;
3366 }
3367
3368 /*
3369  * Called for each probed device when the device is removed or the
3370  * driver is unloaded.
3371  *
3372  * return value
3373  *      None
3374  */
3375 static void mtip_pci_remove(struct pci_dev *pdev)
3376 {
3377         struct driver_data *dd = pci_get_drvdata(pdev);
3378         int counter = 0;
3379
3380         if (mtip_check_surprise_removal(pdev)) {
3381                 while (atomic_read(&dd->drv_cleanup_done) == false) {
3382                         counter++;
3383                         msleep(20);
3384                         if (counter == 10) {
3385                                 /* Cleanup the outstanding commands */
3386                                 mtip_command_cleanup(dd);
3387                                 break;
3388                         }
3389                 }
3390         }
3391         /* Set the atomic variable as 1 in case of SRSI */
3392         atomic_set(&dd->drv_cleanup_done, true);
3393
3394         /* Clean up the block layer. */
3395         mtip_block_remove(dd);
3396
3397         pci_disable_msi(pdev);
3398
3399         kfree(dd);
3400         pcim_iounmap_regions(pdev, 1 << MTIP_ABAR);
3401 }
3402
3403 /*
3404  * Called for each probed device when the device is suspended.
3405  *
3406  * return value
3407  *      0  Success
3408  *      <0 Error
3409  */
3410 static int mtip_pci_suspend(struct pci_dev *pdev, pm_message_t mesg)
3411 {
3412         int rv = 0;
3413         struct driver_data *dd = pci_get_drvdata(pdev);
3414
3415         if (!dd) {
3416                 dev_err(&pdev->dev,
3417                         "Driver private datastructure is NULL\n");
3418                 return -EFAULT;
3419         }
3420
3421         atomic_set(&dd->resumeflag, true);
3422
3423         /* Disable ports & interrupts then send standby immediate */
3424         rv = mtip_block_suspend(dd);
3425         if (rv < 0) {
3426                 dev_err(&pdev->dev,
3427                         "Failed to suspend controller\n");
3428                 return rv;
3429         }
3430
3431         /*
3432          * Save the pci config space to pdev structure &
3433          * disable the device
3434          */
3435         pci_save_state(pdev);
3436         pci_disable_device(pdev);
3437
3438         /* Move to Low power state*/
3439         pci_set_power_state(pdev, PCI_D3hot);
3440
3441         return rv;
3442 }
3443
3444 /*
3445  * Called for each probed device when the device is resumed.
3446  *
3447  * return value
3448  *      0  Success
3449  *      <0 Error
3450  */
3451 static int mtip_pci_resume(struct pci_dev *pdev)
3452 {
3453         int rv = 0;
3454         struct driver_data *dd;
3455
3456         dd = pci_get_drvdata(pdev);
3457         if (!dd) {
3458                 dev_err(&pdev->dev,
3459                         "Driver private datastructure is NULL\n");
3460                 return -EFAULT;
3461         }
3462
3463         /* Move the device to active State */
3464         pci_set_power_state(pdev, PCI_D0);
3465
3466         /* Restore PCI configuration space */
3467         pci_restore_state(pdev);
3468
3469         /* Enable the PCI device*/
3470         rv = pcim_enable_device(pdev);
3471         if (rv < 0) {
3472                 dev_err(&pdev->dev,
3473                         "Failed to enable card during resume\n");
3474                 goto err;
3475         }
3476         pci_set_master(pdev);
3477
3478         /*
3479          * Calls hbaReset, initPort, & startPort function
3480          * then enables interrupts
3481          */
3482         rv = mtip_block_resume(dd);
3483         if (rv < 0)
3484                 dev_err(&pdev->dev, "Unable to resume\n");
3485
3486 err:
3487         atomic_set(&dd->resumeflag, false);
3488
3489         return rv;
3490 }
3491
3492 /*
3493  * Shutdown routine
3494  *
3495  * return value
3496  *      None
3497  */
3498 static void mtip_pci_shutdown(struct pci_dev *pdev)
3499 {
3500         struct driver_data *dd = pci_get_drvdata(pdev);
3501         if (dd)
3502                 mtip_block_shutdown(dd);
3503 }
3504
3505 /*
3506  * This function check_for_surprise_removal is called
3507  * while card is removed from the system and it will
3508  * read the vendor id from the configration space
3509  *
3510  * @pdev Pointer to the pci_dev structure.
3511  *
3512  * return value
3513  *       true if device removed, else false
3514  */
3515 bool mtip_check_surprise_removal(struct pci_dev *pdev)
3516 {
3517         u16 vendor_id = 0;
3518
3519        /* Read the vendorID from the configuration space */
3520         pci_read_config_word(pdev, 0x00, &vendor_id);
3521         if (vendor_id == 0xFFFF)
3522                 return true; /* device removed */
3523
3524         return false; /* device present */
3525 }
3526
3527 /* Table of device ids supported by this driver. */
3528 static DEFINE_PCI_DEVICE_TABLE(mtip_pci_tbl) = {
3529         {  PCI_DEVICE(PCI_VENDOR_ID_MICRON, P320_DEVICE_ID) },
3530         { 0 }
3531 };
3532
3533 /* Structure that describes the PCI driver functions. */
3534 struct pci_driver mtip_pci_driver = {
3535         .name                   = MTIP_DRV_NAME,
3536         .id_table               = mtip_pci_tbl,
3537         .probe                  = mtip_pci_probe,
3538         .remove                 = mtip_pci_remove,
3539         .suspend                = mtip_pci_suspend,
3540         .resume                 = mtip_pci_resume,
3541         .shutdown               = mtip_pci_shutdown,
3542 };
3543
3544 MODULE_DEVICE_TABLE(pci, mtip_pci_tbl);
3545
3546 /*
3547  * Module initialization function.
3548  *
3549  * Called once when the module is loaded. This function allocates a major
3550  * block device number to the Cyclone devices and registers the PCI layer
3551  * of the driver.
3552  *
3553  * Return value
3554  *      0 on success else error code.
3555  */
3556 static int __init mtip_init(void)
3557 {
3558         printk(KERN_INFO MTIP_DRV_NAME " Version " MTIP_DRV_VERSION "\n");
3559
3560         /* Allocate a major block device number to use with this driver. */
3561         mtip_major = register_blkdev(0, MTIP_DRV_NAME);
3562         if (mtip_major < 0) {
3563                 printk(KERN_ERR "Unable to register block device (%d)\n",
3564                 mtip_major);
3565                 return -EBUSY;
3566         }
3567
3568         /* Register our PCI operations. */
3569         return pci_register_driver(&mtip_pci_driver);
3570 }
3571
3572 /*
3573  * Module de-initialization function.
3574  *
3575  * Called once when the module is unloaded. This function deallocates
3576  * the major block device number allocated by mtip_init() and
3577  * unregisters the PCI layer of the driver.
3578  *
3579  * Return value
3580  *      none
3581  */
3582 static void __exit mtip_exit(void)
3583 {
3584         /* Release the allocated major block device number. */
3585         unregister_blkdev(mtip_major, MTIP_DRV_NAME);
3586
3587         /* Unregister the PCI driver. */
3588         pci_unregister_driver(&mtip_pci_driver);
3589 }
3590
3591 MODULE_AUTHOR("Micron Technology, Inc");
3592 MODULE_DESCRIPTION("Micron RealSSD PCIe Block Driver");
3593 MODULE_LICENSE("GPL");
3594 MODULE_VERSION(MTIP_DRV_VERSION);
3595
3596 module_init(mtip_init);
3597 module_exit(mtip_exit);