drivers/block/rsxx/core.c

   1 /*
   2 * Filename: core.c
   3 *
   4 *
   5 * Authors: Joshua Morris <josh.h.morris@us.ibm.com>
   6 *       Philip Kelleher <pjk1939@linux.vnet.ibm.com>
   7 *
   8 * (C) Copyright 2013 IBM Corporation
   9 *
  10 * This program is free software; you can redistribute it and/or
  11 * modify it under the terms of the GNU General Public License as
  12 * published by the Free Software Foundation; either version 2 of the
  13 * License, or (at your option) any later version.
  14 *
  15 * This program is distributed in the hope that it will be useful, but
  16 * WITHOUT ANY WARRANTY; without even the implied warranty of
  17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  18 * General Public License for more details.
  19 *
  20 * You should have received a copy of the GNU General Public License
  21 * along with this program; if not, write to the Free Software Foundation,
  22 * Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  23 */
  24
  25 #include <linux/kernel.h>
  26 #include <linux/init.h>
  27 #include <linux/interrupt.h>
  28 #include <linux/module.h>
  29 #include <linux/pci.h>
  30 #include <linux/reboot.h>
  31 #include <linux/slab.h>
  32 #include <linux/bitops.h>
  33 #include <linux/delay.h>
  34
  35 #include <linux/genhd.h>
  36 #include <linux/idr.h>
  37
  38 #include "rsxx_priv.h"
  39 #include "rsxx_cfg.h"
  40
  41 #define NO_LEGACY 0
  42
  43 MODULE_DESCRIPTION("IBM FlashSystem 70/80 PCIe SSD Device Driver");
  44 MODULE_AUTHOR("Joshua Morris/Philip Kelleher, IBM");
  45 MODULE_LICENSE("GPL");
  46 MODULE_VERSION(DRIVER_VERSION);
  47
  48 static unsigned int force_legacy = NO_LEGACY;
  49 module_param(force_legacy, uint, 0444);
  50 MODULE_PARM_DESC(force_legacy, "Force the use of legacy type PCI interrupts");
  51
  52 static DEFINE_IDA(rsxx_disk_ida);
  53 static DEFINE_SPINLOCK(rsxx_ida_lock);
  54
  55 /*----------------- Interrupt Control & Handling -------------------*/
  56
  57 static void rsxx_mask_interrupts(struct rsxx_cardinfo *card)
  58 {
  59         card->isr_mask = 0;
  60         card->ier_mask = 0;
  61 }
  62
  63 static void __enable_intr(unsigned int *mask, unsigned int intr)
  64 {
  65         *mask |= intr;
  66 }
  67
  68 static void __disable_intr(unsigned int *mask, unsigned int intr)
  69 {
  70         *mask &= ~intr;
  71 }
  72
  73 /*
  74  * NOTE: Disabling the IER will disable the hardware interrupt.
  75  * Disabling the ISR will disable the software handling of the ISR bit.
  76  *
  77  * Enable/Disable interrupt functions assume the card->irq_lock
  78  * is held by the caller.
  79  */
  80 void rsxx_enable_ier(struct rsxx_cardinfo *card, unsigned int intr)
  81 {
  82         if (unlikely(card->halt) ||
  83             unlikely(card->eeh_state))
  84                 return;
  85
  86         __enable_intr(&card->ier_mask, intr);
  87         iowrite32(card->ier_mask, card->regmap + IER);
  88 }
  89
  90 void rsxx_disable_ier(struct rsxx_cardinfo *card, unsigned int intr)
  91 {
  92         if (unlikely(card->eeh_state))
  93                 return;
  94
  95         __disable_intr(&card->ier_mask, intr);
  96         iowrite32(card->ier_mask, card->regmap + IER);
  97 }
  98
  99 void rsxx_enable_ier_and_isr(struct rsxx_cardinfo *card,
 100                                  unsigned int intr)
 101 {
 102         if (unlikely(card->halt) ||
 103             unlikely(card->eeh_state))
 104                 return;
 105
 106         __enable_intr(&card->isr_mask, intr);
 107         __enable_intr(&card->ier_mask, intr);
 108         iowrite32(card->ier_mask, card->regmap + IER);
 109 }
 110 void rsxx_disable_ier_and_isr(struct rsxx_cardinfo *card,
 111                                   unsigned int intr)
 112 {
 113         if (unlikely(card->eeh_state))
 114                 return;
 115
 116         __disable_intr(&card->isr_mask, intr);
 117         __disable_intr(&card->ier_mask, intr);
 118         iowrite32(card->ier_mask, card->regmap + IER);
 119 }
 120
 121 static irqreturn_t rsxx_isr(int irq, void *pdata)
 122 {
 123         struct rsxx_cardinfo *card = pdata;
 124         unsigned int isr;
 125         int handled = 0;
 126         int reread_isr;
 127         int i;
 128
 129         spin_lock(&card->irq_lock);
 130
 131         do {
 132                 reread_isr = 0;
 133
 134                 if (unlikely(card->eeh_state))
 135                         break;
 136
 137                 isr = ioread32(card->regmap + ISR);
 138                 if (isr == 0xffffffff) {
 139                         /*
 140                          * A few systems seem to have an intermittent issue
 141                          * where PCI reads return all Fs, but retrying the read
 142                          * a little later will return as expected.
 143                          */
 144                         dev_info(CARD_TO_DEV(card),
 145                                 "ISR = 0xFFFFFFFF, retrying later\n");
 146                         break;
 147                 }
 148
 149                 isr &= card->isr_mask;
 150                 if (!isr)
 151                         break;
 152
 153                 for (i = 0; i < card->n_targets; i++) {
 154                         if (isr & CR_INTR_DMA(i)) {
 155                                 if (card->ier_mask & CR_INTR_DMA(i)) {
 156                                         rsxx_disable_ier(card, CR_INTR_DMA(i));
 157                                         reread_isr = 1;
 158                                 }
 159                                 queue_work(card->ctrl[i].done_wq,
 160                                            &card->ctrl[i].dma_done_work);
 161                                 handled++;
 162                         }
 163                 }
 164
 165                 if (isr & CR_INTR_CREG) {
 166                         schedule_work(&card->creg_ctrl.done_work);
 167                         handled++;
 168                 }
 169
 170                 if (isr & CR_INTR_EVENT) {
 171                         schedule_work(&card->event_work);
 172                         rsxx_disable_ier_and_isr(card, CR_INTR_EVENT);
 173                         handled++;
 174                 }
 175         } while (reread_isr);
 176
 177         spin_unlock(&card->irq_lock);
 178
 179         return handled ? IRQ_HANDLED : IRQ_NONE;
 180 }
 181
 182 /*----------------- Card Event Handler -------------------*/
 183 static const char * const rsxx_card_state_to_str(unsigned int state)
 184 {
 185         static const char * const state_strings[] = {
 186                 "Unknown", "Shutdown", "Starting", "Formatting",
 187                 "Uninitialized", "Good", "Shutting Down",
 188                 "Fault", "Read Only Fault", "dStroying"
 189         };
 190
 191         return state_strings[ffs(state)];
 192 }
 193
 194 static void card_state_change(struct rsxx_cardinfo *card,
 195                               unsigned int new_state)
 196 {
 197         int st;
 198
 199         dev_info(CARD_TO_DEV(card),
 200                 "card state change detected.(%s -> %s)\n",
 201                 rsxx_card_state_to_str(card->state),
 202                 rsxx_card_state_to_str(new_state));
 203
 204         card->state = new_state;
 205
 206         /* Don't attach DMA interfaces if the card has an invalid config */
 207         if (!card->config_valid)
 208                 return;
 209
 210         switch (new_state) {
 211         case CARD_STATE_RD_ONLY_FAULT:
 212                 dev_crit(CARD_TO_DEV(card),
 213                         "Hardware has entered read-only mode!\n");
 214                 /*
 215                  * Fall through so the DMA devices can be attached and
 216                  * the user can attempt to pull off their data.
 217                  */
 218         case CARD_STATE_GOOD:
 219                 st = rsxx_get_card_size8(card, &card->size8);
 220                 if (st)
 221                         dev_err(CARD_TO_DEV(card),
 222                                 "Failed attaching DMA devices\n");
 223
 224                 if (card->config_valid)
 225                         set_capacity(card->gendisk, card->size8 >> 9);
 226                 break;
 227
 228         case CARD_STATE_FAULT:
 229                 dev_crit(CARD_TO_DEV(card),
 230                         "Hardware Fault reported!\n");
 231                 /* Fall through. */
 232
 233         /* Everything else, detach DMA interface if it's attached. */
 234         case CARD_STATE_SHUTDOWN:
 235         case CARD_STATE_STARTING:
 236         case CARD_STATE_FORMATTING:
 237         case CARD_STATE_UNINITIALIZED:
 238         case CARD_STATE_SHUTTING_DOWN:
 239         /*
 240          * dStroy is a term coined by marketing to represent the low level
 241          * secure erase.
 242          */
 243         case CARD_STATE_DSTROYING:
 244                 set_capacity(card->gendisk, 0);
 245                 break;
 246         }
 247 }
 248
 249 static void card_event_handler(struct work_struct *work)
 250 {
 251         struct rsxx_cardinfo *card;
 252         unsigned int state;
 253         unsigned long flags;
 254         int st;
 255
 256         card = container_of(work, struct rsxx_cardinfo, event_work);
 257
 258         if (unlikely(card->halt))
 259                 return;
 260
 261         /*
 262          * Enable the interrupt now to avoid any weird race conditions where a
 263          * state change might occur while rsxx_get_card_state() is
 264          * processing a returned creg cmd.
 265          */
 266         spin_lock_irqsave(&card->irq_lock, flags);
 267         rsxx_enable_ier_and_isr(card, CR_INTR_EVENT);
 268         spin_unlock_irqrestore(&card->irq_lock, flags);
 269
 270         st = rsxx_get_card_state(card, &state);
 271         if (st) {
 272                 dev_info(CARD_TO_DEV(card),
 273                         "Failed reading state after event.\n");
 274                 return;
 275         }
 276
 277         if (card->state != state)
 278                 card_state_change(card, state);
 279
 280         if (card->creg_ctrl.creg_stats.stat & CREG_STAT_LOG_PENDING)
 281                 rsxx_read_hw_log(card);
 282 }
 283
 284 /*----------------- Card Operations -------------------*/
 285 static int card_shutdown(struct rsxx_cardinfo *card)
 286 {
 287         unsigned int state;
 288         signed long start;
 289         const int timeout = msecs_to_jiffies(120000);
 290         int st;
 291
 292         /* We can't issue a shutdown if the card is in a transition state */
 293         start = jiffies;
 294         do {
 295                 st = rsxx_get_card_state(card, &state);
 296                 if (st)
 297                         return st;
 298         } while (state == CARD_STATE_STARTING &&
 299                  (jiffies - start < timeout));
 300
 301         if (state == CARD_STATE_STARTING)
 302                 return -ETIMEDOUT;
 303
 304         /* Only issue a shutdown if we need to */
 305         if ((state != CARD_STATE_SHUTTING_DOWN) &&
 306             (state != CARD_STATE_SHUTDOWN)) {
 307                 st = rsxx_issue_card_cmd(card, CARD_CMD_SHUTDOWN);
 308                 if (st)
 309                         return st;
 310         }
 311
 312         start = jiffies;
 313         do {
 314                 st = rsxx_get_card_state(card, &state);
 315                 if (st)
 316                         return st;
 317         } while (state != CARD_STATE_SHUTDOWN &&
 318                  (jiffies - start < timeout));
 319
 320         if (state != CARD_STATE_SHUTDOWN)
 321                 return -ETIMEDOUT;
 322
 323         return 0;
 324 }
 325
 326 static int rsxx_eeh_frozen(struct pci_dev *dev)
 327 {
 328         struct rsxx_cardinfo *card = pci_get_drvdata(dev);
 329         int i;
 330         int st;
 331
 332         dev_warn(&dev->dev, "IBM FlashSystem PCI: preparing for slot reset.\n");
 333
 334         card->eeh_state = 1;
 335         rsxx_mask_interrupts(card);
 336
 337         /*
 338          * We need to guarantee that the write for eeh_state and masking
 339          * interrupts does not become reordered. This will prevent a possible
 340          * race condition with the EEH code.
 341          */
 342         wmb();
 343
 344         pci_disable_device(dev);
 345
 346         st = rsxx_eeh_save_issued_dmas(card);
 347         if (st)
 348                 return st;
 349
 350         rsxx_eeh_save_issued_creg(card);
 351
 352         for (i = 0; i < card->n_targets; i++) {
 353                 if (card->ctrl[i].status.buf)
 354                         pci_free_consistent(card->dev, STATUS_BUFFER_SIZE8,
 355                                             card->ctrl[i].status.buf,
 356                                             card->ctrl[i].status.dma_addr);
 357                 if (card->ctrl[i].cmd.buf)
 358                         pci_free_consistent(card->dev, COMMAND_BUFFER_SIZE8,
 359                                             card->ctrl[i].cmd.buf,
 360                                             card->ctrl[i].cmd.dma_addr);
 361         }
 362
 363         return 0;
 364 }
 365
 366 static void rsxx_eeh_failure(struct pci_dev *dev)
 367 {
 368         struct rsxx_cardinfo *card = pci_get_drvdata(dev);
 369         int i;
 370
 371         dev_err(&dev->dev, "IBM FlashSystem PCI: disabling failed card.\n");
 372
 373         card->eeh_state = 1;
 374
 375         for (i = 0; i < card->n_targets; i++)
 376                 del_timer_sync(&card->ctrl[i].activity_timer);
 377
 378         rsxx_eeh_cancel_dmas(card);
 379 }
 380
 381 static int rsxx_eeh_fifo_flush_poll(struct rsxx_cardinfo *card)
 382 {
 383         unsigned int status;
 384         int iter = 0;
 385
 386         /* We need to wait for the hardware to reset */
 387         while (iter++ < 10) {
 388                 status = ioread32(card->regmap + PCI_RECONFIG);
 389
 390                 if (status & RSXX_FLUSH_BUSY) {
 391                         ssleep(1);
 392                         continue;
 393                 }
 394
 395                 if (status & RSXX_FLUSH_TIMEOUT)
 396                         dev_warn(CARD_TO_DEV(card), "HW: flash controller timeout\n");
 397                 return 0;
 398         }
 399
 400         /* Hardware failed resetting itself. */
 401         return -1;
 402 }
 403
 404 static pci_ers_result_t rsxx_error_detected(struct pci_dev *dev,
 405                                             enum pci_channel_state error)
 406 {
 407         int st;
 408
 409         if (dev->revision < RSXX_EEH_SUPPORT)
 410                 return PCI_ERS_RESULT_NONE;
 411
 412         if (error == pci_channel_io_perm_failure) {
 413                 rsxx_eeh_failure(dev);
 414                 return PCI_ERS_RESULT_DISCONNECT;
 415         }
 416
 417         st = rsxx_eeh_frozen(dev);
 418         if (st) {
 419                 dev_err(&dev->dev, "Slot reset setup failed\n");
 420                 rsxx_eeh_failure(dev);
 421                 return PCI_ERS_RESULT_DISCONNECT;
 422         }
 423
 424         return PCI_ERS_RESULT_NEED_RESET;
 425 }
 426
 427 static pci_ers_result_t rsxx_slot_reset(struct pci_dev *dev)
 428 {
 429         struct rsxx_cardinfo *card = pci_get_drvdata(dev);
 430         unsigned long flags;
 431         int i;
 432         int st;
 433
 434         dev_warn(&dev->dev,
 435                 "IBM FlashSystem PCI: recovering from slot reset.\n");
 436
 437         st = pci_enable_device(dev);
 438         if (st)
 439                 goto failed_hw_setup;
 440
 441         pci_set_master(dev);
 442
 443         st = rsxx_eeh_fifo_flush_poll(card);
 444         if (st)
 445                 goto failed_hw_setup;
 446
 447         rsxx_dma_queue_reset(card);
 448
 449         for (i = 0; i < card->n_targets; i++) {
 450                 st = rsxx_hw_buffers_init(dev, &card->ctrl[i]);
 451                 if (st)
 452                         goto failed_hw_buffers_init;
 453         }
 454
 455         if (card->config_valid)
 456                 rsxx_dma_configure(card);
 457
 458         /* Clears the ISR register from spurious interrupts */
 459         st = ioread32(card->regmap + ISR);
 460
 461         card->eeh_state = 0;
 462
 463         st = rsxx_eeh_remap_dmas(card);
 464         if (st)
 465                 goto failed_remap_dmas;
 466
 467         spin_lock_irqsave(&card->irq_lock, flags);
 468         if (card->n_targets & RSXX_MAX_TARGETS)
 469                 rsxx_enable_ier_and_isr(card, CR_INTR_ALL_G);
 470         else
 471                 rsxx_enable_ier_and_isr(card, CR_INTR_ALL_C);
 472         spin_unlock_irqrestore(&card->irq_lock, flags);
 473
 474         rsxx_kick_creg_queue(card);
 475
 476         for (i = 0; i < card->n_targets; i++) {
 477                 spin_lock(&card->ctrl[i].queue_lock);
 478                 if (list_empty(&card->ctrl[i].queue)) {
 479                         spin_unlock(&card->ctrl[i].queue_lock);
 480                         continue;
 481                 }
 482                 spin_unlock(&card->ctrl[i].queue_lock);
 483
 484                 queue_work(card->ctrl[i].issue_wq,
 485                                 &card->ctrl[i].issue_dma_work);
 486         }
 487
 488         dev_info(&dev->dev, "IBM FlashSystem PCI: recovery complete.\n");
 489
 490         return PCI_ERS_RESULT_RECOVERED;
 491
 492 failed_hw_buffers_init:
 493 failed_remap_dmas:
 494         for (i = 0; i < card->n_targets; i++) {
 495                 if (card->ctrl[i].status.buf)
 496                         pci_free_consistent(card->dev,
 497                                         STATUS_BUFFER_SIZE8,
 498                                         card->ctrl[i].status.buf,
 499                                         card->ctrl[i].status.dma_addr);
 500                 if (card->ctrl[i].cmd.buf)
 501                         pci_free_consistent(card->dev,
 502                                         COMMAND_BUFFER_SIZE8,
 503                                         card->ctrl[i].cmd.buf,
 504                                         card->ctrl[i].cmd.dma_addr);
 505         }
 506 failed_hw_setup:
 507         rsxx_eeh_failure(dev);
 508         return PCI_ERS_RESULT_DISCONNECT;
 509
 510 }
 511
 512 /*----------------- Driver Initialization & Setup -------------------*/
 513 /* Returns:   0 if the driver is compatible with the device
 514              -1 if the driver is NOT compatible with the device */
 515 static int rsxx_compatibility_check(struct rsxx_cardinfo *card)
 516 {
 517         unsigned char pci_rev;
 518
 519         pci_read_config_byte(card->dev, PCI_REVISION_ID, &pci_rev);
 520
 521         if (pci_rev > RS70_PCI_REV_SUPPORTED)
 522                 return -1;
 523         return 0;
 524 }
 525
 526 static int rsxx_pci_probe(struct pci_dev *dev,
 527                                         const struct pci_device_id *id)
 528 {
 529         struct rsxx_cardinfo *card;
 530         int st;
 531
 532         dev_info(&dev->dev, "PCI-Flash SSD discovered\n");
 533
 534         card = kzalloc(sizeof(*card), GFP_KERNEL);
 535         if (!card)
 536                 return -ENOMEM;
 537
 538         card->dev = dev;
 539         pci_set_drvdata(dev, card);
 540
 541         do {
 542                 if (!ida_pre_get(&rsxx_disk_ida, GFP_KERNEL)) {
 543                         st = -ENOMEM;
 544                         goto failed_ida_get;
 545                 }
 546
 547                 spin_lock(&rsxx_ida_lock);
 548                 st = ida_get_new(&rsxx_disk_ida, &card->disk_id);
 549                 spin_unlock(&rsxx_ida_lock);
 550         } while (st == -EAGAIN);
 551
 552         if (st)
 553                 goto failed_ida_get;
 554
 555         st = pci_enable_device(dev);
 556         if (st)
 557                 goto failed_enable;
 558
 559         pci_set_master(dev);
 560         pci_set_dma_max_seg_size(dev, RSXX_HW_BLK_SIZE);
 561
 562         st = pci_set_dma_mask(dev, DMA_BIT_MASK(64));
 563         if (st) {
 564                 dev_err(CARD_TO_DEV(card),
 565                         "No usable DMA configuration,aborting\n");
 566                 goto failed_dma_mask;
 567         }
 568
 569         st = pci_request_regions(dev, DRIVER_NAME);
 570         if (st) {
 571                 dev_err(CARD_TO_DEV(card),
 572                         "Failed to request memory region\n");
 573                 goto failed_request_regions;
 574         }
 575
 576         if (pci_resource_len(dev, 0) == 0) {
 577                 dev_err(CARD_TO_DEV(card), "BAR0 has length 0!\n");
 578                 st = -ENOMEM;
 579                 goto failed_iomap;
 580         }
 581
 582         card->regmap = pci_iomap(dev, 0, 0);
 583         if (!card->regmap) {
 584                 dev_err(CARD_TO_DEV(card), "Failed to map BAR0\n");
 585                 st = -ENOMEM;
 586                 goto failed_iomap;
 587         }
 588
 589         spin_lock_init(&card->irq_lock);
 590         card->halt = 0;
 591         card->eeh_state = 0;
 592
 593         spin_lock_irq(&card->irq_lock);
 594         rsxx_disable_ier_and_isr(card, CR_INTR_ALL);
 595         spin_unlock_irq(&card->irq_lock);
 596
 597         if (!force_legacy) {
 598                 st = pci_enable_msi(dev);
 599                 if (st)
 600                         dev_warn(CARD_TO_DEV(card),
 601                                 "Failed to enable MSI\n");
 602         }
 603
 604         st = request_irq(dev->irq, rsxx_isr, IRQF_DISABLED | IRQF_SHARED,
 605                          DRIVER_NAME, card);
 606         if (st) {
 607                 dev_err(CARD_TO_DEV(card),
 608                         "Failed requesting IRQ%d\n", dev->irq);
 609                 goto failed_irq;
 610         }
 611
 612         /************* Setup Processor Command Interface *************/
 613         rsxx_creg_setup(card);
 614
 615         spin_lock_irq(&card->irq_lock);
 616         rsxx_enable_ier_and_isr(card, CR_INTR_CREG);
 617         spin_unlock_irq(&card->irq_lock);
 618
 619         st = rsxx_compatibility_check(card);
 620         if (st) {
 621                 dev_warn(CARD_TO_DEV(card),
 622                         "Incompatible driver detected. Please update the driver.\n");
 623                 st = -EINVAL;
 624                 goto failed_compatiblity_check;
 625         }
 626
 627         /************* Load Card Config *************/
 628         st = rsxx_load_config(card);
 629         if (st)
 630                 dev_err(CARD_TO_DEV(card),
 631                         "Failed loading card config\n");
 632
 633         /************* Setup DMA Engine *************/
 634         st = rsxx_get_num_targets(card, &card->n_targets);
 635         if (st)
 636                 dev_info(CARD_TO_DEV(card),
 637                         "Failed reading the number of DMA targets\n");
 638
 639         card->ctrl = kzalloc(card->n_targets * sizeof(*card->ctrl), GFP_KERNEL);
 640         if (!card->ctrl) {
 641                 st = -ENOMEM;
 642                 goto failed_dma_setup;
 643         }
 644
 645         st = rsxx_dma_setup(card);
 646         if (st) {
 647                 dev_info(CARD_TO_DEV(card),
 648                         "Failed to setup DMA engine\n");
 649                 goto failed_dma_setup;
 650         }
 651
 652         /************* Setup Card Event Handler *************/
 653         INIT_WORK(&card->event_work, card_event_handler);
 654
 655         st = rsxx_setup_dev(card);
 656         if (st)
 657                 goto failed_create_dev;
 658
 659         rsxx_get_card_state(card, &card->state);
 660
 661         dev_info(CARD_TO_DEV(card),
 662                 "card state: %s\n",
 663                 rsxx_card_state_to_str(card->state));
 664
 665         /*
 666          * Now that the DMA Engine and devices have been setup,
 667          * we can enable the event interrupt(it kicks off actions in
 668          * those layers so we couldn't enable it right away.)
 669          */
 670         spin_lock_irq(&card->irq_lock);
 671         rsxx_enable_ier_and_isr(card, CR_INTR_EVENT);
 672         spin_unlock_irq(&card->irq_lock);
 673
 674         if (card->state == CARD_STATE_SHUTDOWN) {
 675                 st = rsxx_issue_card_cmd(card, CARD_CMD_STARTUP);
 676                 if (st)
 677                         dev_crit(CARD_TO_DEV(card),
 678                                 "Failed issuing card startup\n");
 679         } else if (card->state == CARD_STATE_GOOD ||
 680                    card->state == CARD_STATE_RD_ONLY_FAULT) {
 681                 st = rsxx_get_card_size8(card, &card->size8);
 682                 if (st)
 683                         card->size8 = 0;
 684         }
 685
 686         rsxx_attach_dev(card);
 687
 688         return 0;
 689
 690 failed_create_dev:
 691         rsxx_dma_destroy(card);
 692 failed_dma_setup:
 693 failed_compatiblity_check:
 694         spin_lock_irq(&card->irq_lock);
 695         rsxx_disable_ier_and_isr(card, CR_INTR_ALL);
 696         spin_unlock_irq(&card->irq_lock);
 697         free_irq(dev->irq, card);
 698         if (!force_legacy)
 699                 pci_disable_msi(dev);
 700 failed_irq:
 701         pci_iounmap(dev, card->regmap);
 702 failed_iomap:
 703         pci_release_regions(dev);
 704 failed_request_regions:
 705 failed_dma_mask:
 706         pci_disable_device(dev);
 707 failed_enable:
 708         spin_lock(&rsxx_ida_lock);
 709         ida_remove(&rsxx_disk_ida, card->disk_id);
 710         spin_unlock(&rsxx_ida_lock);
 711 failed_ida_get:
 712         kfree(card);
 713
 714         return st;
 715 }
 716
 717 static void rsxx_pci_remove(struct pci_dev *dev)
 718 {
 719         struct rsxx_cardinfo *card = pci_get_drvdata(dev);
 720         unsigned long flags;
 721         int st;
 722         int i;
 723
 724         if (!card)
 725                 return;
 726
 727         dev_info(CARD_TO_DEV(card),
 728                 "Removing PCI-Flash SSD.\n");
 729
 730         rsxx_detach_dev(card);
 731
 732         for (i = 0; i < card->n_targets; i++) {
 733                 spin_lock_irqsave(&card->irq_lock, flags);
 734                 rsxx_disable_ier_and_isr(card, CR_INTR_DMA(i));
 735                 spin_unlock_irqrestore(&card->irq_lock, flags);
 736         }
 737
 738         st = card_shutdown(card);
 739         if (st)
 740                 dev_crit(CARD_TO_DEV(card), "Shutdown failed!\n");
 741
 742         /* Sync outstanding event handlers. */
 743         spin_lock_irqsave(&card->irq_lock, flags);
 744         rsxx_disable_ier_and_isr(card, CR_INTR_EVENT);
 745         spin_unlock_irqrestore(&card->irq_lock, flags);
 746
 747         cancel_work_sync(&card->event_work);
 748
 749         rsxx_destroy_dev(card);
 750         rsxx_dma_destroy(card);
 751
 752         spin_lock_irqsave(&card->irq_lock, flags);
 753         rsxx_disable_ier_and_isr(card, CR_INTR_ALL);
 754         spin_unlock_irqrestore(&card->irq_lock, flags);
 755
 756         /* Prevent work_structs from re-queuing themselves. */
 757         card->halt = 1;
 758
 759         free_irq(dev->irq, card);
 760
 761         if (!force_legacy)
 762                 pci_disable_msi(dev);
 763
 764         rsxx_creg_destroy(card);
 765
 766         pci_iounmap(dev, card->regmap);
 767
 768         pci_disable_device(dev);
 769         pci_release_regions(dev);
 770
 771         kfree(card);
 772 }
 773
 774 static int rsxx_pci_suspend(struct pci_dev *dev, pm_message_t state)
 775 {
 776         /* We don't support suspend at this time. */
 777         return -ENOSYS;
 778 }
 779
 780 static void rsxx_pci_shutdown(struct pci_dev *dev)
 781 {
 782         struct rsxx_cardinfo *card = pci_get_drvdata(dev);
 783         unsigned long flags;
 784         int i;
 785
 786         if (!card)
 787                 return;
 788
 789         dev_info(CARD_TO_DEV(card), "Shutting down PCI-Flash SSD.\n");
 790
 791         rsxx_detach_dev(card);
 792
 793         for (i = 0; i < card->n_targets; i++) {
 794                 spin_lock_irqsave(&card->irq_lock, flags);
 795                 rsxx_disable_ier_and_isr(card, CR_INTR_DMA(i));
 796                 spin_unlock_irqrestore(&card->irq_lock, flags);
 797         }
 798
 799         card_shutdown(card);
 800 }
 801
 802 static const struct pci_error_handlers rsxx_err_handler = {
 803         .error_detected = rsxx_error_detected,
 804         .slot_reset     = rsxx_slot_reset,
 805 };
 806
 807 static DEFINE_PCI_DEVICE_TABLE(rsxx_pci_ids) = {
 808         {PCI_DEVICE(PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_FS70_FLASH)},
 809         {PCI_DEVICE(PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_FS80_FLASH)},
 810         {0,},
 811 };
 812
 813 MODULE_DEVICE_TABLE(pci, rsxx_pci_ids);
 814
 815 static struct pci_driver rsxx_pci_driver = {
 816         .name           = DRIVER_NAME,
 817         .id_table       = rsxx_pci_ids,
 818         .probe          = rsxx_pci_probe,
 819         .remove         = rsxx_pci_remove,
 820         .suspend        = rsxx_pci_suspend,
 821         .shutdown       = rsxx_pci_shutdown,
 822         .err_handler    = &rsxx_err_handler,
 823 };
 824
 825 static int __init rsxx_core_init(void)
 826 {
 827         int st;
 828
 829         st = rsxx_dev_init();
 830         if (st)
 831                 return st;
 832
 833         st = rsxx_dma_init();
 834         if (st)
 835                 goto dma_init_failed;
 836
 837         st = rsxx_creg_init();
 838         if (st)
 839                 goto creg_init_failed;
 840
 841         return pci_register_driver(&rsxx_pci_driver);
 842
 843 creg_init_failed:
 844         rsxx_dma_cleanup();
 845 dma_init_failed:
 846         rsxx_dev_cleanup();
 847
 848         return st;
 849 }
 850
 851 static void __exit rsxx_core_cleanup(void)
 852 {
 853         pci_unregister_driver(&rsxx_pci_driver);
 854         rsxx_creg_cleanup();
 855         rsxx_dma_cleanup();
 856         rsxx_dev_cleanup();
 857 }
 858
 859 module_init(rsxx_core_init);
 860 module_exit(rsxx_core_cleanup);