arch/powerpc/platforms/powernv/pci.c

   1 /*
   2  * Support PCI/PCIe on PowerNV platforms
   3  *
   4  * Currently supports only P5IOC2
   5  *
   6  * Copyright 2011 Benjamin Herrenschmidt, IBM Corp.
   7  *
   8  * This program is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU General Public License
  10  * as published by the Free Software Foundation; either version
  11  * 2 of the License, or (at your option) any later version.
  12  */
  13
  14 #include <linux/kernel.h>
  15 #include <linux/pci.h>
  16 #include <linux/delay.h>
  17 #include <linux/string.h>
  18 #include <linux/init.h>
  19 #include <linux/bootmem.h>
  20 #include <linux/irq.h>
  21 #include <linux/io.h>
  22 #include <linux/msi.h>
  23 #include <linux/iommu.h>
  24
  25 #include <asm/sections.h>
  26 #include <asm/io.h>
  27 #include <asm/prom.h>
  28 #include <asm/pci-bridge.h>
  29 #include <asm/machdep.h>
  30 #include <asm/msi_bitmap.h>
  31 #include <asm/ppc-pci.h>
  32 #include <asm/opal.h>
  33 #include <asm/iommu.h>
  34 #include <asm/tce.h>
  35 #include <asm/firmware.h>
  36 #include <asm/eeh_event.h>
  37 #include <asm/eeh.h>
  38
  39 #include "powernv.h"
  40 #include "pci.h"
  41
  42 /* Delay in usec */
  43 #define PCI_RESET_DELAY_US      3000000
  44
  45 #define cfg_dbg(fmt...) do { } while(0)
  46 //#define cfg_dbg(fmt...)       printk(fmt)
  47
  48 #ifdef CONFIG_PCI_MSI
  49 static int pnv_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type)
  50 {
  51         struct pci_controller *hose = pci_bus_to_host(pdev->bus);
  52         struct pnv_phb *phb = hose->private_data;
  53         struct msi_desc *entry;
  54         struct msi_msg msg;
  55         int hwirq;
  56         unsigned int virq;
  57         int rc;
  58
  59         if (WARN_ON(!phb) || !phb->msi_bmp.bitmap)
  60                 return -ENODEV;
  61
  62         if (pdev->no_64bit_msi && !phb->msi32_support)
  63                 return -ENODEV;
  64
  65         list_for_each_entry(entry, &pdev->msi_list, list) {
  66                 if (!entry->msi_attrib.is_64 && !phb->msi32_support) {
  67                         pr_warn("%s: Supports only 64-bit MSIs\n",
  68                                 pci_name(pdev));
  69                         return -ENXIO;
  70                 }
  71                 hwirq = msi_bitmap_alloc_hwirqs(&phb->msi_bmp, 1);
  72                 if (hwirq < 0) {
  73                         pr_warn("%s: Failed to find a free MSI\n",
  74                                 pci_name(pdev));
  75                         return -ENOSPC;
  76                 }
  77                 virq = irq_create_mapping(NULL, phb->msi_base + hwirq);
  78                 if (virq == NO_IRQ) {
  79                         pr_warn("%s: Failed to map MSI to linux irq\n",
  80                                 pci_name(pdev));
  81                         msi_bitmap_free_hwirqs(&phb->msi_bmp, hwirq, 1);
  82                         return -ENOMEM;
  83                 }
  84                 rc = phb->msi_setup(phb, pdev, phb->msi_base + hwirq,
  85                                     virq, entry->msi_attrib.is_64, &msg);
  86                 if (rc) {
  87                         pr_warn("%s: Failed to setup MSI\n", pci_name(pdev));
  88                         irq_dispose_mapping(virq);
  89                         msi_bitmap_free_hwirqs(&phb->msi_bmp, hwirq, 1);
  90                         return rc;
  91                 }
  92                 irq_set_msi_desc(virq, entry);
  93                 write_msi_msg(virq, &msg);
  94         }
  95         return 0;
  96 }
  97
  98 static void pnv_teardown_msi_irqs(struct pci_dev *pdev)
  99 {
 100         struct pci_controller *hose = pci_bus_to_host(pdev->bus);
 101         struct pnv_phb *phb = hose->private_data;
 102         struct msi_desc *entry;
 103
 104         if (WARN_ON(!phb))
 105                 return;
 106
 107         list_for_each_entry(entry, &pdev->msi_list, list) {
 108                 if (entry->irq == NO_IRQ)
 109                         continue;
 110                 irq_set_msi_desc(entry->irq, NULL);
 111                 msi_bitmap_free_hwirqs(&phb->msi_bmp,
 112                         virq_to_hw(entry->irq) - phb->msi_base, 1);
 113                 irq_dispose_mapping(entry->irq);
 114         }
 115 }
 116 #endif /* CONFIG_PCI_MSI */
 117
 118 static void pnv_pci_dump_p7ioc_diag_data(struct pci_controller *hose,
 119                                          struct OpalIoPhbErrorCommon *common)
 120 {
 121         struct OpalIoP7IOCPhbErrorData *data;
 122         int i;
 123
 124         data = (struct OpalIoP7IOCPhbErrorData *)common;
 125         pr_info("P7IOC PHB#%d Diag-data (Version: %d)\n",
 126                 hose->global_number, be32_to_cpu(common->version));
 127
 128         if (data->brdgCtl)
 129                 pr_info("brdgCtl:     %08x\n",
 130                         be32_to_cpu(data->brdgCtl));
 131         if (data->portStatusReg || data->rootCmplxStatus ||
 132             data->busAgentStatus)
 133                 pr_info("UtlSts:      %08x %08x %08x\n",
 134                         be32_to_cpu(data->portStatusReg),
 135                         be32_to_cpu(data->rootCmplxStatus),
 136                         be32_to_cpu(data->busAgentStatus));
 137         if (data->deviceStatus || data->slotStatus   ||
 138             data->linkStatus   || data->devCmdStatus ||
 139             data->devSecStatus)
 140                 pr_info("RootSts:     %08x %08x %08x %08x %08x\n",
 141                         be32_to_cpu(data->deviceStatus),
 142                         be32_to_cpu(data->slotStatus),
 143                         be32_to_cpu(data->linkStatus),
 144                         be32_to_cpu(data->devCmdStatus),
 145                         be32_to_cpu(data->devSecStatus));
 146         if (data->rootErrorStatus   || data->uncorrErrorStatus ||
 147             data->corrErrorStatus)
 148                 pr_info("RootErrSts:  %08x %08x %08x\n",
 149                         be32_to_cpu(data->rootErrorStatus),
 150                         be32_to_cpu(data->uncorrErrorStatus),
 151                         be32_to_cpu(data->corrErrorStatus));
 152         if (data->tlpHdr1 || data->tlpHdr2 ||
 153             data->tlpHdr3 || data->tlpHdr4)
 154                 pr_info("RootErrLog:  %08x %08x %08x %08x\n",
 155                         be32_to_cpu(data->tlpHdr1),
 156                         be32_to_cpu(data->tlpHdr2),
 157                         be32_to_cpu(data->tlpHdr3),
 158                         be32_to_cpu(data->tlpHdr4));
 159         if (data->sourceId || data->errorClass ||
 160             data->correlator)
 161                 pr_info("RootErrLog1: %08x %016llx %016llx\n",
 162                         be32_to_cpu(data->sourceId),
 163                         be64_to_cpu(data->errorClass),
 164                         be64_to_cpu(data->correlator));
 165         if (data->p7iocPlssr || data->p7iocCsr)
 166                 pr_info("PhbSts:      %016llx %016llx\n",
 167                         be64_to_cpu(data->p7iocPlssr),
 168                         be64_to_cpu(data->p7iocCsr));
 169         if (data->lemFir)
 170                 pr_info("Lem:         %016llx %016llx %016llx\n",
 171                         be64_to_cpu(data->lemFir),
 172                         be64_to_cpu(data->lemErrorMask),
 173                         be64_to_cpu(data->lemWOF));
 174         if (data->phbErrorStatus)
 175                 pr_info("PhbErr:      %016llx %016llx %016llx %016llx\n",
 176                         be64_to_cpu(data->phbErrorStatus),
 177                         be64_to_cpu(data->phbFirstErrorStatus),
 178                         be64_to_cpu(data->phbErrorLog0),
 179                         be64_to_cpu(data->phbErrorLog1));
 180         if (data->mmioErrorStatus)
 181                 pr_info("OutErr:      %016llx %016llx %016llx %016llx\n",
 182                         be64_to_cpu(data->mmioErrorStatus),
 183                         be64_to_cpu(data->mmioFirstErrorStatus),
 184                         be64_to_cpu(data->mmioErrorLog0),
 185                         be64_to_cpu(data->mmioErrorLog1));
 186         if (data->dma0ErrorStatus)
 187                 pr_info("InAErr:      %016llx %016llx %016llx %016llx\n",
 188                         be64_to_cpu(data->dma0ErrorStatus),
 189                         be64_to_cpu(data->dma0FirstErrorStatus),
 190                         be64_to_cpu(data->dma0ErrorLog0),
 191                         be64_to_cpu(data->dma0ErrorLog1));
 192         if (data->dma1ErrorStatus)
 193                 pr_info("InBErr:      %016llx %016llx %016llx %016llx\n",
 194                         be64_to_cpu(data->dma1ErrorStatus),
 195                         be64_to_cpu(data->dma1FirstErrorStatus),
 196                         be64_to_cpu(data->dma1ErrorLog0),
 197                         be64_to_cpu(data->dma1ErrorLog1));
 198
 199         for (i = 0; i < OPAL_P7IOC_NUM_PEST_REGS; i++) {
 200                 if ((data->pestA[i] >> 63) == 0 &&
 201                     (data->pestB[i] >> 63) == 0)
 202                         continue;
 203
 204                 pr_info("PE[%3d] A/B: %016llx %016llx\n",
 205                         i, be64_to_cpu(data->pestA[i]),
 206                         be64_to_cpu(data->pestB[i]));
 207         }
 208 }
 209
 210 static void pnv_pci_dump_phb3_diag_data(struct pci_controller *hose,
 211                                         struct OpalIoPhbErrorCommon *common)
 212 {
 213         struct OpalIoPhb3ErrorData *data;
 214         int i;
 215
 216         data = (struct OpalIoPhb3ErrorData*)common;
 217         pr_info("PHB3 PHB#%d Diag-data (Version: %d)\n",
 218                 hose->global_number, be32_to_cpu(common->version));
 219         if (data->brdgCtl)
 220                 pr_info("brdgCtl:     %08x\n",
 221                         be32_to_cpu(data->brdgCtl));
 222         if (data->portStatusReg || data->rootCmplxStatus ||
 223             data->busAgentStatus)
 224                 pr_info("UtlSts:      %08x %08x %08x\n",
 225                         be32_to_cpu(data->portStatusReg),
 226                         be32_to_cpu(data->rootCmplxStatus),
 227                         be32_to_cpu(data->busAgentStatus));
 228         if (data->deviceStatus || data->slotStatus   ||
 229             data->linkStatus   || data->devCmdStatus ||
 230             data->devSecStatus)
 231                 pr_info("RootSts:     %08x %08x %08x %08x %08x\n",
 232                         be32_to_cpu(data->deviceStatus),
 233                         be32_to_cpu(data->slotStatus),
 234                         be32_to_cpu(data->linkStatus),
 235                         be32_to_cpu(data->devCmdStatus),
 236                         be32_to_cpu(data->devSecStatus));
 237         if (data->rootErrorStatus || data->uncorrErrorStatus ||
 238             data->corrErrorStatus)
 239                 pr_info("RootErrSts:  %08x %08x %08x\n",
 240                         be32_to_cpu(data->rootErrorStatus),
 241                         be32_to_cpu(data->uncorrErrorStatus),
 242                         be32_to_cpu(data->corrErrorStatus));
 243         if (data->tlpHdr1 || data->tlpHdr2 ||
 244             data->tlpHdr3 || data->tlpHdr4)
 245                 pr_info("RootErrLog:  %08x %08x %08x %08x\n",
 246                         be32_to_cpu(data->tlpHdr1),
 247                         be32_to_cpu(data->tlpHdr2),
 248                         be32_to_cpu(data->tlpHdr3),
 249                         be32_to_cpu(data->tlpHdr4));
 250         if (data->sourceId || data->errorClass ||
 251             data->correlator)
 252                 pr_info("RootErrLog1: %08x %016llx %016llx\n",
 253                         be32_to_cpu(data->sourceId),
 254                         be64_to_cpu(data->errorClass),
 255                         be64_to_cpu(data->correlator));
 256         if (data->nFir)
 257                 pr_info("nFir:        %016llx %016llx %016llx\n",
 258                         be64_to_cpu(data->nFir),
 259                         be64_to_cpu(data->nFirMask),
 260                         be64_to_cpu(data->nFirWOF));
 261         if (data->phbPlssr || data->phbCsr)
 262                 pr_info("PhbSts:      %016llx %016llx\n",
 263                         be64_to_cpu(data->phbPlssr),
 264                         be64_to_cpu(data->phbCsr));
 265         if (data->lemFir)
 266                 pr_info("Lem:         %016llx %016llx %016llx\n",
 267                         be64_to_cpu(data->lemFir),
 268                         be64_to_cpu(data->lemErrorMask),
 269                         be64_to_cpu(data->lemWOF));
 270         if (data->phbErrorStatus)
 271                 pr_info("PhbErr:      %016llx %016llx %016llx %016llx\n",
 272                         be64_to_cpu(data->phbErrorStatus),
 273                         be64_to_cpu(data->phbFirstErrorStatus),
 274                         be64_to_cpu(data->phbErrorLog0),
 275                         be64_to_cpu(data->phbErrorLog1));
 276         if (data->mmioErrorStatus)
 277                 pr_info("OutErr:      %016llx %016llx %016llx %016llx\n",
 278                         be64_to_cpu(data->mmioErrorStatus),
 279                         be64_to_cpu(data->mmioFirstErrorStatus),
 280                         be64_to_cpu(data->mmioErrorLog0),
 281                         be64_to_cpu(data->mmioErrorLog1));
 282         if (data->dma0ErrorStatus)
 283                 pr_info("InAErr:      %016llx %016llx %016llx %016llx\n",
 284                         be64_to_cpu(data->dma0ErrorStatus),
 285                         be64_to_cpu(data->dma0FirstErrorStatus),
 286                         be64_to_cpu(data->dma0ErrorLog0),
 287                         be64_to_cpu(data->dma0ErrorLog1));
 288         if (data->dma1ErrorStatus)
 289                 pr_info("InBErr:      %016llx %016llx %016llx %016llx\n",
 290                         be64_to_cpu(data->dma1ErrorStatus),
 291                         be64_to_cpu(data->dma1FirstErrorStatus),
 292                         be64_to_cpu(data->dma1ErrorLog0),
 293                         be64_to_cpu(data->dma1ErrorLog1));
 294
 295         for (i = 0; i < OPAL_PHB3_NUM_PEST_REGS; i++) {
 296                 if ((be64_to_cpu(data->pestA[i]) >> 63) == 0 &&
 297                     (be64_to_cpu(data->pestB[i]) >> 63) == 0)
 298                         continue;
 299
 300                 pr_info("PE[%3d] A/B: %016llx %016llx\n",
 301                                 i, be64_to_cpu(data->pestA[i]),
 302                                 be64_to_cpu(data->pestB[i]));
 303         }
 304 }
 305
 306 void pnv_pci_dump_phb_diag_data(struct pci_controller *hose,
 307                                 unsigned char *log_buff)
 308 {
 309         struct OpalIoPhbErrorCommon *common;
 310
 311         if (!hose || !log_buff)
 312                 return;
 313
 314         common = (struct OpalIoPhbErrorCommon *)log_buff;
 315         switch (be32_to_cpu(common->ioType)) {
 316         case OPAL_PHB_ERROR_DATA_TYPE_P7IOC:
 317                 pnv_pci_dump_p7ioc_diag_data(hose, common);
 318                 break;
 319         case OPAL_PHB_ERROR_DATA_TYPE_PHB3:
 320                 pnv_pci_dump_phb3_diag_data(hose, common);
 321                 break;
 322         default:
 323                 pr_warn("%s: Unrecognized ioType %d\n",
 324                         __func__, be32_to_cpu(common->ioType));
 325         }
 326 }
 327
 328 static void pnv_pci_handle_eeh_config(struct pnv_phb *phb, u32 pe_no)
 329 {
 330         unsigned long flags, rc;
 331         int has_diag, ret = 0;
 332
 333         spin_lock_irqsave(&phb->lock, flags);
 334
 335         /* Fetch PHB diag-data */
 336         rc = opal_pci_get_phb_diag_data2(phb->opal_id, phb->diag.blob,
 337                                          PNV_PCI_DIAG_BUF_SIZE);
 338         has_diag = (rc == OPAL_SUCCESS);
 339
 340         /* If PHB supports compound PE, to handle it */
 341         if (phb->unfreeze_pe) {
 342                 ret = phb->unfreeze_pe(phb,
 343                                        pe_no,
 344                                        OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
 345         } else {
 346                 rc = opal_pci_eeh_freeze_clear(phb->opal_id,
 347                                              pe_no,
 348                                              OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
 349                 if (rc) {
 350                         pr_warn("%s: Failure %ld clearing frozen "
 351                                 "PHB#%x-PE#%x\n",
 352                                 __func__, rc, phb->hose->global_number,
 353                                 pe_no);
 354                         ret = -EIO;
 355                 }
 356         }
 357
 358         /*
 359          * For now, let's only display the diag buffer when we fail to clear
 360          * the EEH status. We'll do more sensible things later when we have
 361          * proper EEH support. We need to make sure we don't pollute ourselves
 362          * with the normal errors generated when probing empty slots
 363          */
 364         if (has_diag && ret)
 365                 pnv_pci_dump_phb_diag_data(phb->hose, phb->diag.blob);
 366
 367         spin_unlock_irqrestore(&phb->lock, flags);
 368 }
 369
 370 static void pnv_pci_config_check_eeh(struct pnv_phb *phb,
 371                                      struct device_node *dn)
 372 {
 373         u8      fstate;
 374         __be16  pcierr;
 375         int     pe_no;
 376         s64     rc;
 377
 378         /*
 379          * Get the PE#. During the PCI probe stage, we might not
 380          * setup that yet. So all ER errors should be mapped to
 381          * reserved PE.
 382          */
 383         pe_no = PCI_DN(dn)->pe_number;
 384         if (pe_no == IODA_INVALID_PE) {
 385                 if (phb->type == PNV_PHB_P5IOC2)
 386                         pe_no = 0;
 387                 else
 388                         pe_no = phb->ioda.reserved_pe;
 389         }
 390
 391         /*
 392          * Fetch frozen state. If the PHB support compound PE,
 393          * we need handle that case.
 394          */
 395         if (phb->get_pe_state) {
 396                 fstate = phb->get_pe_state(phb, pe_no);
 397         } else {
 398                 rc = opal_pci_eeh_freeze_status(phb->opal_id,
 399                                                 pe_no,
 400                                                 &fstate,
 401                                                 &pcierr,
 402                                                 NULL);
 403                 if (rc) {
 404                         pr_warn("%s: Failure %lld getting PHB#%x-PE#%x state\n",
 405                                 __func__, rc, phb->hose->global_number, pe_no);
 406                         return;
 407                 }
 408         }
 409
 410         cfg_dbg(" -> EEH check, bdfn=%04x PE#%d fstate=%x\n",
 411                 (PCI_DN(dn)->busno << 8) | (PCI_DN(dn)->devfn),
 412                 pe_no, fstate);
 413
 414         /* Clear the frozen state if applicable */
 415         if (fstate == OPAL_EEH_STOPPED_MMIO_FREEZE ||
 416             fstate == OPAL_EEH_STOPPED_DMA_FREEZE  ||
 417             fstate == OPAL_EEH_STOPPED_MMIO_DMA_FREEZE) {
 418                 /*
 419                  * If PHB supports compound PE, freeze it for
 420                  * consistency.
 421                  */
 422                 if (phb->freeze_pe)
 423                         phb->freeze_pe(phb, pe_no);
 424
 425                 pnv_pci_handle_eeh_config(phb, pe_no);
 426         }
 427 }
 428
 429 int pnv_pci_cfg_read(struct device_node *dn,
 430                      int where, int size, u32 *val)
 431 {
 432         struct pci_dn *pdn = PCI_DN(dn);
 433         struct pnv_phb *phb = pdn->phb->private_data;
 434         u32 bdfn = (pdn->busno << 8) | pdn->devfn;
 435         s64 rc;
 436
 437         switch (size) {
 438         case 1: {
 439                 u8 v8;
 440                 rc = opal_pci_config_read_byte(phb->opal_id, bdfn, where, &v8);
 441                 *val = (rc == OPAL_SUCCESS) ? v8 : 0xff;
 442                 break;
 443         }
 444         case 2: {
 445                 __be16 v16;
 446                 rc = opal_pci_config_read_half_word(phb->opal_id, bdfn, where,
 447                                                    &v16);
 448                 *val = (rc == OPAL_SUCCESS) ? be16_to_cpu(v16) : 0xffff;
 449                 break;
 450         }
 451         case 4: {
 452                 __be32 v32;
 453                 rc = opal_pci_config_read_word(phb->opal_id, bdfn, where, &v32);
 454                 *val = (rc == OPAL_SUCCESS) ? be32_to_cpu(v32) : 0xffffffff;
 455                 break;
 456         }
 457         default:
 458                 return PCIBIOS_FUNC_NOT_SUPPORTED;
 459         }
 460
 461         cfg_dbg("%s: bus: %x devfn: %x +%x/%x -> %08x\n",
 462                 __func__, pdn->busno, pdn->devfn, where, size, *val);
 463         return PCIBIOS_SUCCESSFUL;
 464 }
 465
 466 int pnv_pci_cfg_write(struct device_node *dn,
 467                       int where, int size, u32 val)
 468 {
 469         struct pci_dn *pdn = PCI_DN(dn);
 470         struct pnv_phb *phb = pdn->phb->private_data;
 471         u32 bdfn = (pdn->busno << 8) | pdn->devfn;
 472
 473         cfg_dbg("%s: bus: %x devfn: %x +%x/%x -> %08x\n",
 474                 pdn->busno, pdn->devfn, where, size, val);
 475         switch (size) {
 476         case 1:
 477                 opal_pci_config_write_byte(phb->opal_id, bdfn, where, val);
 478                 break;
 479         case 2:
 480                 opal_pci_config_write_half_word(phb->opal_id, bdfn, where, val);
 481                 break;
 482         case 4:
 483                 opal_pci_config_write_word(phb->opal_id, bdfn, where, val);
 484                 break;
 485         default:
 486                 return PCIBIOS_FUNC_NOT_SUPPORTED;
 487         }
 488
 489         return PCIBIOS_SUCCESSFUL;
 490 }
 491
 492 #if CONFIG_EEH
 493 static bool pnv_pci_cfg_check(struct pci_controller *hose,
 494                               struct device_node *dn)
 495 {
 496         struct eeh_dev *edev = NULL;
 497         struct pnv_phb *phb = hose->private_data;
 498
 499         /* EEH not enabled ? */
 500         if (!(phb->flags & PNV_PHB_FLAG_EEH))
 501                 return true;
 502
 503         /* PE reset or device removed ? */
 504         edev = of_node_to_eeh_dev(dn);
 505         if (edev) {
 506                 if (edev->pe &&
 507                     (edev->pe->state & EEH_PE_CFG_BLOCKED))
 508                         return false;
 509
 510                 if (edev->mode & EEH_DEV_REMOVED)
 511                         return false;
 512         }
 513
 514         return true;
 515 }
 516 #else
 517 static inline pnv_pci_cfg_check(struct pci_controller *hose,
 518                                 struct device_node *dn)
 519 {
 520         return true;
 521 }
 522 #endif /* CONFIG_EEH */
 523
 524 static int pnv_pci_read_config(struct pci_bus *bus,
 525                                unsigned int devfn,
 526                                int where, int size, u32 *val)
 527 {
 528         struct device_node *dn, *busdn = pci_bus_to_OF_node(bus);
 529         struct pci_dn *pdn;
 530         struct pnv_phb *phb;
 531         bool found = false;
 532         int ret;
 533
 534         *val = 0xFFFFFFFF;
 535         for (dn = busdn->child; dn; dn = dn->sibling) {
 536                 pdn = PCI_DN(dn);
 537                 if (pdn && pdn->devfn == devfn) {
 538                         phb = pdn->phb->private_data;
 539                         found = true;
 540                         break;
 541                 }
 542         }
 543
 544         if (!found || !pnv_pci_cfg_check(pdn->phb, dn))
 545                 return PCIBIOS_DEVICE_NOT_FOUND;
 546
 547         ret = pnv_pci_cfg_read(dn, where, size, val);
 548         if (phb->flags & PNV_PHB_FLAG_EEH) {
 549                 if (*val == EEH_IO_ERROR_VALUE(size) &&
 550                     eeh_dev_check_failure(of_node_to_eeh_dev(dn)))
 551                         return PCIBIOS_DEVICE_NOT_FOUND;
 552         } else {
 553                 pnv_pci_config_check_eeh(phb, dn);
 554         }
 555
 556         return ret;
 557 }
 558
 559 static int pnv_pci_write_config(struct pci_bus *bus,
 560                                 unsigned int devfn,
 561                                 int where, int size, u32 val)
 562 {
 563         struct device_node *dn, *busdn = pci_bus_to_OF_node(bus);
 564         struct pci_dn *pdn;
 565         struct pnv_phb *phb;
 566         bool found = false;
 567         int ret;
 568
 569         for (dn = busdn->child; dn; dn = dn->sibling) {
 570                 pdn = PCI_DN(dn);
 571                 if (pdn && pdn->devfn == devfn) {
 572                         phb = pdn->phb->private_data;
 573                         found = true;
 574                         break;
 575                 }
 576         }
 577
 578         if (!found || !pnv_pci_cfg_check(pdn->phb, dn))
 579                 return PCIBIOS_DEVICE_NOT_FOUND;
 580
 581         ret = pnv_pci_cfg_write(dn, where, size, val);
 582         if (!(phb->flags & PNV_PHB_FLAG_EEH))
 583                 pnv_pci_config_check_eeh(phb, dn);
 584
 585         return ret;
 586 }
 587
 588 struct pci_ops pnv_pci_ops = {
 589         .read  = pnv_pci_read_config,
 590         .write = pnv_pci_write_config,
 591 };
 592
 593 static int pnv_tce_build(struct iommu_table *tbl, long index, long npages,
 594                          unsigned long uaddr, enum dma_data_direction direction,
 595                          struct dma_attrs *attrs, bool rm)
 596 {
 597         u64 proto_tce;
 598         __be64 *tcep, *tces;
 599         u64 rpn;
 600
 601         proto_tce = TCE_PCI_READ; // Read allowed
 602
 603         if (direction != DMA_TO_DEVICE)
 604                 proto_tce |= TCE_PCI_WRITE;
 605
 606         tces = tcep = ((__be64 *)tbl->it_base) + index - tbl->it_offset;
 607         rpn = __pa(uaddr) >> tbl->it_page_shift;
 608
 609         while (npages--)
 610                 *(tcep++) = cpu_to_be64(proto_tce |
 611                                 (rpn++ << tbl->it_page_shift));
 612
 613         /* Some implementations won't cache invalid TCEs and thus may not
 614          * need that flush. We'll probably turn it_type into a bit mask
 615          * of flags if that becomes the case
 616          */
 617         if (tbl->it_type & TCE_PCI_SWINV_CREATE)
 618                 pnv_pci_ioda_tce_invalidate(tbl, tces, tcep - 1, rm);
 619
 620         return 0;
 621 }
 622
 623 static int pnv_tce_build_vm(struct iommu_table *tbl, long index, long npages,
 624                             unsigned long uaddr,
 625                             enum dma_data_direction direction,
 626                             struct dma_attrs *attrs)
 627 {
 628         return pnv_tce_build(tbl, index, npages, uaddr, direction, attrs,
 629                         false);
 630 }
 631
 632 static void pnv_tce_free(struct iommu_table *tbl, long index, long npages,
 633                 bool rm)
 634 {
 635         __be64 *tcep, *tces;
 636
 637         tces = tcep = ((__be64 *)tbl->it_base) + index - tbl->it_offset;
 638
 639         while (npages--)
 640                 *(tcep++) = cpu_to_be64(0);
 641
 642         if (tbl->it_type & TCE_PCI_SWINV_FREE)
 643                 pnv_pci_ioda_tce_invalidate(tbl, tces, tcep - 1, rm);
 644 }
 645
 646 static void pnv_tce_free_vm(struct iommu_table *tbl, long index, long npages)
 647 {
 648         pnv_tce_free(tbl, index, npages, false);
 649 }
 650
 651 static unsigned long pnv_tce_get(struct iommu_table *tbl, long index)
 652 {
 653         return ((u64 *)tbl->it_base)[index - tbl->it_offset];
 654 }
 655
 656 static int pnv_tce_build_rm(struct iommu_table *tbl, long index, long npages,
 657                             unsigned long uaddr,
 658                             enum dma_data_direction direction,
 659                             struct dma_attrs *attrs)
 660 {
 661         return pnv_tce_build(tbl, index, npages, uaddr, direction, attrs, true);
 662 }
 663
 664 static void pnv_tce_free_rm(struct iommu_table *tbl, long index, long npages)
 665 {
 666         pnv_tce_free(tbl, index, npages, true);
 667 }
 668
 669 void pnv_pci_setup_iommu_table(struct iommu_table *tbl,
 670                                void *tce_mem, u64 tce_size,
 671                                u64 dma_offset, unsigned page_shift)
 672 {
 673         tbl->it_blocksize = 16;
 674         tbl->it_base = (unsigned long)tce_mem;
 675         tbl->it_page_shift = page_shift;
 676         tbl->it_offset = dma_offset >> tbl->it_page_shift;
 677         tbl->it_index = 0;
 678         tbl->it_size = tce_size >> 3;
 679         tbl->it_busno = 0;
 680         tbl->it_type = TCE_PCI;
 681 }
 682
 683 static struct iommu_table *pnv_pci_setup_bml_iommu(struct pci_controller *hose)
 684 {
 685         struct iommu_table *tbl;
 686         const __be64 *basep, *swinvp;
 687         const __be32 *sizep;
 688
 689         basep = of_get_property(hose->dn, "linux,tce-base", NULL);
 690         sizep = of_get_property(hose->dn, "linux,tce-size", NULL);
 691         if (basep == NULL || sizep == NULL) {
 692                 pr_err("PCI: %s has missing tce entries !\n",
 693                        hose->dn->full_name);
 694                 return NULL;
 695         }
 696         tbl = kzalloc_node(sizeof(struct iommu_table), GFP_KERNEL, hose->node);
 697         if (WARN_ON(!tbl))
 698                 return NULL;
 699         pnv_pci_setup_iommu_table(tbl, __va(be64_to_cpup(basep)),
 700                                   be32_to_cpup(sizep), 0, IOMMU_PAGE_SHIFT_4K);
 701         iommu_init_table(tbl, hose->node);
 702         iommu_register_group(tbl, pci_domain_nr(hose->bus), 0);
 703
 704         /* Deal with SW invalidated TCEs when needed (BML way) */
 705         swinvp = of_get_property(hose->dn, "linux,tce-sw-invalidate-info",
 706                                  NULL);
 707         if (swinvp) {
 708                 tbl->it_busno = be64_to_cpu(swinvp[1]);
 709                 tbl->it_index = (unsigned long)ioremap(be64_to_cpup(swinvp), 8);
 710                 tbl->it_type = TCE_PCI_SWINV_CREATE | TCE_PCI_SWINV_FREE;
 711         }
 712         return tbl;
 713 }
 714
 715 static void pnv_pci_dma_fallback_setup(struct pci_controller *hose,
 716                                        struct pci_dev *pdev)
 717 {
 718         struct device_node *np = pci_bus_to_OF_node(hose->bus);
 719         struct pci_dn *pdn;
 720
 721         if (np == NULL)
 722                 return;
 723         pdn = PCI_DN(np);
 724         if (!pdn->iommu_table)
 725                 pdn->iommu_table = pnv_pci_setup_bml_iommu(hose);
 726         if (!pdn->iommu_table)
 727                 return;
 728         set_iommu_table_base_and_group(&pdev->dev, pdn->iommu_table);
 729 }
 730
 731 static void pnv_pci_dma_dev_setup(struct pci_dev *pdev)
 732 {
 733         struct pci_controller *hose = pci_bus_to_host(pdev->bus);
 734         struct pnv_phb *phb = hose->private_data;
 735
 736         /* If we have no phb structure, try to setup a fallback based on
 737          * the device-tree (RTAS PCI for example)
 738          */
 739         if (phb && phb->dma_dev_setup)
 740                 phb->dma_dev_setup(phb, pdev);
 741         else
 742                 pnv_pci_dma_fallback_setup(hose, pdev);
 743 }
 744
 745 int pnv_pci_dma_set_mask(struct pci_dev *pdev, u64 dma_mask)
 746 {
 747         struct pci_controller *hose = pci_bus_to_host(pdev->bus);
 748         struct pnv_phb *phb = hose->private_data;
 749
 750         if (phb && phb->dma_set_mask)
 751                 return phb->dma_set_mask(phb, pdev, dma_mask);
 752         return __dma_set_mask(&pdev->dev, dma_mask);
 753 }
 754
 755 u64 pnv_pci_dma_get_required_mask(struct pci_dev *pdev)
 756 {
 757         struct pci_controller *hose = pci_bus_to_host(pdev->bus);
 758         struct pnv_phb *phb = hose->private_data;
 759
 760         if (phb && phb->dma_get_required_mask)
 761                 return phb->dma_get_required_mask(phb, pdev);
 762
 763         return __dma_get_required_mask(&pdev->dev);
 764 }
 765
 766 void pnv_pci_shutdown(void)
 767 {
 768         struct pci_controller *hose;
 769
 770         list_for_each_entry(hose, &hose_list, list_node) {
 771                 struct pnv_phb *phb = hose->private_data;
 772
 773                 if (phb && phb->shutdown)
 774                         phb->shutdown(phb);
 775         }
 776 }
 777
 778 /* Fixup wrong class code in p7ioc and p8 root complex */
 779 static void pnv_p7ioc_rc_quirk(struct pci_dev *dev)
 780 {
 781         dev->class = PCI_CLASS_BRIDGE_PCI << 8;
 782 }
 783 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_IBM, 0x3b9, pnv_p7ioc_rc_quirk);
 784
 785 static int pnv_pci_probe_mode(struct pci_bus *bus)
 786 {
 787         struct pci_controller *hose = pci_bus_to_host(bus);
 788         const __be64 *tstamp;
 789         u64 now, target;
 790
 791
 792         /* We hijack this as a way to ensure we have waited long
 793          * enough since the reset was lifted on the PCI bus
 794          */
 795         if (bus != hose->bus)
 796                 return PCI_PROBE_NORMAL;
 797         tstamp = of_get_property(hose->dn, "reset-clear-timestamp", NULL);
 798         if (!tstamp || !*tstamp)
 799                 return PCI_PROBE_NORMAL;
 800
 801         now = mftb() / tb_ticks_per_usec;
 802         target = (be64_to_cpup(tstamp) / tb_ticks_per_usec)
 803                 + PCI_RESET_DELAY_US;
 804
 805         pr_devel("pci %04d: Reset target: 0x%llx now: 0x%llx\n",
 806                  hose->global_number, target, now);
 807
 808         if (now < target)
 809                 msleep((target - now + 999) / 1000);
 810
 811         return PCI_PROBE_NORMAL;
 812 }
 813
 814 void __init pnv_pci_init(void)
 815 {
 816         struct device_node *np;
 817
 818         pci_add_flags(PCI_CAN_SKIP_ISA_ALIGN);
 819
 820         /* OPAL absent, try POPAL first then RTAS detection of PHBs */
 821         if (!firmware_has_feature(FW_FEATURE_OPAL)) {
 822 #ifdef CONFIG_PPC_POWERNV_RTAS
 823                 init_pci_config_tokens();
 824                 find_and_init_phbs();
 825 #endif /* CONFIG_PPC_POWERNV_RTAS */
 826         }
 827         /* OPAL is here, do our normal stuff */
 828         else {
 829                 int found_ioda = 0;
 830
 831                 /* Look for IODA IO-Hubs. We don't support mixing IODA
 832                  * and p5ioc2 due to the need to change some global
 833                  * probing flags
 834                  */
 835                 for_each_compatible_node(np, NULL, "ibm,ioda-hub") {
 836                         pnv_pci_init_ioda_hub(np);
 837                         found_ioda = 1;
 838                 }
 839
 840                 /* Look for p5ioc2 IO-Hubs */
 841                 if (!found_ioda)
 842                         for_each_compatible_node(np, NULL, "ibm,p5ioc2")
 843                                 pnv_pci_init_p5ioc2_hub(np);
 844
 845                 /* Look for ioda2 built-in PHB3's */
 846                 for_each_compatible_node(np, NULL, "ibm,ioda2-phb")
 847                         pnv_pci_init_ioda2_phb(np);
 848         }
 849
 850         /* Setup the linkage between OF nodes and PHBs */
 851         pci_devs_phb_init();
 852
 853         /* Configure IOMMU DMA hooks */
 854         ppc_md.pci_dma_dev_setup = pnv_pci_dma_dev_setup;
 855         ppc_md.tce_build = pnv_tce_build_vm;
 856         ppc_md.tce_free = pnv_tce_free_vm;
 857         ppc_md.tce_build_rm = pnv_tce_build_rm;
 858         ppc_md.tce_free_rm = pnv_tce_free_rm;
 859         ppc_md.tce_get = pnv_tce_get;
 860         ppc_md.pci_probe_mode = pnv_pci_probe_mode;
 861         set_pci_dma_ops(&dma_iommu_ops);
 862
 863         /* Configure MSIs */
 864 #ifdef CONFIG_PCI_MSI
 865         ppc_md.setup_msi_irqs = pnv_setup_msi_irqs;
 866         ppc_md.teardown_msi_irqs = pnv_teardown_msi_irqs;
 867 #endif
 868 }
 869
 870 static int tce_iommu_bus_notifier(struct notifier_block *nb,
 871                 unsigned long action, void *data)
 872 {
 873         struct device *dev = data;
 874
 875         switch (action) {
 876         case BUS_NOTIFY_ADD_DEVICE:
 877                 return iommu_add_device(dev);
 878         case BUS_NOTIFY_DEL_DEVICE:
 879                 if (dev->iommu_group)
 880                         iommu_del_device(dev);
 881                 return 0;
 882         default:
 883                 return 0;
 884         }
 885 }
 886
 887 static struct notifier_block tce_iommu_bus_nb = {
 888         .notifier_call = tce_iommu_bus_notifier,
 889 };
 890
 891 static int __init tce_iommu_bus_notifier_init(void)
 892 {
 893         bus_register_notifier(&pci_bus_type, &tce_iommu_bus_nb);
 894         return 0;
 895 }
 896 machine_subsys_initcall_sync(powernv, tce_iommu_bus_notifier_init);