arch/powerpc/platforms/powernv/pci.c

   1 /*
   2  * Support PCI/PCIe on PowerNV platforms
   3  *
   4  * Currently supports only P5IOC2
   5  *
   6  * Copyright 2011 Benjamin Herrenschmidt, IBM Corp.
   7  *
   8  * This program is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU General Public License
  10  * as published by the Free Software Foundation; either version
  11  * 2 of the License, or (at your option) any later version.
  12  */
  13
  14 #include <linux/kernel.h>
  15 #include <linux/pci.h>
  16 #include <linux/delay.h>
  17 #include <linux/string.h>
  18 #include <linux/init.h>
  19 #include <linux/irq.h>
  20 #include <linux/io.h>
  21 #include <linux/msi.h>
  22 #include <linux/iommu.h>
  23
  24 #include <asm/sections.h>
  25 #include <asm/io.h>
  26 #include <asm/prom.h>
  27 #include <asm/pci-bridge.h>
  28 #include <asm/machdep.h>
  29 #include <asm/msi_bitmap.h>
  30 #include <asm/ppc-pci.h>
  31 #include <asm/opal.h>
  32 #include <asm/iommu.h>
  33 #include <asm/tce.h>
  34 #include <asm/firmware.h>
  35 #include <asm/eeh_event.h>
  36 #include <asm/eeh.h>
  37
  38 #include "powernv.h"
  39 #include "pci.h"
  40
  41 /* Delay in usec */
  42 #define PCI_RESET_DELAY_US      3000000
  43
  44 #define cfg_dbg(fmt...) do { } while(0)
  45 //#define cfg_dbg(fmt...)       printk(fmt)
  46
  47 #ifdef CONFIG_PCI_MSI
  48 static int pnv_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type)
  49 {
  50         struct pci_controller *hose = pci_bus_to_host(pdev->bus);
  51         struct pnv_phb *phb = hose->private_data;
  52         struct msi_desc *entry;
  53         struct msi_msg msg;
  54         int hwirq;
  55         unsigned int virq;
  56         int rc;
  57
  58         if (WARN_ON(!phb) || !phb->msi_bmp.bitmap)
  59                 return -ENODEV;
  60
  61         if (pdev->no_64bit_msi && !phb->msi32_support)
  62                 return -ENODEV;
  63
  64         list_for_each_entry(entry, &pdev->msi_list, list) {
  65                 if (!entry->msi_attrib.is_64 && !phb->msi32_support) {
  66                         pr_warn("%s: Supports only 64-bit MSIs\n",
  67                                 pci_name(pdev));
  68                         return -ENXIO;
  69                 }
  70                 hwirq = msi_bitmap_alloc_hwirqs(&phb->msi_bmp, 1);
  71                 if (hwirq < 0) {
  72                         pr_warn("%s: Failed to find a free MSI\n",
  73                                 pci_name(pdev));
  74                         return -ENOSPC;
  75                 }
  76                 virq = irq_create_mapping(NULL, phb->msi_base + hwirq);
  77                 if (virq == NO_IRQ) {
  78                         pr_warn("%s: Failed to map MSI to linux irq\n",
  79                                 pci_name(pdev));
  80                         msi_bitmap_free_hwirqs(&phb->msi_bmp, hwirq, 1);
  81                         return -ENOMEM;
  82                 }
  83                 rc = phb->msi_setup(phb, pdev, phb->msi_base + hwirq,
  84                                     virq, entry->msi_attrib.is_64, &msg);
  85                 if (rc) {
  86                         pr_warn("%s: Failed to setup MSI\n", pci_name(pdev));
  87                         irq_dispose_mapping(virq);
  88                         msi_bitmap_free_hwirqs(&phb->msi_bmp, hwirq, 1);
  89                         return rc;
  90                 }
  91                 irq_set_msi_desc(virq, entry);
  92                 pci_write_msi_msg(virq, &msg);
  93         }
  94         return 0;
  95 }
  96
  97 static void pnv_teardown_msi_irqs(struct pci_dev *pdev)
  98 {
  99         struct pci_controller *hose = pci_bus_to_host(pdev->bus);
 100         struct pnv_phb *phb = hose->private_data;
 101         struct msi_desc *entry;
 102
 103         if (WARN_ON(!phb))
 104                 return;
 105
 106         list_for_each_entry(entry, &pdev->msi_list, list) {
 107                 if (entry->irq == NO_IRQ)
 108                         continue;
 109                 irq_set_msi_desc(entry->irq, NULL);
 110                 msi_bitmap_free_hwirqs(&phb->msi_bmp,
 111                         virq_to_hw(entry->irq) - phb->msi_base, 1);
 112                 irq_dispose_mapping(entry->irq);
 113         }
 114 }
 115 #endif /* CONFIG_PCI_MSI */
 116
 117 static void pnv_pci_dump_p7ioc_diag_data(struct pci_controller *hose,
 118                                          struct OpalIoPhbErrorCommon *common)
 119 {
 120         struct OpalIoP7IOCPhbErrorData *data;
 121         int i;
 122
 123         data = (struct OpalIoP7IOCPhbErrorData *)common;
 124         pr_info("P7IOC PHB#%d Diag-data (Version: %d)\n",
 125                 hose->global_number, be32_to_cpu(common->version));
 126
 127         if (data->brdgCtl)
 128                 pr_info("brdgCtl:     %08x\n",
 129                         be32_to_cpu(data->brdgCtl));
 130         if (data->portStatusReg || data->rootCmplxStatus ||
 131             data->busAgentStatus)
 132                 pr_info("UtlSts:      %08x %08x %08x\n",
 133                         be32_to_cpu(data->portStatusReg),
 134                         be32_to_cpu(data->rootCmplxStatus),
 135                         be32_to_cpu(data->busAgentStatus));
 136         if (data->deviceStatus || data->slotStatus   ||
 137             data->linkStatus   || data->devCmdStatus ||
 138             data->devSecStatus)
 139                 pr_info("RootSts:     %08x %08x %08x %08x %08x\n",
 140                         be32_to_cpu(data->deviceStatus),
 141                         be32_to_cpu(data->slotStatus),
 142                         be32_to_cpu(data->linkStatus),
 143                         be32_to_cpu(data->devCmdStatus),
 144                         be32_to_cpu(data->devSecStatus));
 145         if (data->rootErrorStatus   || data->uncorrErrorStatus ||
 146             data->corrErrorStatus)
 147                 pr_info("RootErrSts:  %08x %08x %08x\n",
 148                         be32_to_cpu(data->rootErrorStatus),
 149                         be32_to_cpu(data->uncorrErrorStatus),
 150                         be32_to_cpu(data->corrErrorStatus));
 151         if (data->tlpHdr1 || data->tlpHdr2 ||
 152             data->tlpHdr3 || data->tlpHdr4)
 153                 pr_info("RootErrLog:  %08x %08x %08x %08x\n",
 154                         be32_to_cpu(data->tlpHdr1),
 155                         be32_to_cpu(data->tlpHdr2),
 156                         be32_to_cpu(data->tlpHdr3),
 157                         be32_to_cpu(data->tlpHdr4));
 158         if (data->sourceId || data->errorClass ||
 159             data->correlator)
 160                 pr_info("RootErrLog1: %08x %016llx %016llx\n",
 161                         be32_to_cpu(data->sourceId),
 162                         be64_to_cpu(data->errorClass),
 163                         be64_to_cpu(data->correlator));
 164         if (data->p7iocPlssr || data->p7iocCsr)
 165                 pr_info("PhbSts:      %016llx %016llx\n",
 166                         be64_to_cpu(data->p7iocPlssr),
 167                         be64_to_cpu(data->p7iocCsr));
 168         if (data->lemFir)
 169                 pr_info("Lem:         %016llx %016llx %016llx\n",
 170                         be64_to_cpu(data->lemFir),
 171                         be64_to_cpu(data->lemErrorMask),
 172                         be64_to_cpu(data->lemWOF));
 173         if (data->phbErrorStatus)
 174                 pr_info("PhbErr:      %016llx %016llx %016llx %016llx\n",
 175                         be64_to_cpu(data->phbErrorStatus),
 176                         be64_to_cpu(data->phbFirstErrorStatus),
 177                         be64_to_cpu(data->phbErrorLog0),
 178                         be64_to_cpu(data->phbErrorLog1));
 179         if (data->mmioErrorStatus)
 180                 pr_info("OutErr:      %016llx %016llx %016llx %016llx\n",
 181                         be64_to_cpu(data->mmioErrorStatus),
 182                         be64_to_cpu(data->mmioFirstErrorStatus),
 183                         be64_to_cpu(data->mmioErrorLog0),
 184                         be64_to_cpu(data->mmioErrorLog1));
 185         if (data->dma0ErrorStatus)
 186                 pr_info("InAErr:      %016llx %016llx %016llx %016llx\n",
 187                         be64_to_cpu(data->dma0ErrorStatus),
 188                         be64_to_cpu(data->dma0FirstErrorStatus),
 189                         be64_to_cpu(data->dma0ErrorLog0),
 190                         be64_to_cpu(data->dma0ErrorLog1));
 191         if (data->dma1ErrorStatus)
 192                 pr_info("InBErr:      %016llx %016llx %016llx %016llx\n",
 193                         be64_to_cpu(data->dma1ErrorStatus),
 194                         be64_to_cpu(data->dma1FirstErrorStatus),
 195                         be64_to_cpu(data->dma1ErrorLog0),
 196                         be64_to_cpu(data->dma1ErrorLog1));
 197
 198         for (i = 0; i < OPAL_P7IOC_NUM_PEST_REGS; i++) {
 199                 if ((data->pestA[i] >> 63) == 0 &&
 200                     (data->pestB[i] >> 63) == 0)
 201                         continue;
 202
 203                 pr_info("PE[%3d] A/B: %016llx %016llx\n",
 204                         i, be64_to_cpu(data->pestA[i]),
 205                         be64_to_cpu(data->pestB[i]));
 206         }
 207 }
 208
 209 static void pnv_pci_dump_phb3_diag_data(struct pci_controller *hose,
 210                                         struct OpalIoPhbErrorCommon *common)
 211 {
 212         struct OpalIoPhb3ErrorData *data;
 213         int i;
 214
 215         data = (struct OpalIoPhb3ErrorData*)common;
 216         pr_info("PHB3 PHB#%d Diag-data (Version: %d)\n",
 217                 hose->global_number, be32_to_cpu(common->version));
 218         if (data->brdgCtl)
 219                 pr_info("brdgCtl:     %08x\n",
 220                         be32_to_cpu(data->brdgCtl));
 221         if (data->portStatusReg || data->rootCmplxStatus ||
 222             data->busAgentStatus)
 223                 pr_info("UtlSts:      %08x %08x %08x\n",
 224                         be32_to_cpu(data->portStatusReg),
 225                         be32_to_cpu(data->rootCmplxStatus),
 226                         be32_to_cpu(data->busAgentStatus));
 227         if (data->deviceStatus || data->slotStatus   ||
 228             data->linkStatus   || data->devCmdStatus ||
 229             data->devSecStatus)
 230                 pr_info("RootSts:     %08x %08x %08x %08x %08x\n",
 231                         be32_to_cpu(data->deviceStatus),
 232                         be32_to_cpu(data->slotStatus),
 233                         be32_to_cpu(data->linkStatus),
 234                         be32_to_cpu(data->devCmdStatus),
 235                         be32_to_cpu(data->devSecStatus));
 236         if (data->rootErrorStatus || data->uncorrErrorStatus ||
 237             data->corrErrorStatus)
 238                 pr_info("RootErrSts:  %08x %08x %08x\n",
 239                         be32_to_cpu(data->rootErrorStatus),
 240                         be32_to_cpu(data->uncorrErrorStatus),
 241                         be32_to_cpu(data->corrErrorStatus));
 242         if (data->tlpHdr1 || data->tlpHdr2 ||
 243             data->tlpHdr3 || data->tlpHdr4)
 244                 pr_info("RootErrLog:  %08x %08x %08x %08x\n",
 245                         be32_to_cpu(data->tlpHdr1),
 246                         be32_to_cpu(data->tlpHdr2),
 247                         be32_to_cpu(data->tlpHdr3),
 248                         be32_to_cpu(data->tlpHdr4));
 249         if (data->sourceId || data->errorClass ||
 250             data->correlator)
 251                 pr_info("RootErrLog1: %08x %016llx %016llx\n",
 252                         be32_to_cpu(data->sourceId),
 253                         be64_to_cpu(data->errorClass),
 254                         be64_to_cpu(data->correlator));
 255         if (data->nFir)
 256                 pr_info("nFir:        %016llx %016llx %016llx\n",
 257                         be64_to_cpu(data->nFir),
 258                         be64_to_cpu(data->nFirMask),
 259                         be64_to_cpu(data->nFirWOF));
 260         if (data->phbPlssr || data->phbCsr)
 261                 pr_info("PhbSts:      %016llx %016llx\n",
 262                         be64_to_cpu(data->phbPlssr),
 263                         be64_to_cpu(data->phbCsr));
 264         if (data->lemFir)
 265                 pr_info("Lem:         %016llx %016llx %016llx\n",
 266                         be64_to_cpu(data->lemFir),
 267                         be64_to_cpu(data->lemErrorMask),
 268                         be64_to_cpu(data->lemWOF));
 269         if (data->phbErrorStatus)
 270                 pr_info("PhbErr:      %016llx %016llx %016llx %016llx\n",
 271                         be64_to_cpu(data->phbErrorStatus),
 272                         be64_to_cpu(data->phbFirstErrorStatus),
 273                         be64_to_cpu(data->phbErrorLog0),
 274                         be64_to_cpu(data->phbErrorLog1));
 275         if (data->mmioErrorStatus)
 276                 pr_info("OutErr:      %016llx %016llx %016llx %016llx\n",
 277                         be64_to_cpu(data->mmioErrorStatus),
 278                         be64_to_cpu(data->mmioFirstErrorStatus),
 279                         be64_to_cpu(data->mmioErrorLog0),
 280                         be64_to_cpu(data->mmioErrorLog1));
 281         if (data->dma0ErrorStatus)
 282                 pr_info("InAErr:      %016llx %016llx %016llx %016llx\n",
 283                         be64_to_cpu(data->dma0ErrorStatus),
 284                         be64_to_cpu(data->dma0FirstErrorStatus),
 285                         be64_to_cpu(data->dma0ErrorLog0),
 286                         be64_to_cpu(data->dma0ErrorLog1));
 287         if (data->dma1ErrorStatus)
 288                 pr_info("InBErr:      %016llx %016llx %016llx %016llx\n",
 289                         be64_to_cpu(data->dma1ErrorStatus),
 290                         be64_to_cpu(data->dma1FirstErrorStatus),
 291                         be64_to_cpu(data->dma1ErrorLog0),
 292                         be64_to_cpu(data->dma1ErrorLog1));
 293
 294         for (i = 0; i < OPAL_PHB3_NUM_PEST_REGS; i++) {
 295                 if ((be64_to_cpu(data->pestA[i]) >> 63) == 0 &&
 296                     (be64_to_cpu(data->pestB[i]) >> 63) == 0)
 297                         continue;
 298
 299                 pr_info("PE[%3d] A/B: %016llx %016llx\n",
 300                                 i, be64_to_cpu(data->pestA[i]),
 301                                 be64_to_cpu(data->pestB[i]));
 302         }
 303 }
 304
 305 void pnv_pci_dump_phb_diag_data(struct pci_controller *hose,
 306                                 unsigned char *log_buff)
 307 {
 308         struct OpalIoPhbErrorCommon *common;
 309
 310         if (!hose || !log_buff)
 311                 return;
 312
 313         common = (struct OpalIoPhbErrorCommon *)log_buff;
 314         switch (be32_to_cpu(common->ioType)) {
 315         case OPAL_PHB_ERROR_DATA_TYPE_P7IOC:
 316                 pnv_pci_dump_p7ioc_diag_data(hose, common);
 317                 break;
 318         case OPAL_PHB_ERROR_DATA_TYPE_PHB3:
 319                 pnv_pci_dump_phb3_diag_data(hose, common);
 320                 break;
 321         default:
 322                 pr_warn("%s: Unrecognized ioType %d\n",
 323                         __func__, be32_to_cpu(common->ioType));
 324         }
 325 }
 326
 327 static void pnv_pci_handle_eeh_config(struct pnv_phb *phb, u32 pe_no)
 328 {
 329         unsigned long flags, rc;
 330         int has_diag, ret = 0;
 331
 332         spin_lock_irqsave(&phb->lock, flags);
 333
 334         /* Fetch PHB diag-data */
 335         rc = opal_pci_get_phb_diag_data2(phb->opal_id, phb->diag.blob,
 336                                          PNV_PCI_DIAG_BUF_SIZE);
 337         has_diag = (rc == OPAL_SUCCESS);
 338
 339         /* If PHB supports compound PE, to handle it */
 340         if (phb->unfreeze_pe) {
 341                 ret = phb->unfreeze_pe(phb,
 342                                        pe_no,
 343                                        OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
 344         } else {
 345                 rc = opal_pci_eeh_freeze_clear(phb->opal_id,
 346                                              pe_no,
 347                                              OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
 348                 if (rc) {
 349                         pr_warn("%s: Failure %ld clearing frozen "
 350                                 "PHB#%x-PE#%x\n",
 351                                 __func__, rc, phb->hose->global_number,
 352                                 pe_no);
 353                         ret = -EIO;
 354                 }
 355         }
 356
 357         /*
 358          * For now, let's only display the diag buffer when we fail to clear
 359          * the EEH status. We'll do more sensible things later when we have
 360          * proper EEH support. We need to make sure we don't pollute ourselves
 361          * with the normal errors generated when probing empty slots
 362          */
 363         if (has_diag && ret)
 364                 pnv_pci_dump_phb_diag_data(phb->hose, phb->diag.blob);
 365
 366         spin_unlock_irqrestore(&phb->lock, flags);
 367 }
 368
 369 static void pnv_pci_config_check_eeh(struct pnv_phb *phb,
 370                                      struct device_node *dn)
 371 {
 372         u8      fstate;
 373         __be16  pcierr;
 374         int     pe_no;
 375         s64     rc;
 376
 377         /*
 378          * Get the PE#. During the PCI probe stage, we might not
 379          * setup that yet. So all ER errors should be mapped to
 380          * reserved PE.
 381          */
 382         pe_no = PCI_DN(dn)->pe_number;
 383         if (pe_no == IODA_INVALID_PE) {
 384                 if (phb->type == PNV_PHB_P5IOC2)
 385                         pe_no = 0;
 386                 else
 387                         pe_no = phb->ioda.reserved_pe;
 388         }
 389
 390         /*
 391          * Fetch frozen state. If the PHB support compound PE,
 392          * we need handle that case.
 393          */
 394         if (phb->get_pe_state) {
 395                 fstate = phb->get_pe_state(phb, pe_no);
 396         } else {
 397                 rc = opal_pci_eeh_freeze_status(phb->opal_id,
 398                                                 pe_no,
 399                                                 &fstate,
 400                                                 &pcierr,
 401                                                 NULL);
 402                 if (rc) {
 403                         pr_warn("%s: Failure %lld getting PHB#%x-PE#%x state\n",
 404                                 __func__, rc, phb->hose->global_number, pe_no);
 405                         return;
 406                 }
 407         }
 408
 409         cfg_dbg(" -> EEH check, bdfn=%04x PE#%d fstate=%x\n",
 410                 (PCI_DN(dn)->busno << 8) | (PCI_DN(dn)->devfn),
 411                 pe_no, fstate);
 412
 413         /* Clear the frozen state if applicable */
 414         if (fstate == OPAL_EEH_STOPPED_MMIO_FREEZE ||
 415             fstate == OPAL_EEH_STOPPED_DMA_FREEZE  ||
 416             fstate == OPAL_EEH_STOPPED_MMIO_DMA_FREEZE) {
 417                 /*
 418                  * If PHB supports compound PE, freeze it for
 419                  * consistency.
 420                  */
 421                 if (phb->freeze_pe)
 422                         phb->freeze_pe(phb, pe_no);
 423
 424                 pnv_pci_handle_eeh_config(phb, pe_no);
 425         }
 426 }
 427
 428 int pnv_pci_cfg_read(struct device_node *dn,
 429                      int where, int size, u32 *val)
 430 {
 431         struct pci_dn *pdn = PCI_DN(dn);
 432         struct pnv_phb *phb = pdn->phb->private_data;
 433         u32 bdfn = (pdn->busno << 8) | pdn->devfn;
 434         s64 rc;
 435
 436         switch (size) {
 437         case 1: {
 438                 u8 v8;
 439                 rc = opal_pci_config_read_byte(phb->opal_id, bdfn, where, &v8);
 440                 *val = (rc == OPAL_SUCCESS) ? v8 : 0xff;
 441                 break;
 442         }
 443         case 2: {
 444                 __be16 v16;
 445                 rc = opal_pci_config_read_half_word(phb->opal_id, bdfn, where,
 446                                                    &v16);
 447                 *val = (rc == OPAL_SUCCESS) ? be16_to_cpu(v16) : 0xffff;
 448                 break;
 449         }
 450         case 4: {
 451                 __be32 v32;
 452                 rc = opal_pci_config_read_word(phb->opal_id, bdfn, where, &v32);
 453                 *val = (rc == OPAL_SUCCESS) ? be32_to_cpu(v32) : 0xffffffff;
 454                 break;
 455         }
 456         default:
 457                 return PCIBIOS_FUNC_NOT_SUPPORTED;
 458         }
 459
 460         cfg_dbg("%s: bus: %x devfn: %x +%x/%x -> %08x\n",
 461                 __func__, pdn->busno, pdn->devfn, where, size, *val);
 462         return PCIBIOS_SUCCESSFUL;
 463 }
 464
 465 int pnv_pci_cfg_write(struct device_node *dn,
 466                       int where, int size, u32 val)
 467 {
 468         struct pci_dn *pdn = PCI_DN(dn);
 469         struct pnv_phb *phb = pdn->phb->private_data;
 470         u32 bdfn = (pdn->busno << 8) | pdn->devfn;
 471
 472         cfg_dbg("%s: bus: %x devfn: %x +%x/%x -> %08x\n",
 473                 pdn->busno, pdn->devfn, where, size, val);
 474         switch (size) {
 475         case 1:
 476                 opal_pci_config_write_byte(phb->opal_id, bdfn, where, val);
 477                 break;
 478         case 2:
 479                 opal_pci_config_write_half_word(phb->opal_id, bdfn, where, val);
 480                 break;
 481         case 4:
 482                 opal_pci_config_write_word(phb->opal_id, bdfn, where, val);
 483                 break;
 484         default:
 485                 return PCIBIOS_FUNC_NOT_SUPPORTED;
 486         }
 487
 488         return PCIBIOS_SUCCESSFUL;
 489 }
 490
 491 #if CONFIG_EEH
 492 static bool pnv_pci_cfg_check(struct pci_controller *hose,
 493                               struct device_node *dn)
 494 {
 495         struct eeh_dev *edev = NULL;
 496         struct pnv_phb *phb = hose->private_data;
 497
 498         /* EEH not enabled ? */
 499         if (!(phb->flags & PNV_PHB_FLAG_EEH))
 500                 return true;
 501
 502         /* PE reset or device removed ? */
 503         edev = of_node_to_eeh_dev(dn);
 504         if (edev) {
 505                 if (edev->pe &&
 506                     (edev->pe->state & EEH_PE_CFG_BLOCKED))
 507                         return false;
 508
 509                 if (edev->mode & EEH_DEV_REMOVED)
 510                         return false;
 511         }
 512
 513         return true;
 514 }
 515 #else
 516 static inline pnv_pci_cfg_check(struct pci_controller *hose,
 517                                 struct device_node *dn)
 518 {
 519         return true;
 520 }
 521 #endif /* CONFIG_EEH */
 522
 523 static int pnv_pci_read_config(struct pci_bus *bus,
 524                                unsigned int devfn,
 525                                int where, int size, u32 *val)
 526 {
 527         struct device_node *dn, *busdn = pci_bus_to_OF_node(bus);
 528         struct pci_dn *pdn;
 529         struct pnv_phb *phb;
 530         bool found = false;
 531         int ret;
 532
 533         *val = 0xFFFFFFFF;
 534         for (dn = busdn->child; dn; dn = dn->sibling) {
 535                 pdn = PCI_DN(dn);
 536                 if (pdn && pdn->devfn == devfn) {
 537                         phb = pdn->phb->private_data;
 538                         found = true;
 539                         break;
 540                 }
 541         }
 542
 543         if (!found || !pnv_pci_cfg_check(pdn->phb, dn))
 544                 return PCIBIOS_DEVICE_NOT_FOUND;
 545
 546         ret = pnv_pci_cfg_read(dn, where, size, val);
 547         if (phb->flags & PNV_PHB_FLAG_EEH) {
 548                 if (*val == EEH_IO_ERROR_VALUE(size) &&
 549                     eeh_dev_check_failure(of_node_to_eeh_dev(dn)))
 550                         return PCIBIOS_DEVICE_NOT_FOUND;
 551         } else {
 552                 pnv_pci_config_check_eeh(phb, dn);
 553         }
 554
 555         return ret;
 556 }
 557
 558 static int pnv_pci_write_config(struct pci_bus *bus,
 559                                 unsigned int devfn,
 560                                 int where, int size, u32 val)
 561 {
 562         struct device_node *dn, *busdn = pci_bus_to_OF_node(bus);
 563         struct pci_dn *pdn;
 564         struct pnv_phb *phb;
 565         bool found = false;
 566         int ret;
 567
 568         for (dn = busdn->child; dn; dn = dn->sibling) {
 569                 pdn = PCI_DN(dn);
 570                 if (pdn && pdn->devfn == devfn) {
 571                         phb = pdn->phb->private_data;
 572                         found = true;
 573                         break;
 574                 }
 575         }
 576
 577         if (!found || !pnv_pci_cfg_check(pdn->phb, dn))
 578                 return PCIBIOS_DEVICE_NOT_FOUND;
 579
 580         ret = pnv_pci_cfg_write(dn, where, size, val);
 581         if (!(phb->flags & PNV_PHB_FLAG_EEH))
 582                 pnv_pci_config_check_eeh(phb, dn);
 583
 584         return ret;
 585 }
 586
 587 struct pci_ops pnv_pci_ops = {
 588         .read  = pnv_pci_read_config,
 589         .write = pnv_pci_write_config,
 590 };
 591
 592 static int pnv_tce_build(struct iommu_table *tbl, long index, long npages,
 593                          unsigned long uaddr, enum dma_data_direction direction,
 594                          struct dma_attrs *attrs, bool rm)
 595 {
 596         u64 proto_tce;
 597         __be64 *tcep, *tces;
 598         u64 rpn;
 599
 600         proto_tce = TCE_PCI_READ; // Read allowed
 601
 602         if (direction != DMA_TO_DEVICE)
 603                 proto_tce |= TCE_PCI_WRITE;
 604
 605         tces = tcep = ((__be64 *)tbl->it_base) + index - tbl->it_offset;
 606         rpn = __pa(uaddr) >> tbl->it_page_shift;
 607
 608         while (npages--)
 609                 *(tcep++) = cpu_to_be64(proto_tce |
 610                                 (rpn++ << tbl->it_page_shift));
 611
 612         /* Some implementations won't cache invalid TCEs and thus may not
 613          * need that flush. We'll probably turn it_type into a bit mask
 614          * of flags if that becomes the case
 615          */
 616         if (tbl->it_type & TCE_PCI_SWINV_CREATE)
 617                 pnv_pci_ioda_tce_invalidate(tbl, tces, tcep - 1, rm);
 618
 619         return 0;
 620 }
 621
 622 static int pnv_tce_build_vm(struct iommu_table *tbl, long index, long npages,
 623                             unsigned long uaddr,
 624                             enum dma_data_direction direction,
 625                             struct dma_attrs *attrs)
 626 {
 627         return pnv_tce_build(tbl, index, npages, uaddr, direction, attrs,
 628                         false);
 629 }
 630
 631 static void pnv_tce_free(struct iommu_table *tbl, long index, long npages,
 632                 bool rm)
 633 {
 634         __be64 *tcep, *tces;
 635
 636         tces = tcep = ((__be64 *)tbl->it_base) + index - tbl->it_offset;
 637
 638         while (npages--)
 639                 *(tcep++) = cpu_to_be64(0);
 640
 641         if (tbl->it_type & TCE_PCI_SWINV_FREE)
 642                 pnv_pci_ioda_tce_invalidate(tbl, tces, tcep - 1, rm);
 643 }
 644
 645 static void pnv_tce_free_vm(struct iommu_table *tbl, long index, long npages)
 646 {
 647         pnv_tce_free(tbl, index, npages, false);
 648 }
 649
 650 static unsigned long pnv_tce_get(struct iommu_table *tbl, long index)
 651 {
 652         return ((u64 *)tbl->it_base)[index - tbl->it_offset];
 653 }
 654
 655 static int pnv_tce_build_rm(struct iommu_table *tbl, long index, long npages,
 656                             unsigned long uaddr,
 657                             enum dma_data_direction direction,
 658                             struct dma_attrs *attrs)
 659 {
 660         return pnv_tce_build(tbl, index, npages, uaddr, direction, attrs, true);
 661 }
 662
 663 static void pnv_tce_free_rm(struct iommu_table *tbl, long index, long npages)
 664 {
 665         pnv_tce_free(tbl, index, npages, true);
 666 }
 667
 668 void pnv_pci_setup_iommu_table(struct iommu_table *tbl,
 669                                void *tce_mem, u64 tce_size,
 670                                u64 dma_offset, unsigned page_shift)
 671 {
 672         tbl->it_blocksize = 16;
 673         tbl->it_base = (unsigned long)tce_mem;
 674         tbl->it_page_shift = page_shift;
 675         tbl->it_offset = dma_offset >> tbl->it_page_shift;
 676         tbl->it_index = 0;
 677         tbl->it_size = tce_size >> 3;
 678         tbl->it_busno = 0;
 679         tbl->it_type = TCE_PCI;
 680 }
 681
 682 static struct iommu_table *pnv_pci_setup_bml_iommu(struct pci_controller *hose)
 683 {
 684         struct iommu_table *tbl;
 685         const __be64 *basep, *swinvp;
 686         const __be32 *sizep;
 687
 688         basep = of_get_property(hose->dn, "linux,tce-base", NULL);
 689         sizep = of_get_property(hose->dn, "linux,tce-size", NULL);
 690         if (basep == NULL || sizep == NULL) {
 691                 pr_err("PCI: %s has missing tce entries !\n",
 692                        hose->dn->full_name);
 693                 return NULL;
 694         }
 695         tbl = kzalloc_node(sizeof(struct iommu_table), GFP_KERNEL, hose->node);
 696         if (WARN_ON(!tbl))
 697                 return NULL;
 698         pnv_pci_setup_iommu_table(tbl, __va(be64_to_cpup(basep)),
 699                                   be32_to_cpup(sizep), 0, IOMMU_PAGE_SHIFT_4K);
 700         iommu_init_table(tbl, hose->node);
 701         iommu_register_group(tbl, pci_domain_nr(hose->bus), 0);
 702
 703         /* Deal with SW invalidated TCEs when needed (BML way) */
 704         swinvp = of_get_property(hose->dn, "linux,tce-sw-invalidate-info",
 705                                  NULL);
 706         if (swinvp) {
 707                 tbl->it_busno = be64_to_cpu(swinvp[1]);
 708                 tbl->it_index = (unsigned long)ioremap(be64_to_cpup(swinvp), 8);
 709                 tbl->it_type = TCE_PCI_SWINV_CREATE | TCE_PCI_SWINV_FREE;
 710         }
 711         return tbl;
 712 }
 713
 714 static void pnv_pci_dma_fallback_setup(struct pci_controller *hose,
 715                                        struct pci_dev *pdev)
 716 {
 717         struct device_node *np = pci_bus_to_OF_node(hose->bus);
 718         struct pci_dn *pdn;
 719
 720         if (np == NULL)
 721                 return;
 722         pdn = PCI_DN(np);
 723         if (!pdn->iommu_table)
 724                 pdn->iommu_table = pnv_pci_setup_bml_iommu(hose);
 725         if (!pdn->iommu_table)
 726                 return;
 727         set_iommu_table_base_and_group(&pdev->dev, pdn->iommu_table);
 728 }
 729
 730 static void pnv_pci_dma_dev_setup(struct pci_dev *pdev)
 731 {
 732         struct pci_controller *hose = pci_bus_to_host(pdev->bus);
 733         struct pnv_phb *phb = hose->private_data;
 734
 735         /* If we have no phb structure, try to setup a fallback based on
 736          * the device-tree (RTAS PCI for example)
 737          */
 738         if (phb && phb->dma_dev_setup)
 739                 phb->dma_dev_setup(phb, pdev);
 740         else
 741                 pnv_pci_dma_fallback_setup(hose, pdev);
 742 }
 743
 744 int pnv_pci_dma_set_mask(struct pci_dev *pdev, u64 dma_mask)
 745 {
 746         struct pci_controller *hose = pci_bus_to_host(pdev->bus);
 747         struct pnv_phb *phb = hose->private_data;
 748
 749         if (phb && phb->dma_set_mask)
 750                 return phb->dma_set_mask(phb, pdev, dma_mask);
 751         return __dma_set_mask(&pdev->dev, dma_mask);
 752 }
 753
 754 u64 pnv_pci_dma_get_required_mask(struct pci_dev *pdev)
 755 {
 756         struct pci_controller *hose = pci_bus_to_host(pdev->bus);
 757         struct pnv_phb *phb = hose->private_data;
 758
 759         if (phb && phb->dma_get_required_mask)
 760                 return phb->dma_get_required_mask(phb, pdev);
 761
 762         return __dma_get_required_mask(&pdev->dev);
 763 }
 764
 765 void pnv_pci_shutdown(void)
 766 {
 767         struct pci_controller *hose;
 768
 769         list_for_each_entry(hose, &hose_list, list_node) {
 770                 struct pnv_phb *phb = hose->private_data;
 771
 772                 if (phb && phb->shutdown)
 773                         phb->shutdown(phb);
 774         }
 775 }
 776
 777 /* Fixup wrong class code in p7ioc and p8 root complex */
 778 static void pnv_p7ioc_rc_quirk(struct pci_dev *dev)
 779 {
 780         dev->class = PCI_CLASS_BRIDGE_PCI << 8;
 781 }
 782 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_IBM, 0x3b9, pnv_p7ioc_rc_quirk);
 783
 784 static int pnv_pci_probe_mode(struct pci_bus *bus)
 785 {
 786         struct pci_controller *hose = pci_bus_to_host(bus);
 787         const __be64 *tstamp;
 788         u64 now, target;
 789
 790
 791         /* We hijack this as a way to ensure we have waited long
 792          * enough since the reset was lifted on the PCI bus
 793          */
 794         if (bus != hose->bus)
 795                 return PCI_PROBE_NORMAL;
 796         tstamp = of_get_property(hose->dn, "reset-clear-timestamp", NULL);
 797         if (!tstamp || !*tstamp)
 798                 return PCI_PROBE_NORMAL;
 799
 800         now = mftb() / tb_ticks_per_usec;
 801         target = (be64_to_cpup(tstamp) / tb_ticks_per_usec)
 802                 + PCI_RESET_DELAY_US;
 803
 804         pr_devel("pci %04d: Reset target: 0x%llx now: 0x%llx\n",
 805                  hose->global_number, target, now);
 806
 807         if (now < target)
 808                 msleep((target - now + 999) / 1000);
 809
 810         return PCI_PROBE_NORMAL;
 811 }
 812
 813 void __init pnv_pci_init(void)
 814 {
 815         struct device_node *np;
 816
 817         pci_add_flags(PCI_CAN_SKIP_ISA_ALIGN);
 818
 819         /* OPAL absent, try POPAL first then RTAS detection of PHBs */
 820         if (!firmware_has_feature(FW_FEATURE_OPAL)) {
 821 #ifdef CONFIG_PPC_POWERNV_RTAS
 822                 init_pci_config_tokens();
 823                 find_and_init_phbs();
 824 #endif /* CONFIG_PPC_POWERNV_RTAS */
 825         }
 826         /* OPAL is here, do our normal stuff */
 827         else {
 828                 int found_ioda = 0;
 829
 830                 /* Look for IODA IO-Hubs. We don't support mixing IODA
 831                  * and p5ioc2 due to the need to change some global
 832                  * probing flags
 833                  */
 834                 for_each_compatible_node(np, NULL, "ibm,ioda-hub") {
 835                         pnv_pci_init_ioda_hub(np);
 836                         found_ioda = 1;
 837                 }
 838
 839                 /* Look for p5ioc2 IO-Hubs */
 840                 if (!found_ioda)
 841                         for_each_compatible_node(np, NULL, "ibm,p5ioc2")
 842                                 pnv_pci_init_p5ioc2_hub(np);
 843
 844                 /* Look for ioda2 built-in PHB3's */
 845                 for_each_compatible_node(np, NULL, "ibm,ioda2-phb")
 846                         pnv_pci_init_ioda2_phb(np);
 847         }
 848
 849         /* Setup the linkage between OF nodes and PHBs */
 850         pci_devs_phb_init();
 851
 852         /* Configure IOMMU DMA hooks */
 853         ppc_md.pci_dma_dev_setup = pnv_pci_dma_dev_setup;
 854         ppc_md.tce_build = pnv_tce_build_vm;
 855         ppc_md.tce_free = pnv_tce_free_vm;
 856         ppc_md.tce_build_rm = pnv_tce_build_rm;
 857         ppc_md.tce_free_rm = pnv_tce_free_rm;
 858         ppc_md.tce_get = pnv_tce_get;
 859         ppc_md.pci_probe_mode = pnv_pci_probe_mode;
 860         set_pci_dma_ops(&dma_iommu_ops);
 861
 862         /* Configure MSIs */
 863 #ifdef CONFIG_PCI_MSI
 864         ppc_md.setup_msi_irqs = pnv_setup_msi_irqs;
 865         ppc_md.teardown_msi_irqs = pnv_teardown_msi_irqs;
 866 #endif
 867 }
 868
 869 static int tce_iommu_bus_notifier(struct notifier_block *nb,
 870                 unsigned long action, void *data)
 871 {
 872         struct device *dev = data;
 873
 874         switch (action) {
 875         case BUS_NOTIFY_ADD_DEVICE:
 876                 return iommu_add_device(dev);
 877         case BUS_NOTIFY_DEL_DEVICE:
 878                 if (dev->iommu_group)
 879                         iommu_del_device(dev);
 880                 return 0;
 881         default:
 882                 return 0;
 883         }
 884 }
 885
 886 static struct notifier_block tce_iommu_bus_nb = {
 887         .notifier_call = tce_iommu_bus_notifier,
 888 };
 889
 890 static int __init tce_iommu_bus_notifier_init(void)
 891 {
 892         bus_register_notifier(&pci_bus_type, &tce_iommu_bus_nb);
 893         return 0;
 894 }
 895 machine_subsys_initcall_sync(powernv, tce_iommu_bus_notifier_init);