drivers/iommu/arm-smmu.c

   1 /*
   2  * IOMMU API for ARM architected SMMU implementations.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11  * GNU General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public License
  14  * along with this program; if not, write to the Free Software
  15  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  16  *
  17  * Copyright (C) 2013 ARM Limited
  18  *
  19  * Author: Will Deacon <will.deacon@arm.com>
  20  *
  21  * This driver currently supports:
  22  *      - SMMUv1 and v2 implementations
  23  *      - Stream-matching and stream-indexing
  24  *      - v7/v8 long-descriptor format
  25  *      - Non-secure access to the SMMU
  26  *      - Context fault reporting
  27  */
  28
  29 #define pr_fmt(fmt) "arm-smmu: " fmt
  30
  31 #include <linux/atomic.h>
  32 #include <linux/delay.h>
  33 #include <linux/dma-iommu.h>
  34 #include <linux/dma-mapping.h>
  35 #include <linux/err.h>
  36 #include <linux/interrupt.h>
  37 #include <linux/io.h>
  38 #include <linux/io-64-nonatomic-hi-lo.h>
  39 #include <linux/iommu.h>
  40 #include <linux/iopoll.h>
  41 #include <linux/module.h>
  42 #include <linux/of.h>
  43 #include <linux/of_address.h>
  44 #include <linux/of_device.h>
  45 #include <linux/of_iommu.h>
  46 #include <linux/pci.h>
  47 #include <linux/platform_device.h>
  48 #include <linux/slab.h>
  49 #include <linux/spinlock.h>
  50
  51 #include <linux/amba/bus.h>
  52
  53 #include "io-pgtable.h"
  54
  55 /* Maximum number of context banks per SMMU */
  56 #define ARM_SMMU_MAX_CBS                128
  57
  58 /* SMMU global address space */
  59 #define ARM_SMMU_GR0(smmu)              ((smmu)->base)
  60 #define ARM_SMMU_GR1(smmu)              ((smmu)->base + (1 << (smmu)->pgshift))
  61
  62 /*
  63  * SMMU global address space with conditional offset to access secure
  64  * aliases of non-secure registers (e.g. nsCR0: 0x400, nsGFSR: 0x448,
  65  * nsGFSYNR0: 0x450)
  66  */
  67 #define ARM_SMMU_GR0_NS(smmu)                                           \
  68         ((smmu)->base +                                                 \
  69                 ((smmu->options & ARM_SMMU_OPT_SECURE_CFG_ACCESS)       \
  70                         ? 0x400 : 0))
  71
  72 /*
  73  * Some 64-bit registers only make sense to write atomically, but in such
  74  * cases all the data relevant to AArch32 formats lies within the lower word,
  75  * therefore this actually makes more sense than it might first appear.
  76  */
  77 #ifdef CONFIG_64BIT
  78 #define smmu_write_atomic_lq            writeq_relaxed
  79 #else
  80 #define smmu_write_atomic_lq            writel_relaxed
  81 #endif
  82
  83 /* Configuration registers */
  84 #define ARM_SMMU_GR0_sCR0               0x0
  85 #define sCR0_CLIENTPD                   (1 << 0)
  86 #define sCR0_GFRE                       (1 << 1)
  87 #define sCR0_GFIE                       (1 << 2)
  88 #define sCR0_GCFGFRE                    (1 << 4)
  89 #define sCR0_GCFGFIE                    (1 << 5)
  90 #define sCR0_USFCFG                     (1 << 10)
  91 #define sCR0_VMIDPNE                    (1 << 11)
  92 #define sCR0_PTM                        (1 << 12)
  93 #define sCR0_FB                         (1 << 13)
  94 #define sCR0_VMID16EN                   (1 << 31)
  95 #define sCR0_BSU_SHIFT                  14
  96 #define sCR0_BSU_MASK                   0x3
  97
  98 /* Auxiliary Configuration register */
  99 #define ARM_SMMU_GR0_sACR               0x10
 100
 101 /* Identification registers */
 102 #define ARM_SMMU_GR0_ID0                0x20
 103 #define ARM_SMMU_GR0_ID1                0x24
 104 #define ARM_SMMU_GR0_ID2                0x28
 105 #define ARM_SMMU_GR0_ID3                0x2c
 106 #define ARM_SMMU_GR0_ID4                0x30
 107 #define ARM_SMMU_GR0_ID5                0x34
 108 #define ARM_SMMU_GR0_ID6                0x38
 109 #define ARM_SMMU_GR0_ID7                0x3c
 110 #define ARM_SMMU_GR0_sGFSR              0x48
 111 #define ARM_SMMU_GR0_sGFSYNR0           0x50
 112 #define ARM_SMMU_GR0_sGFSYNR1           0x54
 113 #define ARM_SMMU_GR0_sGFSYNR2           0x58
 114
 115 #define ID0_S1TS                        (1 << 30)
 116 #define ID0_S2TS                        (1 << 29)
 117 #define ID0_NTS                         (1 << 28)
 118 #define ID0_SMS                         (1 << 27)
 119 #define ID0_ATOSNS                      (1 << 26)
 120 #define ID0_PTFS_NO_AARCH32             (1 << 25)
 121 #define ID0_PTFS_NO_AARCH32S            (1 << 24)
 122 #define ID0_CTTW                        (1 << 14)
 123 #define ID0_NUMIRPT_SHIFT               16
 124 #define ID0_NUMIRPT_MASK                0xff
 125 #define ID0_NUMSIDB_SHIFT               9
 126 #define ID0_NUMSIDB_MASK                0xf
 127 #define ID0_NUMSMRG_SHIFT               0
 128 #define ID0_NUMSMRG_MASK                0xff
 129
 130 #define ID1_PAGESIZE                    (1 << 31)
 131 #define ID1_NUMPAGENDXB_SHIFT           28
 132 #define ID1_NUMPAGENDXB_MASK            7
 133 #define ID1_NUMS2CB_SHIFT               16
 134 #define ID1_NUMS2CB_MASK                0xff
 135 #define ID1_NUMCB_SHIFT                 0
 136 #define ID1_NUMCB_MASK                  0xff
 137
 138 #define ID2_OAS_SHIFT                   4
 139 #define ID2_OAS_MASK                    0xf
 140 #define ID2_IAS_SHIFT                   0
 141 #define ID2_IAS_MASK                    0xf
 142 #define ID2_UBS_SHIFT                   8
 143 #define ID2_UBS_MASK                    0xf
 144 #define ID2_PTFS_4K                     (1 << 12)
 145 #define ID2_PTFS_16K                    (1 << 13)
 146 #define ID2_PTFS_64K                    (1 << 14)
 147 #define ID2_VMID16                      (1 << 15)
 148
 149 #define ID7_MAJOR_SHIFT                 4
 150 #define ID7_MAJOR_MASK                  0xf
 151
 152 /* Global TLB invalidation */
 153 #define ARM_SMMU_GR0_TLBIVMID           0x64
 154 #define ARM_SMMU_GR0_TLBIALLNSNH        0x68
 155 #define ARM_SMMU_GR0_TLBIALLH           0x6c
 156 #define ARM_SMMU_GR0_sTLBGSYNC          0x70
 157 #define ARM_SMMU_GR0_sTLBGSTATUS        0x74
 158 #define sTLBGSTATUS_GSACTIVE            (1 << 0)
 159 #define TLB_LOOP_TIMEOUT                1000000 /* 1s! */
 160
 161 /* Stream mapping registers */
 162 #define ARM_SMMU_GR0_SMR(n)             (0x800 + ((n) << 2))
 163 #define SMR_VALID                       (1 << 31)
 164 #define SMR_MASK_SHIFT                  16
 165 #define SMR_ID_SHIFT                    0
 166
 167 #define ARM_SMMU_GR0_S2CR(n)            (0xc00 + ((n) << 2))
 168 #define S2CR_CBNDX_SHIFT                0
 169 #define S2CR_CBNDX_MASK                 0xff
 170 #define S2CR_TYPE_SHIFT                 16
 171 #define S2CR_TYPE_MASK                  0x3
 172 enum arm_smmu_s2cr_type {
 173         S2CR_TYPE_TRANS,
 174         S2CR_TYPE_BYPASS,
 175         S2CR_TYPE_FAULT,
 176 };
 177
 178 #define S2CR_PRIVCFG_SHIFT              24
 179 #define S2CR_PRIVCFG_MASK               0x3
 180 enum arm_smmu_s2cr_privcfg {
 181         S2CR_PRIVCFG_DEFAULT,
 182         S2CR_PRIVCFG_DIPAN,
 183         S2CR_PRIVCFG_UNPRIV,
 184         S2CR_PRIVCFG_PRIV,
 185 };
 186
 187 /* Context bank attribute registers */
 188 #define ARM_SMMU_GR1_CBAR(n)            (0x0 + ((n) << 2))
 189 #define CBAR_VMID_SHIFT                 0
 190 #define CBAR_VMID_MASK                  0xff
 191 #define CBAR_S1_BPSHCFG_SHIFT           8
 192 #define CBAR_S1_BPSHCFG_MASK            3
 193 #define CBAR_S1_BPSHCFG_NSH             3
 194 #define CBAR_S1_MEMATTR_SHIFT           12
 195 #define CBAR_S1_MEMATTR_MASK            0xf
 196 #define CBAR_S1_MEMATTR_WB              0xf
 197 #define CBAR_TYPE_SHIFT                 16
 198 #define CBAR_TYPE_MASK                  0x3
 199 #define CBAR_TYPE_S2_TRANS              (0 << CBAR_TYPE_SHIFT)
 200 #define CBAR_TYPE_S1_TRANS_S2_BYPASS    (1 << CBAR_TYPE_SHIFT)
 201 #define CBAR_TYPE_S1_TRANS_S2_FAULT     (2 << CBAR_TYPE_SHIFT)
 202 #define CBAR_TYPE_S1_TRANS_S2_TRANS     (3 << CBAR_TYPE_SHIFT)
 203 #define CBAR_IRPTNDX_SHIFT              24
 204 #define CBAR_IRPTNDX_MASK               0xff
 205
 206 #define ARM_SMMU_GR1_CBA2R(n)           (0x800 + ((n) << 2))
 207 #define CBA2R_RW64_32BIT                (0 << 0)
 208 #define CBA2R_RW64_64BIT                (1 << 0)
 209 #define CBA2R_VMID_SHIFT                16
 210 #define CBA2R_VMID_MASK                 0xffff
 211
 212 /* Translation context bank */
 213 #define ARM_SMMU_CB_BASE(smmu)          ((smmu)->base + ((smmu)->size >> 1))
 214 #define ARM_SMMU_CB(smmu, n)            ((n) * (1 << (smmu)->pgshift))
 215
 216 #define ARM_SMMU_CB_SCTLR               0x0
 217 #define ARM_SMMU_CB_ACTLR               0x4
 218 #define ARM_SMMU_CB_RESUME              0x8
 219 #define ARM_SMMU_CB_TTBCR2              0x10
 220 #define ARM_SMMU_CB_TTBR0               0x20
 221 #define ARM_SMMU_CB_TTBR1               0x28
 222 #define ARM_SMMU_CB_TTBCR               0x30
 223 #define ARM_SMMU_CB_CONTEXTIDR          0x34
 224 #define ARM_SMMU_CB_S1_MAIR0            0x38
 225 #define ARM_SMMU_CB_S1_MAIR1            0x3c
 226 #define ARM_SMMU_CB_PAR                 0x50
 227 #define ARM_SMMU_CB_FSR                 0x58
 228 #define ARM_SMMU_CB_FAR                 0x60
 229 #define ARM_SMMU_CB_FSYNR0              0x68
 230 #define ARM_SMMU_CB_S1_TLBIVA           0x600
 231 #define ARM_SMMU_CB_S1_TLBIASID         0x610
 232 #define ARM_SMMU_CB_S1_TLBIVAL          0x620
 233 #define ARM_SMMU_CB_S2_TLBIIPAS2        0x630
 234 #define ARM_SMMU_CB_S2_TLBIIPAS2L       0x638
 235 #define ARM_SMMU_CB_ATS1PR              0x800
 236 #define ARM_SMMU_CB_ATSR                0x8f0
 237
 238 #define SCTLR_S1_ASIDPNE                (1 << 12)
 239 #define SCTLR_CFCFG                     (1 << 7)
 240 #define SCTLR_CFIE                      (1 << 6)
 241 #define SCTLR_CFRE                      (1 << 5)
 242 #define SCTLR_E                         (1 << 4)
 243 #define SCTLR_AFE                       (1 << 2)
 244 #define SCTLR_TRE                       (1 << 1)
 245 #define SCTLR_M                         (1 << 0)
 246
 247 #define ARM_MMU500_ACTLR_CPRE           (1 << 1)
 248
 249 #define ARM_MMU500_ACR_CACHE_LOCK       (1 << 26)
 250
 251 #define CB_PAR_F                        (1 << 0)
 252
 253 #define ATSR_ACTIVE                     (1 << 0)
 254
 255 #define RESUME_RETRY                    (0 << 0)
 256 #define RESUME_TERMINATE                (1 << 0)
 257
 258 #define TTBCR2_SEP_SHIFT                15
 259 #define TTBCR2_SEP_UPSTREAM             (0x7 << TTBCR2_SEP_SHIFT)
 260
 261 #define TTBRn_ASID_SHIFT                48
 262
 263 #define FSR_MULTI                       (1 << 31)
 264 #define FSR_SS                          (1 << 30)
 265 #define FSR_UUT                         (1 << 8)
 266 #define FSR_ASF                         (1 << 7)
 267 #define FSR_TLBLKF                      (1 << 6)
 268 #define FSR_TLBMCF                      (1 << 5)
 269 #define FSR_EF                          (1 << 4)
 270 #define FSR_PF                          (1 << 3)
 271 #define FSR_AFF                         (1 << 2)
 272 #define FSR_TF                          (1 << 1)
 273
 274 #define FSR_IGN                         (FSR_AFF | FSR_ASF | \
 275                                          FSR_TLBMCF | FSR_TLBLKF)
 276 #define FSR_FAULT                       (FSR_MULTI | FSR_SS | FSR_UUT | \
 277                                          FSR_EF | FSR_PF | FSR_TF | FSR_IGN)
 278
 279 #define FSYNR0_WNR                      (1 << 4)
 280
 281 static int force_stage;
 282 module_param(force_stage, int, S_IRUGO);
 283 MODULE_PARM_DESC(force_stage,
 284         "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
 285 static bool disable_bypass;
 286 module_param(disable_bypass, bool, S_IRUGO);
 287 MODULE_PARM_DESC(disable_bypass,
 288         "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
 289
 290 enum arm_smmu_arch_version {
 291         ARM_SMMU_V1,
 292         ARM_SMMU_V1_64K,
 293         ARM_SMMU_V2,
 294 };
 295
 296 enum arm_smmu_implementation {
 297         GENERIC_SMMU,
 298         ARM_MMU500,
 299         CAVIUM_SMMUV2,
 300 };
 301
 302 struct arm_smmu_s2cr {
 303         struct iommu_group              *group;
 304         int                             count;
 305         enum arm_smmu_s2cr_type         type;
 306         enum arm_smmu_s2cr_privcfg      privcfg;
 307         u8                              cbndx;
 308 };
 309
 310 #define s2cr_init_val (struct arm_smmu_s2cr){                           \
 311         .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS,    \
 312 }
 313
 314 struct arm_smmu_smr {
 315         u16                             mask;
 316         u16                             id;
 317         bool                            valid;
 318 };
 319
 320 struct arm_smmu_master_cfg {
 321         struct arm_smmu_device          *smmu;
 322         s16                             smendx[];
 323 };
 324 #define INVALID_SMENDX                  -1
 325 #define __fwspec_cfg(fw) ((struct arm_smmu_master_cfg *)fw->iommu_priv)
 326 #define fwspec_smmu(fw)  (__fwspec_cfg(fw)->smmu)
 327 #define for_each_cfg_sme(fw, i, idx) \
 328         for (i = 0; idx = __fwspec_cfg(fw)->smendx[i], i < fw->num_ids; ++i)
 329
 330 struct arm_smmu_device {
 331         struct device                   *dev;
 332
 333         void __iomem                    *base;
 334         unsigned long                   size;
 335         unsigned long                   pgshift;
 336
 337 #define ARM_SMMU_FEAT_COHERENT_WALK     (1 << 0)
 338 #define ARM_SMMU_FEAT_STREAM_MATCH      (1 << 1)
 339 #define ARM_SMMU_FEAT_TRANS_S1          (1 << 2)
 340 #define ARM_SMMU_FEAT_TRANS_S2          (1 << 3)
 341 #define ARM_SMMU_FEAT_TRANS_NESTED      (1 << 4)
 342 #define ARM_SMMU_FEAT_TRANS_OPS         (1 << 5)
 343 #define ARM_SMMU_FEAT_VMID16            (1 << 6)
 344 #define ARM_SMMU_FEAT_FMT_AARCH64_4K    (1 << 7)
 345 #define ARM_SMMU_FEAT_FMT_AARCH64_16K   (1 << 8)
 346 #define ARM_SMMU_FEAT_FMT_AARCH64_64K   (1 << 9)
 347 #define ARM_SMMU_FEAT_FMT_AARCH32_L     (1 << 10)
 348 #define ARM_SMMU_FEAT_FMT_AARCH32_S     (1 << 11)
 349         u32                             features;
 350
 351 #define ARM_SMMU_OPT_SECURE_CFG_ACCESS (1 << 0)
 352         u32                             options;
 353         enum arm_smmu_arch_version      version;
 354         enum arm_smmu_implementation    model;
 355
 356         u32                             num_context_banks;
 357         u32                             num_s2_context_banks;
 358         DECLARE_BITMAP(context_map, ARM_SMMU_MAX_CBS);
 359         atomic_t                        irptndx;
 360
 361         u32                             num_mapping_groups;
 362         u16                             streamid_mask;
 363         u16                             smr_mask_mask;
 364         struct arm_smmu_smr             *smrs;
 365         struct arm_smmu_s2cr            *s2crs;
 366         struct mutex                    stream_map_mutex;
 367
 368         unsigned long                   va_size;
 369         unsigned long                   ipa_size;
 370         unsigned long                   pa_size;
 371         unsigned long                   pgsize_bitmap;
 372
 373         u32                             num_global_irqs;
 374         u32                             num_context_irqs;
 375         unsigned int                    *irqs;
 376
 377         u32                             cavium_id_base; /* Specific to Cavium */
 378 };
 379
 380 enum arm_smmu_context_fmt {
 381         ARM_SMMU_CTX_FMT_NONE,
 382         ARM_SMMU_CTX_FMT_AARCH64,
 383         ARM_SMMU_CTX_FMT_AARCH32_L,
 384         ARM_SMMU_CTX_FMT_AARCH32_S,
 385 };
 386
 387 struct arm_smmu_cfg {
 388         u8                              cbndx;
 389         u8                              irptndx;
 390         u32                             cbar;
 391         enum arm_smmu_context_fmt       fmt;
 392 };
 393 #define INVALID_IRPTNDX                 0xff
 394
 395 #define ARM_SMMU_CB_ASID(smmu, cfg) ((u16)(smmu)->cavium_id_base + (cfg)->cbndx)
 396 #define ARM_SMMU_CB_VMID(smmu, cfg) ((u16)(smmu)->cavium_id_base + (cfg)->cbndx + 1)
 397
 398 enum arm_smmu_domain_stage {
 399         ARM_SMMU_DOMAIN_S1 = 0,
 400         ARM_SMMU_DOMAIN_S2,
 401         ARM_SMMU_DOMAIN_NESTED,
 402 };
 403
 404 struct arm_smmu_domain {
 405         struct arm_smmu_device          *smmu;
 406         struct io_pgtable_ops           *pgtbl_ops;
 407         spinlock_t                      pgtbl_lock;
 408         struct arm_smmu_cfg             cfg;
 409         enum arm_smmu_domain_stage      stage;
 410         struct mutex                    init_mutex; /* Protects smmu pointer */
 411         struct iommu_domain             domain;
 412 };
 413
 414 struct arm_smmu_option_prop {
 415         u32 opt;
 416         const char *prop;
 417 };
 418
 419 static atomic_t cavium_smmu_context_count = ATOMIC_INIT(0);
 420
 421 static bool using_legacy_binding, using_generic_binding;
 422
 423 static struct arm_smmu_option_prop arm_smmu_options[] = {
 424         { ARM_SMMU_OPT_SECURE_CFG_ACCESS, "calxeda,smmu-secure-config-access" },
 425         { 0, NULL},
 426 };
 427
 428 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
 429 {
 430         return container_of(dom, struct arm_smmu_domain, domain);
 431 }
 432
 433 static void parse_driver_options(struct arm_smmu_device *smmu)
 434 {
 435         int i = 0;
 436
 437         do {
 438                 if (of_property_read_bool(smmu->dev->of_node,
 439                                                 arm_smmu_options[i].prop)) {
 440                         smmu->options |= arm_smmu_options[i].opt;
 441                         dev_notice(smmu->dev, "option %s\n",
 442                                 arm_smmu_options[i].prop);
 443                 }
 444         } while (arm_smmu_options[++i].opt);
 445 }
 446
 447 static struct device_node *dev_get_dev_node(struct device *dev)
 448 {
 449         if (dev_is_pci(dev)) {
 450                 struct pci_bus *bus = to_pci_dev(dev)->bus;
 451
 452                 while (!pci_is_root_bus(bus))
 453                         bus = bus->parent;
 454                 return of_node_get(bus->bridge->parent->of_node);
 455         }
 456
 457         return of_node_get(dev->of_node);
 458 }
 459
 460 static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
 461 {
 462         *((__be32 *)data) = cpu_to_be32(alias);
 463         return 0; /* Continue walking */
 464 }
 465
 466 static int __find_legacy_master_phandle(struct device *dev, void *data)
 467 {
 468         struct of_phandle_iterator *it = *(void **)data;
 469         struct device_node *np = it->node;
 470         int err;
 471
 472         of_for_each_phandle(it, err, dev->of_node, "mmu-masters",
 473                             "#stream-id-cells", 0)
 474                 if (it->node == np) {
 475                         *(void **)data = dev;
 476                         return 1;
 477                 }
 478         it->node = np;
 479         return err == -ENOENT ? 0 : err;
 480 }
 481
 482 static struct platform_driver arm_smmu_driver;
 483 static struct iommu_ops arm_smmu_ops;
 484
 485 static int arm_smmu_register_legacy_master(struct device *dev,
 486                                            struct arm_smmu_device **smmu)
 487 {
 488         struct device *smmu_dev;
 489         struct device_node *np;
 490         struct of_phandle_iterator it;
 491         void *data = &it;
 492         u32 *sids;
 493         __be32 pci_sid;
 494         int err;
 495
 496         np = dev_get_dev_node(dev);
 497         if (!np || !of_find_property(np, "#stream-id-cells", NULL)) {
 498                 of_node_put(np);
 499                 return -ENODEV;
 500         }
 501
 502         it.node = np;
 503         err = driver_for_each_device(&arm_smmu_driver.driver, NULL, &data,
 504                                      __find_legacy_master_phandle);
 505         smmu_dev = data;
 506         of_node_put(np);
 507         if (err == 0)
 508                 return -ENODEV;
 509         if (err < 0)
 510                 return err;
 511
 512         if (dev_is_pci(dev)) {
 513                 /* "mmu-masters" assumes Stream ID == Requester ID */
 514                 pci_for_each_dma_alias(to_pci_dev(dev), __arm_smmu_get_pci_sid,
 515                                        &pci_sid);
 516                 it.cur = &pci_sid;
 517                 it.cur_count = 1;
 518         }
 519
 520         err = iommu_fwspec_init(dev, &smmu_dev->of_node->fwnode,
 521                                 &arm_smmu_ops);
 522         if (err)
 523                 return err;
 524
 525         sids = kcalloc(it.cur_count, sizeof(*sids), GFP_KERNEL);
 526         if (!sids)
 527                 return -ENOMEM;
 528
 529         *smmu = dev_get_drvdata(smmu_dev);
 530         of_phandle_iterator_args(&it, sids, it.cur_count);
 531         err = iommu_fwspec_add_ids(dev, sids, it.cur_count);
 532         kfree(sids);
 533         return err;
 534 }
 535
 536 static int __arm_smmu_alloc_bitmap(unsigned long *map, int start, int end)
 537 {
 538         int idx;
 539
 540         do {
 541                 idx = find_next_zero_bit(map, end, start);
 542                 if (idx == end)
 543                         return -ENOSPC;
 544         } while (test_and_set_bit(idx, map));
 545
 546         return idx;
 547 }
 548
 549 static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
 550 {
 551         clear_bit(idx, map);
 552 }
 553
 554 /* Wait for any pending TLB invalidations to complete */
 555 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu)
 556 {
 557         int count = 0;
 558         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
 559
 560         writel_relaxed(0, gr0_base + ARM_SMMU_GR0_sTLBGSYNC);
 561         while (readl_relaxed(gr0_base + ARM_SMMU_GR0_sTLBGSTATUS)
 562                & sTLBGSTATUS_GSACTIVE) {
 563                 cpu_relax();
 564                 if (++count == TLB_LOOP_TIMEOUT) {
 565                         dev_err_ratelimited(smmu->dev,
 566                         "TLB sync timed out -- SMMU may be deadlocked\n");
 567                         return;
 568                 }
 569                 udelay(1);
 570         }
 571 }
 572
 573 static void arm_smmu_tlb_sync(void *cookie)
 574 {
 575         struct arm_smmu_domain *smmu_domain = cookie;
 576         __arm_smmu_tlb_sync(smmu_domain->smmu);
 577 }
 578
 579 static void arm_smmu_tlb_inv_context(void *cookie)
 580 {
 581         struct arm_smmu_domain *smmu_domain = cookie;
 582         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 583         struct arm_smmu_device *smmu = smmu_domain->smmu;
 584         bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 585         void __iomem *base;
 586
 587         if (stage1) {
 588                 base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 589                 writel_relaxed(ARM_SMMU_CB_ASID(smmu, cfg),
 590                                base + ARM_SMMU_CB_S1_TLBIASID);
 591         } else {
 592                 base = ARM_SMMU_GR0(smmu);
 593                 writel_relaxed(ARM_SMMU_CB_VMID(smmu, cfg),
 594                                base + ARM_SMMU_GR0_TLBIVMID);
 595         }
 596
 597         __arm_smmu_tlb_sync(smmu);
 598 }
 599
 600 static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
 601                                           size_t granule, bool leaf, void *cookie)
 602 {
 603         struct arm_smmu_domain *smmu_domain = cookie;
 604         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 605         struct arm_smmu_device *smmu = smmu_domain->smmu;
 606         bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 607         void __iomem *reg;
 608
 609         if (stage1) {
 610                 reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 611                 reg += leaf ? ARM_SMMU_CB_S1_TLBIVAL : ARM_SMMU_CB_S1_TLBIVA;
 612
 613                 if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) {
 614                         iova &= ~12UL;
 615                         iova |= ARM_SMMU_CB_ASID(smmu, cfg);
 616                         do {
 617                                 writel_relaxed(iova, reg);
 618                                 iova += granule;
 619                         } while (size -= granule);
 620                 } else {
 621                         iova >>= 12;
 622                         iova |= (u64)ARM_SMMU_CB_ASID(smmu, cfg) << 48;
 623                         do {
 624                                 writeq_relaxed(iova, reg);
 625                                 iova += granule >> 12;
 626                         } while (size -= granule);
 627                 }
 628         } else if (smmu->version == ARM_SMMU_V2) {
 629                 reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 630                 reg += leaf ? ARM_SMMU_CB_S2_TLBIIPAS2L :
 631                               ARM_SMMU_CB_S2_TLBIIPAS2;
 632                 iova >>= 12;
 633                 do {
 634                         smmu_write_atomic_lq(iova, reg);
 635                         iova += granule >> 12;
 636                 } while (size -= granule);
 637         } else {
 638                 reg = ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_TLBIVMID;
 639                 writel_relaxed(ARM_SMMU_CB_VMID(smmu, cfg), reg);
 640         }
 641 }
 642
 643 static struct iommu_gather_ops arm_smmu_gather_ops = {
 644         .tlb_flush_all  = arm_smmu_tlb_inv_context,
 645         .tlb_add_flush  = arm_smmu_tlb_inv_range_nosync,
 646         .tlb_sync       = arm_smmu_tlb_sync,
 647 };
 648
 649 static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
 650 {
 651         u32 fsr, fsynr;
 652         unsigned long iova;
 653         struct iommu_domain *domain = dev;
 654         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 655         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 656         struct arm_smmu_device *smmu = smmu_domain->smmu;
 657         void __iomem *cb_base;
 658
 659         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 660         fsr = readl_relaxed(cb_base + ARM_SMMU_CB_FSR);
 661
 662         if (!(fsr & FSR_FAULT))
 663                 return IRQ_NONE;
 664
 665         fsynr = readl_relaxed(cb_base + ARM_SMMU_CB_FSYNR0);
 666         iova = readq_relaxed(cb_base + ARM_SMMU_CB_FAR);
 667
 668         dev_err_ratelimited(smmu->dev,
 669         "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cb=%d\n",
 670                             fsr, iova, fsynr, cfg->cbndx);
 671
 672         writel(fsr, cb_base + ARM_SMMU_CB_FSR);
 673         return IRQ_HANDLED;
 674 }
 675
 676 static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
 677 {
 678         u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
 679         struct arm_smmu_device *smmu = dev;
 680         void __iomem *gr0_base = ARM_SMMU_GR0_NS(smmu);
 681
 682         gfsr = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSR);
 683         gfsynr0 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR0);
 684         gfsynr1 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR1);
 685         gfsynr2 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR2);
 686
 687         if (!gfsr)
 688                 return IRQ_NONE;
 689
 690         dev_err_ratelimited(smmu->dev,
 691                 "Unexpected global fault, this could be serious\n");
 692         dev_err_ratelimited(smmu->dev,
 693                 "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
 694                 gfsr, gfsynr0, gfsynr1, gfsynr2);
 695
 696         writel(gfsr, gr0_base + ARM_SMMU_GR0_sGFSR);
 697         return IRQ_HANDLED;
 698 }
 699
 700 static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
 701                                        struct io_pgtable_cfg *pgtbl_cfg)
 702 {
 703         u32 reg, reg2;
 704         u64 reg64;
 705         bool stage1;
 706         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 707         struct arm_smmu_device *smmu = smmu_domain->smmu;
 708         void __iomem *cb_base, *gr1_base;
 709
 710         gr1_base = ARM_SMMU_GR1(smmu);
 711         stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 712         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 713
 714         if (smmu->version > ARM_SMMU_V1) {
 715                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
 716                         reg = CBA2R_RW64_64BIT;
 717                 else
 718                         reg = CBA2R_RW64_32BIT;
 719                 /* 16-bit VMIDs live in CBA2R */
 720                 if (smmu->features & ARM_SMMU_FEAT_VMID16)
 721                         reg |= ARM_SMMU_CB_VMID(smmu, cfg) << CBA2R_VMID_SHIFT;
 722
 723                 writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBA2R(cfg->cbndx));
 724         }
 725
 726         /* CBAR */
 727         reg = cfg->cbar;
 728         if (smmu->version < ARM_SMMU_V2)
 729                 reg |= cfg->irptndx << CBAR_IRPTNDX_SHIFT;
 730
 731         /*
 732          * Use the weakest shareability/memory types, so they are
 733          * overridden by the ttbcr/pte.
 734          */
 735         if (stage1) {
 736                 reg |= (CBAR_S1_BPSHCFG_NSH << CBAR_S1_BPSHCFG_SHIFT) |
 737                         (CBAR_S1_MEMATTR_WB << CBAR_S1_MEMATTR_SHIFT);
 738         } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) {
 739                 /* 8-bit VMIDs live in CBAR */
 740                 reg |= ARM_SMMU_CB_VMID(smmu, cfg) << CBAR_VMID_SHIFT;
 741         }
 742         writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBAR(cfg->cbndx));
 743
 744         /* TTBRs */
 745         if (stage1) {
 746                 u16 asid = ARM_SMMU_CB_ASID(smmu, cfg);
 747
 748                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 749                         reg = pgtbl_cfg->arm_v7s_cfg.ttbr[0];
 750                         writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR0);
 751                         reg = pgtbl_cfg->arm_v7s_cfg.ttbr[1];
 752                         writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR1);
 753                         writel_relaxed(asid, cb_base + ARM_SMMU_CB_CONTEXTIDR);
 754                 } else {
 755                         reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
 756                         reg64 |= (u64)asid << TTBRn_ASID_SHIFT;
 757                         writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0);
 758                         reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[1];
 759                         reg64 |= (u64)asid << TTBRn_ASID_SHIFT;
 760                         writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR1);
 761                 }
 762         } else {
 763                 reg64 = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
 764                 writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0);
 765         }
 766
 767         /* TTBCR */
 768         if (stage1) {
 769                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 770                         reg = pgtbl_cfg->arm_v7s_cfg.tcr;
 771                         reg2 = 0;
 772                 } else {
 773                         reg = pgtbl_cfg->arm_lpae_s1_cfg.tcr;
 774                         reg2 = pgtbl_cfg->arm_lpae_s1_cfg.tcr >> 32;
 775                         reg2 |= TTBCR2_SEP_UPSTREAM;
 776                 }
 777                 if (smmu->version > ARM_SMMU_V1)
 778                         writel_relaxed(reg2, cb_base + ARM_SMMU_CB_TTBCR2);
 779         } else {
 780                 reg = pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
 781         }
 782         writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR);
 783
 784         /* MAIRs (stage-1 only) */
 785         if (stage1) {
 786                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 787                         reg = pgtbl_cfg->arm_v7s_cfg.prrr;
 788                         reg2 = pgtbl_cfg->arm_v7s_cfg.nmrr;
 789                 } else {
 790                         reg = pgtbl_cfg->arm_lpae_s1_cfg.mair[0];
 791                         reg2 = pgtbl_cfg->arm_lpae_s1_cfg.mair[1];
 792                 }
 793                 writel_relaxed(reg, cb_base + ARM_SMMU_CB_S1_MAIR0);
 794                 writel_relaxed(reg2, cb_base + ARM_SMMU_CB_S1_MAIR1);
 795         }
 796
 797         /* SCTLR */
 798         reg = SCTLR_CFIE | SCTLR_CFRE | SCTLR_AFE | SCTLR_TRE | SCTLR_M;
 799         if (stage1)
 800                 reg |= SCTLR_S1_ASIDPNE;
 801 #ifdef __BIG_ENDIAN
 802         reg |= SCTLR_E;
 803 #endif
 804         writel_relaxed(reg, cb_base + ARM_SMMU_CB_SCTLR);
 805 }
 806
 807 static int arm_smmu_init_domain_context(struct iommu_domain *domain,
 808                                         struct arm_smmu_device *smmu)
 809 {
 810         int irq, start, ret = 0;
 811         unsigned long ias, oas;
 812         struct io_pgtable_ops *pgtbl_ops;
 813         struct io_pgtable_cfg pgtbl_cfg;
 814         enum io_pgtable_fmt fmt;
 815         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 816         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 817
 818         mutex_lock(&smmu_domain->init_mutex);
 819         if (smmu_domain->smmu)
 820                 goto out_unlock;
 821
 822         /*
 823          * Mapping the requested stage onto what we support is surprisingly
 824          * complicated, mainly because the spec allows S1+S2 SMMUs without
 825          * support for nested translation. That means we end up with the
 826          * following table:
 827          *
 828          * Requested        Supported        Actual
 829          *     S1               N              S1
 830          *     S1             S1+S2            S1
 831          *     S1               S2             S2
 832          *     S1               S1             S1
 833          *     N                N              N
 834          *     N              S1+S2            S2
 835          *     N                S2             S2
 836          *     N                S1             S1
 837          *
 838          * Note that you can't actually request stage-2 mappings.
 839          */
 840         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
 841                 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
 842         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
 843                 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
 844
 845         /*
 846          * Choosing a suitable context format is even more fiddly. Until we
 847          * grow some way for the caller to express a preference, and/or move
 848          * the decision into the io-pgtable code where it arguably belongs,
 849          * just aim for the closest thing to the rest of the system, and hope
 850          * that the hardware isn't esoteric enough that we can't assume AArch64
 851          * support to be a superset of AArch32 support...
 852          */
 853         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L)
 854                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L;
 855         if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) &&
 856             !IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) &&
 857             (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) &&
 858             (smmu_domain->stage == ARM_SMMU_DOMAIN_S1))
 859                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S;
 860         if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) &&
 861             (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K |
 862                                ARM_SMMU_FEAT_FMT_AARCH64_16K |
 863                                ARM_SMMU_FEAT_FMT_AARCH64_4K)))
 864                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64;
 865
 866         if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) {
 867                 ret = -EINVAL;
 868                 goto out_unlock;
 869         }
 870
 871         switch (smmu_domain->stage) {
 872         case ARM_SMMU_DOMAIN_S1:
 873                 cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS;
 874                 start = smmu->num_s2_context_banks;
 875                 ias = smmu->va_size;
 876                 oas = smmu->ipa_size;
 877                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
 878                         fmt = ARM_64_LPAE_S1;
 879                 } else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) {
 880                         fmt = ARM_32_LPAE_S1;
 881                         ias = min(ias, 32UL);
 882                         oas = min(oas, 40UL);
 883                 } else {
 884                         fmt = ARM_V7S;
 885                         ias = min(ias, 32UL);
 886                         oas = min(oas, 32UL);
 887                 }
 888                 break;
 889         case ARM_SMMU_DOMAIN_NESTED:
 890                 /*
 891                  * We will likely want to change this if/when KVM gets
 892                  * involved.
 893                  */
 894         case ARM_SMMU_DOMAIN_S2:
 895                 cfg->cbar = CBAR_TYPE_S2_TRANS;
 896                 start = 0;
 897                 ias = smmu->ipa_size;
 898                 oas = smmu->pa_size;
 899                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
 900                         fmt = ARM_64_LPAE_S2;
 901                 } else {
 902                         fmt = ARM_32_LPAE_S2;
 903                         ias = min(ias, 40UL);
 904                         oas = min(oas, 40UL);
 905                 }
 906                 break;
 907         default:
 908                 ret = -EINVAL;
 909                 goto out_unlock;
 910         }
 911
 912         ret = __arm_smmu_alloc_bitmap(smmu->context_map, start,
 913                                       smmu->num_context_banks);
 914         if (ret < 0)
 915                 goto out_unlock;
 916
 917         cfg->cbndx = ret;
 918         if (smmu->version < ARM_SMMU_V2) {
 919                 cfg->irptndx = atomic_inc_return(&smmu->irptndx);
 920                 cfg->irptndx %= smmu->num_context_irqs;
 921         } else {
 922                 cfg->irptndx = cfg->cbndx;
 923         }
 924
 925         pgtbl_cfg = (struct io_pgtable_cfg) {
 926                 .pgsize_bitmap  = smmu->pgsize_bitmap,
 927                 .ias            = ias,
 928                 .oas            = oas,
 929                 .tlb            = &arm_smmu_gather_ops,
 930                 .iommu_dev      = smmu->dev,
 931         };
 932
 933         smmu_domain->smmu = smmu;
 934         pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
 935         if (!pgtbl_ops) {
 936                 ret = -ENOMEM;
 937                 goto out_clear_smmu;
 938         }
 939
 940         /* Update the domain's page sizes to reflect the page table format */
 941         domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
 942         domain->geometry.aperture_end = (1UL << ias) - 1;
 943         domain->geometry.force_aperture = true;
 944
 945         /* Initialise the context bank with our page table cfg */
 946         arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
 947
 948         /*
 949          * Request context fault interrupt. Do this last to avoid the
 950          * handler seeing a half-initialised domain state.
 951          */
 952         irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
 953         ret = devm_request_irq(smmu->dev, irq, arm_smmu_context_fault,
 954                                IRQF_SHARED, "arm-smmu-context-fault", domain);
 955         if (ret < 0) {
 956                 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
 957                         cfg->irptndx, irq);
 958                 cfg->irptndx = INVALID_IRPTNDX;
 959         }
 960
 961         mutex_unlock(&smmu_domain->init_mutex);
 962
 963         /* Publish page table ops for map/unmap */
 964         smmu_domain->pgtbl_ops = pgtbl_ops;
 965         return 0;
 966
 967 out_clear_smmu:
 968         smmu_domain->smmu = NULL;
 969 out_unlock:
 970         mutex_unlock(&smmu_domain->init_mutex);
 971         return ret;
 972 }
 973
 974 static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
 975 {
 976         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 977         struct arm_smmu_device *smmu = smmu_domain->smmu;
 978         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 979         void __iomem *cb_base;
 980         int irq;
 981
 982         if (!smmu)
 983                 return;
 984
 985         /*
 986          * Disable the context bank and free the page tables before freeing
 987          * it.
 988          */
 989         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 990         writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
 991
 992         if (cfg->irptndx != INVALID_IRPTNDX) {
 993                 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
 994                 devm_free_irq(smmu->dev, irq, domain);
 995         }
 996
 997         free_io_pgtable_ops(smmu_domain->pgtbl_ops);
 998         __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
 999 }
1000
1001 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
1002 {
1003         struct arm_smmu_domain *smmu_domain;
1004
1005         if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA)
1006                 return NULL;
1007         /*
1008          * Allocate the domain and initialise some of its data structures.
1009          * We can't really do anything meaningful until we've added a
1010          * master.
1011          */
1012         smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
1013         if (!smmu_domain)
1014                 return NULL;
1015
1016         if (type == IOMMU_DOMAIN_DMA && (using_legacy_binding ||
1017             iommu_get_dma_cookie(&smmu_domain->domain))) {
1018                 kfree(smmu_domain);
1019                 return NULL;
1020         }
1021
1022         mutex_init(&smmu_domain->init_mutex);
1023         spin_lock_init(&smmu_domain->pgtbl_lock);
1024
1025         return &smmu_domain->domain;
1026 }
1027
1028 static void arm_smmu_domain_free(struct iommu_domain *domain)
1029 {
1030         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1031
1032         /*
1033          * Free the domain resources. We assume that all devices have
1034          * already been detached.
1035          */
1036         iommu_put_dma_cookie(domain);
1037         arm_smmu_destroy_domain_context(domain);
1038         kfree(smmu_domain);
1039 }
1040
1041 static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx)
1042 {
1043         struct arm_smmu_smr *smr = smmu->smrs + idx;
1044         u32 reg = smr->id << SMR_ID_SHIFT | smr->mask << SMR_MASK_SHIFT;
1045
1046         if (smr->valid)
1047                 reg |= SMR_VALID;
1048         writel_relaxed(reg, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_SMR(idx));
1049 }
1050
1051 static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
1052 {
1053         struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
1054         u32 reg = (s2cr->type & S2CR_TYPE_MASK) << S2CR_TYPE_SHIFT |
1055                   (s2cr->cbndx & S2CR_CBNDX_MASK) << S2CR_CBNDX_SHIFT |
1056                   (s2cr->privcfg & S2CR_PRIVCFG_MASK) << S2CR_PRIVCFG_SHIFT;
1057
1058         writel_relaxed(reg, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_S2CR(idx));
1059 }
1060
1061 static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx)
1062 {
1063         arm_smmu_write_s2cr(smmu, idx);
1064         if (smmu->smrs)
1065                 arm_smmu_write_smr(smmu, idx);
1066 }
1067
1068 static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)
1069 {
1070         struct arm_smmu_smr *smrs = smmu->smrs;
1071         int i, free_idx = -ENOSPC;
1072
1073         /* Stream indexing is blissfully easy */
1074         if (!smrs)
1075                 return id;
1076
1077         /* Validating SMRs is... less so */
1078         for (i = 0; i < smmu->num_mapping_groups; ++i) {
1079                 if (!smrs[i].valid) {
1080                         /*
1081                          * Note the first free entry we come across, which
1082                          * we'll claim in the end if nothing else matches.
1083                          */
1084                         if (free_idx < 0)
1085                                 free_idx = i;
1086                         continue;
1087                 }
1088                 /*
1089                  * If the new entry is _entirely_ matched by an existing entry,
1090                  * then reuse that, with the guarantee that there also cannot
1091                  * be any subsequent conflicting entries. In normal use we'd
1092                  * expect simply identical entries for this case, but there's
1093                  * no harm in accommodating the generalisation.
1094                  */
1095                 if ((mask & smrs[i].mask) == mask &&
1096                     !((id ^ smrs[i].id) & ~smrs[i].mask))
1097                         return i;
1098                 /*
1099                  * If the new entry has any other overlap with an existing one,
1100                  * though, then there always exists at least one stream ID
1101                  * which would cause a conflict, and we can't allow that risk.
1102                  */
1103                 if (!((id ^ smrs[i].id) & ~(smrs[i].mask | mask)))
1104                         return -EINVAL;
1105         }
1106
1107         return free_idx;
1108 }
1109
1110 static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx)
1111 {
1112         if (--smmu->s2crs[idx].count)
1113                 return false;
1114
1115         smmu->s2crs[idx] = s2cr_init_val;
1116         if (smmu->smrs)
1117                 smmu->smrs[idx].valid = false;
1118
1119         return true;
1120 }
1121
1122 static int arm_smmu_master_alloc_smes(struct device *dev)
1123 {
1124         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1125         struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv;
1126         struct arm_smmu_device *smmu = cfg->smmu;
1127         struct arm_smmu_smr *smrs = smmu->smrs;
1128         struct iommu_group *group;
1129         int i, idx, ret;
1130
1131         mutex_lock(&smmu->stream_map_mutex);
1132         /* Figure out a viable stream map entry allocation */
1133         for_each_cfg_sme(fwspec, i, idx) {
1134                 u16 sid = fwspec->ids[i];
1135                 u16 mask = fwspec->ids[i] >> SMR_MASK_SHIFT;
1136
1137                 if (idx != INVALID_SMENDX) {
1138                         ret = -EEXIST;
1139                         goto out_err;
1140                 }
1141
1142                 ret = arm_smmu_find_sme(smmu, sid, mask);
1143                 if (ret < 0)
1144                         goto out_err;
1145
1146                 idx = ret;
1147                 if (smrs && smmu->s2crs[idx].count == 0) {
1148                         smrs[idx].id = sid;
1149                         smrs[idx].mask = mask;
1150                         smrs[idx].valid = true;
1151                 }
1152                 smmu->s2crs[idx].count++;
1153                 cfg->smendx[i] = (s16)idx;
1154         }
1155
1156         group = iommu_group_get_for_dev(dev);
1157         if (!group)
1158                 group = ERR_PTR(-ENOMEM);
1159         if (IS_ERR(group)) {
1160                 ret = PTR_ERR(group);
1161                 goto out_err;
1162         }
1163         iommu_group_put(group);
1164
1165         /* It worked! Now, poke the actual hardware */
1166         for_each_cfg_sme(fwspec, i, idx) {
1167                 arm_smmu_write_sme(smmu, idx);
1168                 smmu->s2crs[idx].group = group;
1169         }
1170
1171         mutex_unlock(&smmu->stream_map_mutex);
1172         return 0;
1173
1174 out_err:
1175         while (i--) {
1176                 arm_smmu_free_sme(smmu, cfg->smendx[i]);
1177                 cfg->smendx[i] = INVALID_SMENDX;
1178         }
1179         mutex_unlock(&smmu->stream_map_mutex);
1180         return ret;
1181 }
1182
1183 static void arm_smmu_master_free_smes(struct iommu_fwspec *fwspec)
1184 {
1185         struct arm_smmu_device *smmu = fwspec_smmu(fwspec);
1186         struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv;
1187         int i, idx;
1188
1189         mutex_lock(&smmu->stream_map_mutex);
1190         for_each_cfg_sme(fwspec, i, idx) {
1191                 if (arm_smmu_free_sme(smmu, idx))
1192                         arm_smmu_write_sme(smmu, idx);
1193                 cfg->smendx[i] = INVALID_SMENDX;
1194         }
1195         mutex_unlock(&smmu->stream_map_mutex);
1196 }
1197
1198 static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
1199                                       struct iommu_fwspec *fwspec)
1200 {
1201         struct arm_smmu_device *smmu = smmu_domain->smmu;
1202         struct arm_smmu_s2cr *s2cr = smmu->s2crs;
1203         enum arm_smmu_s2cr_type type = S2CR_TYPE_TRANS;
1204         u8 cbndx = smmu_domain->cfg.cbndx;
1205         int i, idx;
1206
1207         for_each_cfg_sme(fwspec, i, idx) {
1208                 if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx)
1209                         continue;
1210
1211                 s2cr[idx].type = type;
1212                 s2cr[idx].privcfg = S2CR_PRIVCFG_UNPRIV;
1213                 s2cr[idx].cbndx = cbndx;
1214                 arm_smmu_write_s2cr(smmu, idx);
1215         }
1216         return 0;
1217 }
1218
1219 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1220 {
1221         int ret;
1222         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1223         struct arm_smmu_device *smmu;
1224         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1225
1226         if (!fwspec || fwspec->ops != &arm_smmu_ops) {
1227                 dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n");
1228                 return -ENXIO;
1229         }
1230
1231         smmu = fwspec_smmu(fwspec);
1232         /* Ensure that the domain is finalised */
1233         ret = arm_smmu_init_domain_context(domain, smmu);
1234         if (ret < 0)
1235                 return ret;
1236
1237         /*
1238          * Sanity check the domain. We don't support domains across
1239          * different SMMUs.
1240          */
1241         if (smmu_domain->smmu != smmu) {
1242                 dev_err(dev,
1243                         "cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n",
1244                         dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev));
1245                 return -EINVAL;
1246         }
1247
1248         /* Looks ok, so add the device to the domain */
1249         return arm_smmu_domain_add_master(smmu_domain, fwspec);
1250 }
1251
1252 static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
1253                         phys_addr_t paddr, size_t size, int prot)
1254 {
1255         int ret;
1256         unsigned long flags;
1257         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1258         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1259
1260         if (!ops)
1261                 return -ENODEV;
1262
1263         spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1264         ret = ops->map(ops, iova, paddr, size, prot);
1265         spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1266         return ret;
1267 }
1268
1269 static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
1270                              size_t size)
1271 {
1272         size_t ret;
1273         unsigned long flags;
1274         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1275         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1276
1277         if (!ops)
1278                 return 0;
1279
1280         spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1281         ret = ops->unmap(ops, iova, size);
1282         spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1283         return ret;
1284 }
1285
1286 static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
1287                                               dma_addr_t iova)
1288 {
1289         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1290         struct arm_smmu_device *smmu = smmu_domain->smmu;
1291         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1292         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1293         struct device *dev = smmu->dev;
1294         void __iomem *cb_base;
1295         u32 tmp;
1296         u64 phys;
1297         unsigned long va;
1298
1299         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
1300
1301         /* ATS1 registers can only be written atomically */
1302         va = iova & ~0xfffUL;
1303         if (smmu->version == ARM_SMMU_V2)
1304                 smmu_write_atomic_lq(va, cb_base + ARM_SMMU_CB_ATS1PR);
1305         else /* Register is only 32-bit in v1 */
1306                 writel_relaxed(va, cb_base + ARM_SMMU_CB_ATS1PR);
1307
1308         if (readl_poll_timeout_atomic(cb_base + ARM_SMMU_CB_ATSR, tmp,
1309                                       !(tmp & ATSR_ACTIVE), 5, 50)) {
1310                 dev_err(dev,
1311                         "iova to phys timed out on %pad. Falling back to software table walk.\n",
1312                         &iova);
1313                 return ops->iova_to_phys(ops, iova);
1314         }
1315
1316         phys = readq_relaxed(cb_base + ARM_SMMU_CB_PAR);
1317         if (phys & CB_PAR_F) {
1318                 dev_err(dev, "translation fault!\n");
1319                 dev_err(dev, "PAR = 0x%llx\n", phys);
1320                 return 0;
1321         }
1322
1323         return (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff);
1324 }
1325
1326 static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
1327                                         dma_addr_t iova)
1328 {
1329         phys_addr_t ret;
1330         unsigned long flags;
1331         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1332         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1333
1334         if (!ops)
1335                 return 0;
1336
1337         spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1338         if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
1339                         smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1340                 ret = arm_smmu_iova_to_phys_hard(domain, iova);
1341         } else {
1342                 ret = ops->iova_to_phys(ops, iova);
1343         }
1344
1345         spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1346
1347         return ret;
1348 }
1349
1350 static bool arm_smmu_capable(enum iommu_cap cap)
1351 {
1352         switch (cap) {
1353         case IOMMU_CAP_CACHE_COHERENCY:
1354                 /*
1355                  * Return true here as the SMMU can always send out coherent
1356                  * requests.
1357                  */
1358                 return true;
1359         case IOMMU_CAP_INTR_REMAP:
1360                 return true; /* MSIs are just memory writes */
1361         case IOMMU_CAP_NOEXEC:
1362                 return true;
1363         default:
1364                 return false;
1365         }
1366 }
1367
1368 static int arm_smmu_match_node(struct device *dev, void *data)
1369 {
1370         return dev->of_node == data;
1371 }
1372
1373 static struct arm_smmu_device *arm_smmu_get_by_node(struct device_node *np)
1374 {
1375         struct device *dev = driver_find_device(&arm_smmu_driver.driver, NULL,
1376                                                 np, arm_smmu_match_node);
1377         put_device(dev);
1378         return dev ? dev_get_drvdata(dev) : NULL;
1379 }
1380
1381 static int arm_smmu_add_device(struct device *dev)
1382 {
1383         struct arm_smmu_device *smmu;
1384         struct arm_smmu_master_cfg *cfg;
1385         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1386         int i, ret;
1387
1388         if (using_legacy_binding) {
1389                 ret = arm_smmu_register_legacy_master(dev, &smmu);
1390                 fwspec = dev->iommu_fwspec;
1391                 if (ret)
1392                         goto out_free;
1393         } else if (fwspec) {
1394                 smmu = arm_smmu_get_by_node(to_of_node(fwspec->iommu_fwnode));
1395         } else {
1396                 return -ENODEV;
1397         }
1398
1399         ret = -EINVAL;
1400         for (i = 0; i < fwspec->num_ids; i++) {
1401                 u16 sid = fwspec->ids[i];
1402                 u16 mask = fwspec->ids[i] >> SMR_MASK_SHIFT;
1403
1404                 if (sid & ~smmu->streamid_mask) {
1405                         dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n",
1406                                 sid, smmu->streamid_mask);
1407                         goto out_free;
1408                 }
1409                 if (mask & ~smmu->smr_mask_mask) {
1410                         dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n",
1411                                 sid, smmu->smr_mask_mask);
1412                         goto out_free;
1413                 }
1414         }
1415
1416         ret = -ENOMEM;
1417         cfg = kzalloc(offsetof(struct arm_smmu_master_cfg, smendx[i]),
1418                       GFP_KERNEL);
1419         if (!cfg)
1420                 goto out_free;
1421
1422         cfg->smmu = smmu;
1423         fwspec->iommu_priv = cfg;
1424         while (i--)
1425                 cfg->smendx[i] = INVALID_SMENDX;
1426
1427         ret = arm_smmu_master_alloc_smes(dev);
1428         if (ret)
1429                 goto out_free;
1430
1431         return 0;
1432
1433 out_free:
1434         if (fwspec)
1435                 kfree(fwspec->iommu_priv);
1436         iommu_fwspec_free(dev);
1437         return ret;
1438 }
1439
1440 static void arm_smmu_remove_device(struct device *dev)
1441 {
1442         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1443
1444         if (!fwspec || fwspec->ops != &arm_smmu_ops)
1445                 return;
1446
1447         arm_smmu_master_free_smes(fwspec);
1448         iommu_group_remove_device(dev);
1449         kfree(fwspec->iommu_priv);
1450         iommu_fwspec_free(dev);
1451 }
1452
1453 static struct iommu_group *arm_smmu_device_group(struct device *dev)
1454 {
1455         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1456         struct arm_smmu_device *smmu = fwspec_smmu(fwspec);
1457         struct iommu_group *group = NULL;
1458         int i, idx;
1459
1460         for_each_cfg_sme(fwspec, i, idx) {
1461                 if (group && smmu->s2crs[idx].group &&
1462                     group != smmu->s2crs[idx].group)
1463                         return ERR_PTR(-EINVAL);
1464
1465                 group = smmu->s2crs[idx].group;
1466         }
1467
1468         if (group)
1469                 return group;
1470
1471         if (dev_is_pci(dev))
1472                 group = pci_device_group(dev);
1473         else
1474                 group = generic_device_group(dev);
1475
1476         return group;
1477 }
1478
1479 static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
1480                                     enum iommu_attr attr, void *data)
1481 {
1482         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1483
1484         switch (attr) {
1485         case DOMAIN_ATTR_NESTING:
1486                 *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
1487                 return 0;
1488         default:
1489                 return -ENODEV;
1490         }
1491 }
1492
1493 static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
1494                                     enum iommu_attr attr, void *data)
1495 {
1496         int ret = 0;
1497         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1498
1499         mutex_lock(&smmu_domain->init_mutex);
1500
1501         switch (attr) {
1502         case DOMAIN_ATTR_NESTING:
1503                 if (smmu_domain->smmu) {
1504                         ret = -EPERM;
1505                         goto out_unlock;
1506                 }
1507
1508                 if (*(int *)data)
1509                         smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
1510                 else
1511                         smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1512
1513                 break;
1514         default:
1515                 ret = -ENODEV;
1516         }
1517
1518 out_unlock:
1519         mutex_unlock(&smmu_domain->init_mutex);
1520         return ret;
1521 }
1522
1523 static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args)
1524 {
1525         u32 fwid = 0;
1526
1527         if (args->args_count > 0)
1528                 fwid |= (u16)args->args[0];
1529
1530         if (args->args_count > 1)
1531                 fwid |= (u16)args->args[1] << SMR_MASK_SHIFT;
1532
1533         return iommu_fwspec_add_ids(dev, &fwid, 1);
1534 }
1535
1536 static struct iommu_ops arm_smmu_ops = {
1537         .capable                = arm_smmu_capable,
1538         .domain_alloc           = arm_smmu_domain_alloc,
1539         .domain_free            = arm_smmu_domain_free,
1540         .attach_dev             = arm_smmu_attach_dev,
1541         .map                    = arm_smmu_map,
1542         .unmap                  = arm_smmu_unmap,
1543         .map_sg                 = default_iommu_map_sg,
1544         .iova_to_phys           = arm_smmu_iova_to_phys,
1545         .add_device             = arm_smmu_add_device,
1546         .remove_device          = arm_smmu_remove_device,
1547         .device_group           = arm_smmu_device_group,
1548         .domain_get_attr        = arm_smmu_domain_get_attr,
1549         .domain_set_attr        = arm_smmu_domain_set_attr,
1550         .of_xlate               = arm_smmu_of_xlate,
1551         .pgsize_bitmap          = -1UL, /* Restricted during device attach */
1552 };
1553
1554 static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
1555 {
1556         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1557         void __iomem *cb_base;
1558         int i;
1559         u32 reg, major;
1560
1561         /* clear global FSR */
1562         reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
1563         writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
1564
1565         /*
1566          * Reset stream mapping groups: Initial values mark all SMRn as
1567          * invalid and all S2CRn as bypass unless overridden.
1568          */
1569         for (i = 0; i < smmu->num_mapping_groups; ++i)
1570                 arm_smmu_write_sme(smmu, i);
1571
1572         /*
1573          * Before clearing ARM_MMU500_ACTLR_CPRE, need to
1574          * clear CACHE_LOCK bit of ACR first. And, CACHE_LOCK
1575          * bit is only present in MMU-500r2 onwards.
1576          */
1577         reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID7);
1578         major = (reg >> ID7_MAJOR_SHIFT) & ID7_MAJOR_MASK;
1579         if ((smmu->model == ARM_MMU500) && (major >= 2)) {
1580                 reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_sACR);
1581                 reg &= ~ARM_MMU500_ACR_CACHE_LOCK;
1582                 writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_sACR);
1583         }
1584
1585         /* Make sure all context banks are disabled and clear CB_FSR  */
1586         for (i = 0; i < smmu->num_context_banks; ++i) {
1587                 cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, i);
1588                 writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
1589                 writel_relaxed(FSR_FAULT, cb_base + ARM_SMMU_CB_FSR);
1590                 /*
1591                  * Disable MMU-500's not-particularly-beneficial next-page
1592                  * prefetcher for the sake of errata #841119 and #826419.
1593                  */
1594                 if (smmu->model == ARM_MMU500) {
1595                         reg = readl_relaxed(cb_base + ARM_SMMU_CB_ACTLR);
1596                         reg &= ~ARM_MMU500_ACTLR_CPRE;
1597                         writel_relaxed(reg, cb_base + ARM_SMMU_CB_ACTLR);
1598                 }
1599         }
1600
1601         /* Invalidate the TLB, just in case */
1602         writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLH);
1603         writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLNSNH);
1604
1605         reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
1606
1607         /* Enable fault reporting */
1608         reg |= (sCR0_GFRE | sCR0_GFIE | sCR0_GCFGFRE | sCR0_GCFGFIE);
1609
1610         /* Disable TLB broadcasting. */
1611         reg |= (sCR0_VMIDPNE | sCR0_PTM);
1612
1613         /* Enable client access, handling unmatched streams as appropriate */
1614         reg &= ~sCR0_CLIENTPD;
1615         if (disable_bypass)
1616                 reg |= sCR0_USFCFG;
1617         else
1618                 reg &= ~sCR0_USFCFG;
1619
1620         /* Disable forced broadcasting */
1621         reg &= ~sCR0_FB;
1622
1623         /* Don't upgrade barriers */
1624         reg &= ~(sCR0_BSU_MASK << sCR0_BSU_SHIFT);
1625
1626         if (smmu->features & ARM_SMMU_FEAT_VMID16)
1627                 reg |= sCR0_VMID16EN;
1628
1629         /* Push the button */
1630         __arm_smmu_tlb_sync(smmu);
1631         writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
1632 }
1633
1634 static int arm_smmu_id_size_to_bits(int size)
1635 {
1636         switch (size) {
1637         case 0:
1638                 return 32;
1639         case 1:
1640                 return 36;
1641         case 2:
1642                 return 40;
1643         case 3:
1644                 return 42;
1645         case 4:
1646                 return 44;
1647         case 5:
1648         default:
1649                 return 48;
1650         }
1651 }
1652
1653 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1654 {
1655         unsigned long size;
1656         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1657         u32 id;
1658         bool cttw_dt, cttw_reg;
1659         int i;
1660
1661         dev_notice(smmu->dev, "probing hardware configuration...\n");
1662         dev_notice(smmu->dev, "SMMUv%d with:\n",
1663                         smmu->version == ARM_SMMU_V2 ? 2 : 1);
1664
1665         /* ID0 */
1666         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID0);
1667
1668         /* Restrict available stages based on module parameter */
1669         if (force_stage == 1)
1670                 id &= ~(ID0_S2TS | ID0_NTS);
1671         else if (force_stage == 2)
1672                 id &= ~(ID0_S1TS | ID0_NTS);
1673
1674         if (id & ID0_S1TS) {
1675                 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
1676                 dev_notice(smmu->dev, "\tstage 1 translation\n");
1677         }
1678
1679         if (id & ID0_S2TS) {
1680                 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
1681                 dev_notice(smmu->dev, "\tstage 2 translation\n");
1682         }
1683
1684         if (id & ID0_NTS) {
1685                 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;
1686                 dev_notice(smmu->dev, "\tnested translation\n");
1687         }
1688
1689         if (!(smmu->features &
1690                 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) {
1691                 dev_err(smmu->dev, "\tno translation support!\n");
1692                 return -ENODEV;
1693         }
1694
1695         if ((id & ID0_S1TS) &&
1696                 ((smmu->version < ARM_SMMU_V2) || !(id & ID0_ATOSNS))) {
1697                 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;
1698                 dev_notice(smmu->dev, "\taddress translation ops\n");
1699         }
1700
1701         /*
1702          * In order for DMA API calls to work properly, we must defer to what
1703          * the DT says about coherency, regardless of what the hardware claims.
1704          * Fortunately, this also opens up a workaround for systems where the
1705          * ID register value has ended up configured incorrectly.
1706          */
1707         cttw_dt = of_dma_is_coherent(smmu->dev->of_node);
1708         cttw_reg = !!(id & ID0_CTTW);
1709         if (cttw_dt)
1710                 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
1711         if (cttw_dt || cttw_reg)
1712                 dev_notice(smmu->dev, "\t%scoherent table walk\n",
1713                            cttw_dt ? "" : "non-");
1714         if (cttw_dt != cttw_reg)
1715                 dev_notice(smmu->dev,
1716                            "\t(IDR0.CTTW overridden by dma-coherent property)\n");
1717
1718         /* Max. number of entries we have for stream matching/indexing */
1719         size = 1 << ((id >> ID0_NUMSIDB_SHIFT) & ID0_NUMSIDB_MASK);
1720         smmu->streamid_mask = size - 1;
1721         if (id & ID0_SMS) {
1722                 u32 smr;
1723
1724                 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
1725                 size = (id >> ID0_NUMSMRG_SHIFT) & ID0_NUMSMRG_MASK;
1726                 if (size == 0) {
1727                         dev_err(smmu->dev,
1728                                 "stream-matching supported, but no SMRs present!\n");
1729                         return -ENODEV;
1730                 }
1731
1732                 /*
1733                  * SMR.ID bits may not be preserved if the corresponding MASK
1734                  * bits are set, so check each one separately. We can reject
1735                  * masters later if they try to claim IDs outside these masks.
1736                  */
1737                 smr = smmu->streamid_mask << SMR_ID_SHIFT;
1738                 writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0));
1739                 smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0));
1740                 smmu->streamid_mask = smr >> SMR_ID_SHIFT;
1741
1742                 smr = smmu->streamid_mask << SMR_MASK_SHIFT;
1743                 writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0));
1744                 smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0));
1745                 smmu->smr_mask_mask = smr >> SMR_MASK_SHIFT;
1746
1747                 /* Zero-initialised to mark as invalid */
1748                 smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs),
1749                                           GFP_KERNEL);
1750                 if (!smmu->smrs)
1751                         return -ENOMEM;
1752
1753                 dev_notice(smmu->dev,
1754                            "\tstream matching with %lu register groups, mask 0x%x",
1755                            size, smmu->smr_mask_mask);
1756         }
1757         /* s2cr->type == 0 means translation, so initialise explicitly */
1758         smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs),
1759                                          GFP_KERNEL);
1760         if (!smmu->s2crs)
1761                 return -ENOMEM;
1762         for (i = 0; i < size; i++)
1763                 smmu->s2crs[i] = s2cr_init_val;
1764
1765         smmu->num_mapping_groups = size;
1766         mutex_init(&smmu->stream_map_mutex);
1767
1768         if (smmu->version < ARM_SMMU_V2 || !(id & ID0_PTFS_NO_AARCH32)) {
1769                 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L;
1770                 if (!(id & ID0_PTFS_NO_AARCH32S))
1771                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S;
1772         }
1773
1774         /* ID1 */
1775         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID1);
1776         smmu->pgshift = (id & ID1_PAGESIZE) ? 16 : 12;
1777
1778         /* Check for size mismatch of SMMU address space from mapped region */
1779         size = 1 << (((id >> ID1_NUMPAGENDXB_SHIFT) & ID1_NUMPAGENDXB_MASK) + 1);
1780         size *= 2 << smmu->pgshift;
1781         if (smmu->size != size)
1782                 dev_warn(smmu->dev,
1783                         "SMMU address space size (0x%lx) differs from mapped region size (0x%lx)!\n",
1784                         size, smmu->size);
1785
1786         smmu->num_s2_context_banks = (id >> ID1_NUMS2CB_SHIFT) & ID1_NUMS2CB_MASK;
1787         smmu->num_context_banks = (id >> ID1_NUMCB_SHIFT) & ID1_NUMCB_MASK;
1788         if (smmu->num_s2_context_banks > smmu->num_context_banks) {
1789                 dev_err(smmu->dev, "impossible number of S2 context banks!\n");
1790                 return -ENODEV;
1791         }
1792         dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",
1793                    smmu->num_context_banks, smmu->num_s2_context_banks);
1794         /*
1795          * Cavium CN88xx erratum #27704.
1796          * Ensure ASID and VMID allocation is unique across all SMMUs in
1797          * the system.
1798          */
1799         if (smmu->model == CAVIUM_SMMUV2) {
1800                 smmu->cavium_id_base =
1801                         atomic_add_return(smmu->num_context_banks,
1802                                           &cavium_smmu_context_count);
1803                 smmu->cavium_id_base -= smmu->num_context_banks;
1804         }
1805
1806         /* ID2 */
1807         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID2);
1808         size = arm_smmu_id_size_to_bits((id >> ID2_IAS_SHIFT) & ID2_IAS_MASK);
1809         smmu->ipa_size = size;
1810
1811         /* The output mask is also applied for bypass */
1812         size = arm_smmu_id_size_to_bits((id >> ID2_OAS_SHIFT) & ID2_OAS_MASK);
1813         smmu->pa_size = size;
1814
1815         if (id & ID2_VMID16)
1816                 smmu->features |= ARM_SMMU_FEAT_VMID16;
1817
1818         /*
1819          * What the page table walker can address actually depends on which
1820          * descriptor format is in use, but since a) we don't know that yet,
1821          * and b) it can vary per context bank, this will have to do...
1822          */
1823         if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))
1824                 dev_warn(smmu->dev,
1825                          "failed to set DMA mask for table walker\n");
1826
1827         if (smmu->version < ARM_SMMU_V2) {
1828                 smmu->va_size = smmu->ipa_size;
1829                 if (smmu->version == ARM_SMMU_V1_64K)
1830                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1831         } else {
1832                 size = (id >> ID2_UBS_SHIFT) & ID2_UBS_MASK;
1833                 smmu->va_size = arm_smmu_id_size_to_bits(size);
1834                 if (id & ID2_PTFS_4K)
1835                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K;
1836                 if (id & ID2_PTFS_16K)
1837                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K;
1838                 if (id & ID2_PTFS_64K)
1839                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1840         }
1841
1842         /* Now we've corralled the various formats, what'll it do? */
1843         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S)
1844                 smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M;
1845         if (smmu->features &
1846             (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K))
1847                 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
1848         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K)
1849                 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
1850         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K)
1851                 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
1852
1853         if (arm_smmu_ops.pgsize_bitmap == -1UL)
1854                 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
1855         else
1856                 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
1857         dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n",
1858                    smmu->pgsize_bitmap);
1859
1860
1861         if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
1862                 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
1863                            smmu->va_size, smmu->ipa_size);
1864
1865         if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)
1866                 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
1867                            smmu->ipa_size, smmu->pa_size);
1868
1869         return 0;
1870 }
1871
1872 struct arm_smmu_match_data {
1873         enum arm_smmu_arch_version version;
1874         enum arm_smmu_implementation model;
1875 };
1876
1877 #define ARM_SMMU_MATCH_DATA(name, ver, imp)     \
1878 static struct arm_smmu_match_data name = { .version = ver, .model = imp }
1879
1880 ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU);
1881 ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU);
1882 ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU);
1883 ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500);
1884 ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2);
1885
1886 static const struct of_device_id arm_smmu_of_match[] = {
1887         { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 },
1888         { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 },
1889         { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 },
1890         { .compatible = "arm,mmu-401", .data = &arm_mmu401 },
1891         { .compatible = "arm,mmu-500", .data = &arm_mmu500 },
1892         { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 },
1893         { },
1894 };
1895 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
1896
1897 static int arm_smmu_device_dt_probe(struct platform_device *pdev)
1898 {
1899         const struct arm_smmu_match_data *data;
1900         struct resource *res;
1901         struct arm_smmu_device *smmu;
1902         struct device *dev = &pdev->dev;
1903         int num_irqs, i, err;
1904         bool legacy_binding;
1905
1906         legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL);
1907         if (legacy_binding && !using_generic_binding) {
1908                 if (!using_legacy_binding)
1909                         pr_notice("deprecated \"mmu-masters\" DT property in use; DMA API support unavailable\n");
1910                 using_legacy_binding = true;
1911         } else if (!legacy_binding && !using_legacy_binding) {
1912                 using_generic_binding = true;
1913         } else {
1914                 dev_err(dev, "not probing due to mismatched DT properties\n");
1915                 return -ENODEV;
1916         }
1917
1918         smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
1919         if (!smmu) {
1920                 dev_err(dev, "failed to allocate arm_smmu_device\n");
1921                 return -ENOMEM;
1922         }
1923         smmu->dev = dev;
1924
1925         data = of_device_get_match_data(dev);
1926         smmu->version = data->version;
1927         smmu->model = data->model;
1928
1929         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1930         smmu->base = devm_ioremap_resource(dev, res);
1931         if (IS_ERR(smmu->base))
1932                 return PTR_ERR(smmu->base);
1933         smmu->size = resource_size(res);
1934
1935         if (of_property_read_u32(dev->of_node, "#global-interrupts",
1936                                  &smmu->num_global_irqs)) {
1937                 dev_err(dev, "missing #global-interrupts property\n");
1938                 return -ENODEV;
1939         }
1940
1941         num_irqs = 0;
1942         while ((res = platform_get_resource(pdev, IORESOURCE_IRQ, num_irqs))) {
1943                 num_irqs++;
1944                 if (num_irqs > smmu->num_global_irqs)
1945                         smmu->num_context_irqs++;
1946         }
1947
1948         if (!smmu->num_context_irqs) {
1949                 dev_err(dev, "found %d interrupts but expected at least %d\n",
1950                         num_irqs, smmu->num_global_irqs + 1);
1951                 return -ENODEV;
1952         }
1953
1954         smmu->irqs = devm_kzalloc(dev, sizeof(*smmu->irqs) * num_irqs,
1955                                   GFP_KERNEL);
1956         if (!smmu->irqs) {
1957                 dev_err(dev, "failed to allocate %d irqs\n", num_irqs);
1958                 return -ENOMEM;
1959         }
1960
1961         for (i = 0; i < num_irqs; ++i) {
1962                 int irq = platform_get_irq(pdev, i);
1963
1964                 if (irq < 0) {
1965                         dev_err(dev, "failed to get irq index %d\n", i);
1966                         return -ENODEV;
1967                 }
1968                 smmu->irqs[i] = irq;
1969         }
1970
1971         err = arm_smmu_device_cfg_probe(smmu);
1972         if (err)
1973                 return err;
1974
1975         parse_driver_options(smmu);
1976
1977         if (smmu->version == ARM_SMMU_V2 &&
1978             smmu->num_context_banks != smmu->num_context_irqs) {
1979                 dev_err(dev,
1980                         "found only %d context interrupt(s) but %d required\n",
1981                         smmu->num_context_irqs, smmu->num_context_banks);
1982                 return -ENODEV;
1983         }
1984
1985         for (i = 0; i < smmu->num_global_irqs; ++i) {
1986                 err = devm_request_irq(smmu->dev, smmu->irqs[i],
1987                                        arm_smmu_global_fault,
1988                                        IRQF_SHARED,
1989                                        "arm-smmu global fault",
1990                                        smmu);
1991                 if (err) {
1992                         dev_err(dev, "failed to request global IRQ %d (%u)\n",
1993                                 i, smmu->irqs[i]);
1994                         return err;
1995                 }
1996         }
1997
1998         of_iommu_set_ops(dev->of_node, &arm_smmu_ops);
1999         platform_set_drvdata(pdev, smmu);
2000         arm_smmu_device_reset(smmu);
2001
2002         /* Oh, for a proper bus abstraction */
2003         if (!iommu_present(&platform_bus_type))
2004                 bus_set_iommu(&platform_bus_type, &arm_smmu_ops);
2005 #ifdef CONFIG_ARM_AMBA
2006         if (!iommu_present(&amba_bustype))
2007                 bus_set_iommu(&amba_bustype, &arm_smmu_ops);
2008 #endif
2009 #ifdef CONFIG_PCI
2010         if (!iommu_present(&pci_bus_type)) {
2011                 pci_request_acs();
2012                 bus_set_iommu(&pci_bus_type, &arm_smmu_ops);
2013         }
2014 #endif
2015         return 0;
2016 }
2017
2018 static int arm_smmu_device_remove(struct platform_device *pdev)
2019 {
2020         struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2021
2022         if (!smmu)
2023                 return -ENODEV;
2024
2025         if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
2026                 dev_err(&pdev->dev, "removing device with active domains!\n");
2027
2028         /* Turn the thing off */
2029         writel(sCR0_CLIENTPD, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
2030         return 0;
2031 }
2032
2033 static struct platform_driver arm_smmu_driver = {
2034         .driver = {
2035                 .name           = "arm-smmu",
2036                 .of_match_table = of_match_ptr(arm_smmu_of_match),
2037         },
2038         .probe  = arm_smmu_device_dt_probe,
2039         .remove = arm_smmu_device_remove,
2040 };
2041
2042 static int __init arm_smmu_init(void)
2043 {
2044         static bool registered;
2045         int ret = 0;
2046
2047         if (!registered) {
2048                 ret = platform_driver_register(&arm_smmu_driver);
2049                 registered = !ret;
2050         }
2051         return ret;
2052 }
2053
2054 static void __exit arm_smmu_exit(void)
2055 {
2056         return platform_driver_unregister(&arm_smmu_driver);
2057 }
2058
2059 subsys_initcall(arm_smmu_init);
2060 module_exit(arm_smmu_exit);
2061
2062 static int __init arm_smmu_of_init(struct device_node *np)
2063 {
2064         int ret = arm_smmu_init();
2065
2066         if (ret)
2067                 return ret;
2068
2069         if (!of_platform_device_create(np, NULL, platform_bus_type.dev_root))
2070                 return -ENODEV;
2071
2072         return 0;
2073 }
2074 IOMMU_OF_DECLARE(arm_smmuv1, "arm,smmu-v1", arm_smmu_of_init);
2075 IOMMU_OF_DECLARE(arm_smmuv2, "arm,smmu-v2", arm_smmu_of_init);
2076 IOMMU_OF_DECLARE(arm_mmu400, "arm,mmu-400", arm_smmu_of_init);
2077 IOMMU_OF_DECLARE(arm_mmu401, "arm,mmu-401", arm_smmu_of_init);
2078 IOMMU_OF_DECLARE(arm_mmu500, "arm,mmu-500", arm_smmu_of_init);
2079 IOMMU_OF_DECLARE(cavium_smmuv2, "cavium,smmu-v2", arm_smmu_of_init);
2080
2081 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations");
2082 MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>");
2083 MODULE_LICENSE("GPL v2");