drivers/iommu/arm-smmu.c

   1 /*
   2  * IOMMU API for ARM architected SMMU implementations.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11  * GNU General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public License
  14  * along with this program; if not, write to the Free Software
  15  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  16  *
  17  * Copyright (C) 2013 ARM Limited
  18  *
  19  * Author: Will Deacon <will.deacon@arm.com>
  20  *
  21  * This driver currently supports:
  22  *      - SMMUv1 and v2 implementations
  23  *      - Stream-matching and stream-indexing
  24  *      - v7/v8 long-descriptor format
  25  *      - Non-secure access to the SMMU
  26  *      - Context fault reporting
  27  */
  28
  29 #define pr_fmt(fmt) "arm-smmu: " fmt
  30
  31 #include <linux/delay.h>
  32 #include <linux/dma-iommu.h>
  33 #include <linux/dma-mapping.h>
  34 #include <linux/err.h>
  35 #include <linux/interrupt.h>
  36 #include <linux/io.h>
  37 #include <linux/io-64-nonatomic-hi-lo.h>
  38 #include <linux/iommu.h>
  39 #include <linux/iopoll.h>
  40 #include <linux/module.h>
  41 #include <linux/of.h>
  42 #include <linux/of_address.h>
  43 #include <linux/pci.h>
  44 #include <linux/platform_device.h>
  45 #include <linux/slab.h>
  46 #include <linux/spinlock.h>
  47
  48 #include <linux/amba/bus.h>
  49
  50 #include "io-pgtable.h"
  51
  52 /* Maximum number of stream IDs assigned to a single device */
  53 #define MAX_MASTER_STREAMIDS            128
  54
  55 /* Maximum number of context banks per SMMU */
  56 #define ARM_SMMU_MAX_CBS                128
  57
  58 /* Maximum number of mapping groups per SMMU */
  59 #define ARM_SMMU_MAX_SMRS               128
  60
  61 /* SMMU global address space */
  62 #define ARM_SMMU_GR0(smmu)              ((smmu)->base)
  63 #define ARM_SMMU_GR1(smmu)              ((smmu)->base + (1 << (smmu)->pgshift))
  64
  65 /*
  66  * SMMU global address space with conditional offset to access secure
  67  * aliases of non-secure registers (e.g. nsCR0: 0x400, nsGFSR: 0x448,
  68  * nsGFSYNR0: 0x450)
  69  */
  70 #define ARM_SMMU_GR0_NS(smmu)                                           \
  71         ((smmu)->base +                                                 \
  72                 ((smmu->options & ARM_SMMU_OPT_SECURE_CFG_ACCESS)       \
  73                         ? 0x400 : 0))
  74
  75 /*
  76  * Some 64-bit registers only make sense to write atomically, but in such
  77  * cases all the data relevant to AArch32 formats lies within the lower word,
  78  * therefore this actually makes more sense than it might first appear.
  79  */
  80 #ifdef CONFIG_64BIT
  81 #define smmu_write_atomic_lq            writeq_relaxed
  82 #else
  83 #define smmu_write_atomic_lq            writel_relaxed
  84 #endif
  85
  86 /* Configuration registers */
  87 #define ARM_SMMU_GR0_sCR0               0x0
  88 #define sCR0_CLIENTPD                   (1 << 0)
  89 #define sCR0_GFRE                       (1 << 1)
  90 #define sCR0_GFIE                       (1 << 2)
  91 #define sCR0_GCFGFRE                    (1 << 4)
  92 #define sCR0_GCFGFIE                    (1 << 5)
  93 #define sCR0_USFCFG                     (1 << 10)
  94 #define sCR0_VMIDPNE                    (1 << 11)
  95 #define sCR0_PTM                        (1 << 12)
  96 #define sCR0_FB                         (1 << 13)
  97 #define sCR0_VMID16EN                   (1 << 31)
  98 #define sCR0_BSU_SHIFT                  14
  99 #define sCR0_BSU_MASK                   0x3
 100
 101 /* Auxiliary Configuration register */
 102 #define ARM_SMMU_GR0_sACR               0x10
 103
 104 /* Identification registers */
 105 #define ARM_SMMU_GR0_ID0                0x20
 106 #define ARM_SMMU_GR0_ID1                0x24
 107 #define ARM_SMMU_GR0_ID2                0x28
 108 #define ARM_SMMU_GR0_ID3                0x2c
 109 #define ARM_SMMU_GR0_ID4                0x30
 110 #define ARM_SMMU_GR0_ID5                0x34
 111 #define ARM_SMMU_GR0_ID6                0x38
 112 #define ARM_SMMU_GR0_ID7                0x3c
 113 #define ARM_SMMU_GR0_sGFSR              0x48
 114 #define ARM_SMMU_GR0_sGFSYNR0           0x50
 115 #define ARM_SMMU_GR0_sGFSYNR1           0x54
 116 #define ARM_SMMU_GR0_sGFSYNR2           0x58
 117
 118 #define ID0_S1TS                        (1 << 30)
 119 #define ID0_S2TS                        (1 << 29)
 120 #define ID0_NTS                         (1 << 28)
 121 #define ID0_SMS                         (1 << 27)
 122 #define ID0_ATOSNS                      (1 << 26)
 123 #define ID0_PTFS_NO_AARCH32             (1 << 25)
 124 #define ID0_PTFS_NO_AARCH32S            (1 << 24)
 125 #define ID0_CTTW                        (1 << 14)
 126 #define ID0_NUMIRPT_SHIFT               16
 127 #define ID0_NUMIRPT_MASK                0xff
 128 #define ID0_NUMSIDB_SHIFT               9
 129 #define ID0_NUMSIDB_MASK                0xf
 130 #define ID0_NUMSMRG_SHIFT               0
 131 #define ID0_NUMSMRG_MASK                0xff
 132
 133 #define ID1_PAGESIZE                    (1 << 31)
 134 #define ID1_NUMPAGENDXB_SHIFT           28
 135 #define ID1_NUMPAGENDXB_MASK            7
 136 #define ID1_NUMS2CB_SHIFT               16
 137 #define ID1_NUMS2CB_MASK                0xff
 138 #define ID1_NUMCB_SHIFT                 0
 139 #define ID1_NUMCB_MASK                  0xff
 140
 141 #define ID2_OAS_SHIFT                   4
 142 #define ID2_OAS_MASK                    0xf
 143 #define ID2_IAS_SHIFT                   0
 144 #define ID2_IAS_MASK                    0xf
 145 #define ID2_UBS_SHIFT                   8
 146 #define ID2_UBS_MASK                    0xf
 147 #define ID2_PTFS_4K                     (1 << 12)
 148 #define ID2_PTFS_16K                    (1 << 13)
 149 #define ID2_PTFS_64K                    (1 << 14)
 150 #define ID2_VMID16                      (1 << 15)
 151
 152 #define ID7_MAJOR_SHIFT                 4
 153 #define ID7_MAJOR_MASK                  0xf
 154
 155 /* Global TLB invalidation */
 156 #define ARM_SMMU_GR0_TLBIVMID           0x64
 157 #define ARM_SMMU_GR0_TLBIALLNSNH        0x68
 158 #define ARM_SMMU_GR0_TLBIALLH           0x6c
 159 #define ARM_SMMU_GR0_sTLBGSYNC          0x70
 160 #define ARM_SMMU_GR0_sTLBGSTATUS        0x74
 161 #define sTLBGSTATUS_GSACTIVE            (1 << 0)
 162 #define TLB_LOOP_TIMEOUT                1000000 /* 1s! */
 163
 164 /* Stream mapping registers */
 165 #define ARM_SMMU_GR0_SMR(n)             (0x800 + ((n) << 2))
 166 #define SMR_VALID                       (1 << 31)
 167 #define SMR_MASK_SHIFT                  16
 168 #define SMR_MASK_MASK                   0x7fff
 169 #define SMR_ID_SHIFT                    0
 170 #define SMR_ID_MASK                     0x7fff
 171
 172 #define ARM_SMMU_GR0_S2CR(n)            (0xc00 + ((n) << 2))
 173 #define S2CR_CBNDX_SHIFT                0
 174 #define S2CR_CBNDX_MASK                 0xff
 175 #define S2CR_TYPE_SHIFT                 16
 176 #define S2CR_TYPE_MASK                  0x3
 177 #define S2CR_TYPE_TRANS                 (0 << S2CR_TYPE_SHIFT)
 178 #define S2CR_TYPE_BYPASS                (1 << S2CR_TYPE_SHIFT)
 179 #define S2CR_TYPE_FAULT                 (2 << S2CR_TYPE_SHIFT)
 180
 181 #define S2CR_PRIVCFG_SHIFT              24
 182 #define S2CR_PRIVCFG_UNPRIV             (2 << S2CR_PRIVCFG_SHIFT)
 183
 184 /* Context bank attribute registers */
 185 #define ARM_SMMU_GR1_CBAR(n)            (0x0 + ((n) << 2))
 186 #define CBAR_VMID_SHIFT                 0
 187 #define CBAR_VMID_MASK                  0xff
 188 #define CBAR_S1_BPSHCFG_SHIFT           8
 189 #define CBAR_S1_BPSHCFG_MASK            3
 190 #define CBAR_S1_BPSHCFG_NSH             3
 191 #define CBAR_S1_MEMATTR_SHIFT           12
 192 #define CBAR_S1_MEMATTR_MASK            0xf
 193 #define CBAR_S1_MEMATTR_WB              0xf
 194 #define CBAR_TYPE_SHIFT                 16
 195 #define CBAR_TYPE_MASK                  0x3
 196 #define CBAR_TYPE_S2_TRANS              (0 << CBAR_TYPE_SHIFT)
 197 #define CBAR_TYPE_S1_TRANS_S2_BYPASS    (1 << CBAR_TYPE_SHIFT)
 198 #define CBAR_TYPE_S1_TRANS_S2_FAULT     (2 << CBAR_TYPE_SHIFT)
 199 #define CBAR_TYPE_S1_TRANS_S2_TRANS     (3 << CBAR_TYPE_SHIFT)
 200 #define CBAR_IRPTNDX_SHIFT              24
 201 #define CBAR_IRPTNDX_MASK               0xff
 202
 203 #define ARM_SMMU_GR1_CBA2R(n)           (0x800 + ((n) << 2))
 204 #define CBA2R_RW64_32BIT                (0 << 0)
 205 #define CBA2R_RW64_64BIT                (1 << 0)
 206 #define CBA2R_VMID_SHIFT                16
 207 #define CBA2R_VMID_MASK                 0xffff
 208
 209 /* Translation context bank */
 210 #define ARM_SMMU_CB_BASE(smmu)          ((smmu)->base + ((smmu)->size >> 1))
 211 #define ARM_SMMU_CB(smmu, n)            ((n) * (1 << (smmu)->pgshift))
 212
 213 #define ARM_SMMU_CB_SCTLR               0x0
 214 #define ARM_SMMU_CB_ACTLR               0x4
 215 #define ARM_SMMU_CB_RESUME              0x8
 216 #define ARM_SMMU_CB_TTBCR2              0x10
 217 #define ARM_SMMU_CB_TTBR0               0x20
 218 #define ARM_SMMU_CB_TTBR1               0x28
 219 #define ARM_SMMU_CB_TTBCR               0x30
 220 #define ARM_SMMU_CB_S1_MAIR0            0x38
 221 #define ARM_SMMU_CB_S1_MAIR1            0x3c
 222 #define ARM_SMMU_CB_PAR                 0x50
 223 #define ARM_SMMU_CB_FSR                 0x58
 224 #define ARM_SMMU_CB_FAR                 0x60
 225 #define ARM_SMMU_CB_FSYNR0              0x68
 226 #define ARM_SMMU_CB_S1_TLBIVA           0x600
 227 #define ARM_SMMU_CB_S1_TLBIASID         0x610
 228 #define ARM_SMMU_CB_S1_TLBIVAL          0x620
 229 #define ARM_SMMU_CB_S2_TLBIIPAS2        0x630
 230 #define ARM_SMMU_CB_S2_TLBIIPAS2L       0x638
 231 #define ARM_SMMU_CB_ATS1PR              0x800
 232 #define ARM_SMMU_CB_ATSR                0x8f0
 233
 234 #define SCTLR_S1_ASIDPNE                (1 << 12)
 235 #define SCTLR_CFCFG                     (1 << 7)
 236 #define SCTLR_CFIE                      (1 << 6)
 237 #define SCTLR_CFRE                      (1 << 5)
 238 #define SCTLR_E                         (1 << 4)
 239 #define SCTLR_AFE                       (1 << 2)
 240 #define SCTLR_TRE                       (1 << 1)
 241 #define SCTLR_M                         (1 << 0)
 242 #define SCTLR_EAE_SBOP                  (SCTLR_AFE | SCTLR_TRE)
 243
 244 #define ARM_MMU500_ACTLR_CPRE           (1 << 1)
 245
 246 #define ARM_MMU500_ACR_CACHE_LOCK       (1 << 26)
 247
 248 #define CB_PAR_F                        (1 << 0)
 249
 250 #define ATSR_ACTIVE                     (1 << 0)
 251
 252 #define RESUME_RETRY                    (0 << 0)
 253 #define RESUME_TERMINATE                (1 << 0)
 254
 255 #define TTBCR2_SEP_SHIFT                15
 256 #define TTBCR2_SEP_UPSTREAM             (0x7 << TTBCR2_SEP_SHIFT)
 257
 258 #define TTBRn_ASID_SHIFT                48
 259
 260 #define FSR_MULTI                       (1 << 31)
 261 #define FSR_SS                          (1 << 30)
 262 #define FSR_UUT                         (1 << 8)
 263 #define FSR_ASF                         (1 << 7)
 264 #define FSR_TLBLKF                      (1 << 6)
 265 #define FSR_TLBMCF                      (1 << 5)
 266 #define FSR_EF                          (1 << 4)
 267 #define FSR_PF                          (1 << 3)
 268 #define FSR_AFF                         (1 << 2)
 269 #define FSR_TF                          (1 << 1)
 270
 271 #define FSR_IGN                         (FSR_AFF | FSR_ASF | \
 272                                          FSR_TLBMCF | FSR_TLBLKF)
 273 #define FSR_FAULT                       (FSR_MULTI | FSR_SS | FSR_UUT | \
 274                                          FSR_EF | FSR_PF | FSR_TF | FSR_IGN)
 275
 276 #define FSYNR0_WNR                      (1 << 4)
 277
 278 static int force_stage;
 279 module_param(force_stage, int, S_IRUGO);
 280 MODULE_PARM_DESC(force_stage,
 281         "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
 282 static bool disable_bypass;
 283 module_param(disable_bypass, bool, S_IRUGO);
 284 MODULE_PARM_DESC(disable_bypass,
 285         "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
 286
 287 enum arm_smmu_arch_version {
 288         ARM_SMMU_V1,
 289         ARM_SMMU_V1_64K,
 290         ARM_SMMU_V2,
 291 };
 292
 293 enum arm_smmu_implementation {
 294         GENERIC_SMMU,
 295         ARM_MMU500,
 296         CAVIUM_SMMUV2,
 297 };
 298
 299 struct arm_smmu_smr {
 300         u8                              idx;
 301         u16                             mask;
 302         u16                             id;
 303 };
 304
 305 struct arm_smmu_master_cfg {
 306         int                             num_streamids;
 307         u16                             streamids[MAX_MASTER_STREAMIDS];
 308         struct arm_smmu_smr             *smrs;
 309 };
 310
 311 struct arm_smmu_master {
 312         struct device_node              *of_node;
 313         struct rb_node                  node;
 314         struct arm_smmu_master_cfg      cfg;
 315 };
 316
 317 struct arm_smmu_device {
 318         struct device                   *dev;
 319
 320         void __iomem                    *base;
 321         unsigned long                   size;
 322         unsigned long                   pgshift;
 323
 324 #define ARM_SMMU_FEAT_COHERENT_WALK     (1 << 0)
 325 #define ARM_SMMU_FEAT_STREAM_MATCH      (1 << 1)
 326 #define ARM_SMMU_FEAT_TRANS_S1          (1 << 2)
 327 #define ARM_SMMU_FEAT_TRANS_S2          (1 << 3)
 328 #define ARM_SMMU_FEAT_TRANS_NESTED      (1 << 4)
 329 #define ARM_SMMU_FEAT_TRANS_OPS         (1 << 5)
 330 #define ARM_SMMU_FEAT_VMID16            (1 << 6)
 331 #define ARM_SMMU_FEAT_FMT_AARCH64_4K    (1 << 7)
 332 #define ARM_SMMU_FEAT_FMT_AARCH64_16K   (1 << 8)
 333 #define ARM_SMMU_FEAT_FMT_AARCH64_64K   (1 << 9)
 334 #define ARM_SMMU_FEAT_FMT_AARCH32_L     (1 << 10)
 335 #define ARM_SMMU_FEAT_FMT_AARCH32_S     (1 << 11)
 336         u32                             features;
 337
 338 #define ARM_SMMU_OPT_SECURE_CFG_ACCESS (1 << 0)
 339         u32                             options;
 340         enum arm_smmu_arch_version      version;
 341         enum arm_smmu_implementation    model;
 342
 343         u32                             num_context_banks;
 344         u32                             num_s2_context_banks;
 345         DECLARE_BITMAP(context_map, ARM_SMMU_MAX_CBS);
 346         atomic_t                        irptndx;
 347
 348         u32                             num_mapping_groups;
 349         DECLARE_BITMAP(smr_map, ARM_SMMU_MAX_SMRS);
 350
 351         unsigned long                   va_size;
 352         unsigned long                   ipa_size;
 353         unsigned long                   pa_size;
 354         unsigned long                   pgsize_bitmap;
 355
 356         u32                             num_global_irqs;
 357         u32                             num_context_irqs;
 358         unsigned int                    *irqs;
 359
 360         struct list_head                list;
 361         struct rb_root                  masters;
 362
 363         u32                             cavium_id_base; /* Specific to Cavium */
 364 };
 365
 366 enum arm_smmu_context_fmt {
 367         ARM_SMMU_CTX_FMT_NONE,
 368         ARM_SMMU_CTX_FMT_AARCH64,
 369         ARM_SMMU_CTX_FMT_AARCH32_L,
 370         ARM_SMMU_CTX_FMT_AARCH32_S,
 371 };
 372
 373 struct arm_smmu_cfg {
 374         u8                              cbndx;
 375         u8                              irptndx;
 376         u32                             cbar;
 377         enum arm_smmu_context_fmt       fmt;
 378 };
 379 #define INVALID_IRPTNDX                 0xff
 380
 381 #define ARM_SMMU_CB_ASID(smmu, cfg) ((u16)(smmu)->cavium_id_base + (cfg)->cbndx)
 382 #define ARM_SMMU_CB_VMID(smmu, cfg) ((u16)(smmu)->cavium_id_base + (cfg)->cbndx + 1)
 383
 384 enum arm_smmu_domain_stage {
 385         ARM_SMMU_DOMAIN_S1 = 0,
 386         ARM_SMMU_DOMAIN_S2,
 387         ARM_SMMU_DOMAIN_NESTED,
 388 };
 389
 390 struct arm_smmu_domain {
 391         struct arm_smmu_device          *smmu;
 392         struct io_pgtable_ops           *pgtbl_ops;
 393         spinlock_t                      pgtbl_lock;
 394         struct arm_smmu_cfg             cfg;
 395         enum arm_smmu_domain_stage      stage;
 396         struct mutex                    init_mutex; /* Protects smmu pointer */
 397         struct iommu_domain             domain;
 398 };
 399
 400 struct arm_smmu_phandle_args {
 401         struct device_node *np;
 402         int args_count;
 403         uint32_t args[MAX_MASTER_STREAMIDS];
 404 };
 405
 406 static DEFINE_SPINLOCK(arm_smmu_devices_lock);
 407 static LIST_HEAD(arm_smmu_devices);
 408
 409 struct arm_smmu_option_prop {
 410         u32 opt;
 411         const char *prop;
 412 };
 413
 414 static atomic_t cavium_smmu_context_count = ATOMIC_INIT(0);
 415
 416 static struct arm_smmu_option_prop arm_smmu_options[] = {
 417         { ARM_SMMU_OPT_SECURE_CFG_ACCESS, "calxeda,smmu-secure-config-access" },
 418         { 0, NULL},
 419 };
 420
 421 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
 422 {
 423         return container_of(dom, struct arm_smmu_domain, domain);
 424 }
 425
 426 static void parse_driver_options(struct arm_smmu_device *smmu)
 427 {
 428         int i = 0;
 429
 430         do {
 431                 if (of_property_read_bool(smmu->dev->of_node,
 432                                                 arm_smmu_options[i].prop)) {
 433                         smmu->options |= arm_smmu_options[i].opt;
 434                         dev_notice(smmu->dev, "option %s\n",
 435                                 arm_smmu_options[i].prop);
 436                 }
 437         } while (arm_smmu_options[++i].opt);
 438 }
 439
 440 static struct device_node *dev_get_dev_node(struct device *dev)
 441 {
 442         if (dev_is_pci(dev)) {
 443                 struct pci_bus *bus = to_pci_dev(dev)->bus;
 444
 445                 while (!pci_is_root_bus(bus))
 446                         bus = bus->parent;
 447                 return bus->bridge->parent->of_node;
 448         }
 449
 450         return dev->of_node;
 451 }
 452
 453 static struct arm_smmu_master *find_smmu_master(struct arm_smmu_device *smmu,
 454                                                 struct device_node *dev_node)
 455 {
 456         struct rb_node *node = smmu->masters.rb_node;
 457
 458         while (node) {
 459                 struct arm_smmu_master *master;
 460
 461                 master = container_of(node, struct arm_smmu_master, node);
 462
 463                 if (dev_node < master->of_node)
 464                         node = node->rb_left;
 465                 else if (dev_node > master->of_node)
 466                         node = node->rb_right;
 467                 else
 468                         return master;
 469         }
 470
 471         return NULL;
 472 }
 473
 474 static struct arm_smmu_master_cfg *
 475 find_smmu_master_cfg(struct device *dev)
 476 {
 477         struct arm_smmu_master_cfg *cfg = NULL;
 478         struct iommu_group *group = iommu_group_get(dev);
 479
 480         if (group) {
 481                 cfg = iommu_group_get_iommudata(group);
 482                 iommu_group_put(group);
 483         }
 484
 485         return cfg;
 486 }
 487
 488 static int insert_smmu_master(struct arm_smmu_device *smmu,
 489                               struct arm_smmu_master *master)
 490 {
 491         struct rb_node **new, *parent;
 492
 493         new = &smmu->masters.rb_node;
 494         parent = NULL;
 495         while (*new) {
 496                 struct arm_smmu_master *this
 497                         = container_of(*new, struct arm_smmu_master, node);
 498
 499                 parent = *new;
 500                 if (master->of_node < this->of_node)
 501                         new = &((*new)->rb_left);
 502                 else if (master->of_node > this->of_node)
 503                         new = &((*new)->rb_right);
 504                 else
 505                         return -EEXIST;
 506         }
 507
 508         rb_link_node(&master->node, parent, new);
 509         rb_insert_color(&master->node, &smmu->masters);
 510         return 0;
 511 }
 512
 513 static int register_smmu_master(struct arm_smmu_device *smmu,
 514                                 struct device *dev,
 515                                 struct arm_smmu_phandle_args *masterspec)
 516 {
 517         int i;
 518         struct arm_smmu_master *master;
 519
 520         master = find_smmu_master(smmu, masterspec->np);
 521         if (master) {
 522                 dev_err(dev,
 523                         "rejecting multiple registrations for master device %s\n",
 524                         masterspec->np->name);
 525                 return -EBUSY;
 526         }
 527
 528         if (masterspec->args_count > MAX_MASTER_STREAMIDS) {
 529                 dev_err(dev,
 530                         "reached maximum number (%d) of stream IDs for master device %s\n",
 531                         MAX_MASTER_STREAMIDS, masterspec->np->name);
 532                 return -ENOSPC;
 533         }
 534
 535         master = devm_kzalloc(dev, sizeof(*master), GFP_KERNEL);
 536         if (!master)
 537                 return -ENOMEM;
 538
 539         master->of_node                 = masterspec->np;
 540         master->cfg.num_streamids       = masterspec->args_count;
 541
 542         for (i = 0; i < master->cfg.num_streamids; ++i) {
 543                 u16 streamid = masterspec->args[i];
 544
 545                 if (!(smmu->features & ARM_SMMU_FEAT_STREAM_MATCH) &&
 546                      (streamid >= smmu->num_mapping_groups)) {
 547                         dev_err(dev,
 548                                 "stream ID for master device %s greater than maximum allowed (%d)\n",
 549                                 masterspec->np->name, smmu->num_mapping_groups);
 550                         return -ERANGE;
 551                 }
 552                 master->cfg.streamids[i] = streamid;
 553         }
 554         return insert_smmu_master(smmu, master);
 555 }
 556
 557 static struct arm_smmu_device *find_smmu_for_device(struct device *dev)
 558 {
 559         struct arm_smmu_device *smmu;
 560         struct arm_smmu_master *master = NULL;
 561         struct device_node *dev_node = dev_get_dev_node(dev);
 562
 563         spin_lock(&arm_smmu_devices_lock);
 564         list_for_each_entry(smmu, &arm_smmu_devices, list) {
 565                 master = find_smmu_master(smmu, dev_node);
 566                 if (master)
 567                         break;
 568         }
 569         spin_unlock(&arm_smmu_devices_lock);
 570
 571         return master ? smmu : NULL;
 572 }
 573
 574 static int __arm_smmu_alloc_bitmap(unsigned long *map, int start, int end)
 575 {
 576         int idx;
 577
 578         do {
 579                 idx = find_next_zero_bit(map, end, start);
 580                 if (idx == end)
 581                         return -ENOSPC;
 582         } while (test_and_set_bit(idx, map));
 583
 584         return idx;
 585 }
 586
 587 static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
 588 {
 589         clear_bit(idx, map);
 590 }
 591
 592 /* Wait for any pending TLB invalidations to complete */
 593 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu)
 594 {
 595         int count = 0;
 596         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
 597
 598         writel_relaxed(0, gr0_base + ARM_SMMU_GR0_sTLBGSYNC);
 599         while (readl_relaxed(gr0_base + ARM_SMMU_GR0_sTLBGSTATUS)
 600                & sTLBGSTATUS_GSACTIVE) {
 601                 cpu_relax();
 602                 if (++count == TLB_LOOP_TIMEOUT) {
 603                         dev_err_ratelimited(smmu->dev,
 604                         "TLB sync timed out -- SMMU may be deadlocked\n");
 605                         return;
 606                 }
 607                 udelay(1);
 608         }
 609 }
 610
 611 static void arm_smmu_tlb_sync(void *cookie)
 612 {
 613         struct arm_smmu_domain *smmu_domain = cookie;
 614         __arm_smmu_tlb_sync(smmu_domain->smmu);
 615 }
 616
 617 static void arm_smmu_tlb_inv_context(void *cookie)
 618 {
 619         struct arm_smmu_domain *smmu_domain = cookie;
 620         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 621         struct arm_smmu_device *smmu = smmu_domain->smmu;
 622         bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 623         void __iomem *base;
 624
 625         if (stage1) {
 626                 base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 627                 writel_relaxed(ARM_SMMU_CB_ASID(smmu, cfg),
 628                                base + ARM_SMMU_CB_S1_TLBIASID);
 629         } else {
 630                 base = ARM_SMMU_GR0(smmu);
 631                 writel_relaxed(ARM_SMMU_CB_VMID(smmu, cfg),
 632                                base + ARM_SMMU_GR0_TLBIVMID);
 633         }
 634
 635         __arm_smmu_tlb_sync(smmu);
 636 }
 637
 638 static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
 639                                           size_t granule, bool leaf, void *cookie)
 640 {
 641         struct arm_smmu_domain *smmu_domain = cookie;
 642         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 643         struct arm_smmu_device *smmu = smmu_domain->smmu;
 644         bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 645         void __iomem *reg;
 646
 647         if (stage1) {
 648                 reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 649                 reg += leaf ? ARM_SMMU_CB_S1_TLBIVAL : ARM_SMMU_CB_S1_TLBIVA;
 650
 651                 if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) {
 652                         iova &= ~12UL;
 653                         iova |= ARM_SMMU_CB_ASID(smmu, cfg);
 654                         do {
 655                                 writel_relaxed(iova, reg);
 656                                 iova += granule;
 657                         } while (size -= granule);
 658                 } else {
 659                         iova >>= 12;
 660                         iova |= (u64)ARM_SMMU_CB_ASID(smmu, cfg) << 48;
 661                         do {
 662                                 writeq_relaxed(iova, reg);
 663                                 iova += granule >> 12;
 664                         } while (size -= granule);
 665                 }
 666         } else if (smmu->version == ARM_SMMU_V2) {
 667                 reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 668                 reg += leaf ? ARM_SMMU_CB_S2_TLBIIPAS2L :
 669                               ARM_SMMU_CB_S2_TLBIIPAS2;
 670                 iova >>= 12;
 671                 do {
 672                         smmu_write_atomic_lq(iova, reg);
 673                         iova += granule >> 12;
 674                 } while (size -= granule);
 675         } else {
 676                 reg = ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_TLBIVMID;
 677                 writel_relaxed(ARM_SMMU_CB_VMID(smmu, cfg), reg);
 678         }
 679 }
 680
 681 static struct iommu_gather_ops arm_smmu_gather_ops = {
 682         .tlb_flush_all  = arm_smmu_tlb_inv_context,
 683         .tlb_add_flush  = arm_smmu_tlb_inv_range_nosync,
 684         .tlb_sync       = arm_smmu_tlb_sync,
 685 };
 686
 687 static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
 688 {
 689         u32 fsr, fsynr;
 690         unsigned long iova;
 691         struct iommu_domain *domain = dev;
 692         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 693         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 694         struct arm_smmu_device *smmu = smmu_domain->smmu;
 695         void __iomem *cb_base;
 696
 697         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 698         fsr = readl_relaxed(cb_base + ARM_SMMU_CB_FSR);
 699
 700         if (!(fsr & FSR_FAULT))
 701                 return IRQ_NONE;
 702
 703         fsynr = readl_relaxed(cb_base + ARM_SMMU_CB_FSYNR0);
 704         iova = readq_relaxed(cb_base + ARM_SMMU_CB_FAR);
 705
 706         dev_err_ratelimited(smmu->dev,
 707         "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cb=%d\n",
 708                             fsr, iova, fsynr, cfg->cbndx);
 709
 710         writel(fsr, cb_base + ARM_SMMU_CB_FSR);
 711         return IRQ_HANDLED;
 712 }
 713
 714 static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
 715 {
 716         u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
 717         struct arm_smmu_device *smmu = dev;
 718         void __iomem *gr0_base = ARM_SMMU_GR0_NS(smmu);
 719
 720         gfsr = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSR);
 721         gfsynr0 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR0);
 722         gfsynr1 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR1);
 723         gfsynr2 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR2);
 724
 725         if (!gfsr)
 726                 return IRQ_NONE;
 727
 728         dev_err_ratelimited(smmu->dev,
 729                 "Unexpected global fault, this could be serious\n");
 730         dev_err_ratelimited(smmu->dev,
 731                 "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
 732                 gfsr, gfsynr0, gfsynr1, gfsynr2);
 733
 734         writel(gfsr, gr0_base + ARM_SMMU_GR0_sGFSR);
 735         return IRQ_HANDLED;
 736 }
 737
 738 static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
 739                                        struct io_pgtable_cfg *pgtbl_cfg)
 740 {
 741         u32 reg;
 742         u64 reg64;
 743         bool stage1;
 744         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 745         struct arm_smmu_device *smmu = smmu_domain->smmu;
 746         void __iomem *cb_base, *gr1_base;
 747
 748         gr1_base = ARM_SMMU_GR1(smmu);
 749         stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 750         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 751
 752         if (smmu->version > ARM_SMMU_V1) {
 753                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
 754                         reg = CBA2R_RW64_64BIT;
 755                 else
 756                         reg = CBA2R_RW64_32BIT;
 757                 /* 16-bit VMIDs live in CBA2R */
 758                 if (smmu->features & ARM_SMMU_FEAT_VMID16)
 759                         reg |= ARM_SMMU_CB_VMID(smmu, cfg) << CBA2R_VMID_SHIFT;
 760
 761                 writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBA2R(cfg->cbndx));
 762         }
 763
 764         /* CBAR */
 765         reg = cfg->cbar;
 766         if (smmu->version < ARM_SMMU_V2)
 767                 reg |= cfg->irptndx << CBAR_IRPTNDX_SHIFT;
 768
 769         /*
 770          * Use the weakest shareability/memory types, so they are
 771          * overridden by the ttbcr/pte.
 772          */
 773         if (stage1) {
 774                 reg |= (CBAR_S1_BPSHCFG_NSH << CBAR_S1_BPSHCFG_SHIFT) |
 775                         (CBAR_S1_MEMATTR_WB << CBAR_S1_MEMATTR_SHIFT);
 776         } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) {
 777                 /* 8-bit VMIDs live in CBAR */
 778                 reg |= ARM_SMMU_CB_VMID(smmu, cfg) << CBAR_VMID_SHIFT;
 779         }
 780         writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBAR(cfg->cbndx));
 781
 782         /* TTBRs */
 783         if (stage1) {
 784                 reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
 785
 786                 reg64 |= ((u64)ARM_SMMU_CB_ASID(smmu, cfg)) << TTBRn_ASID_SHIFT;
 787                 writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0);
 788
 789                 reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[1];
 790                 reg64 |= ((u64)ARM_SMMU_CB_ASID(smmu, cfg)) << TTBRn_ASID_SHIFT;
 791                 writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR1);
 792         } else {
 793                 reg64 = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
 794                 writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0);
 795         }
 796
 797         /* TTBCR */
 798         if (stage1) {
 799                 reg = pgtbl_cfg->arm_lpae_s1_cfg.tcr;
 800                 writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR);
 801                 if (smmu->version > ARM_SMMU_V1) {
 802                         reg = pgtbl_cfg->arm_lpae_s1_cfg.tcr >> 32;
 803                         reg |= TTBCR2_SEP_UPSTREAM;
 804                         writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR2);
 805                 }
 806         } else {
 807                 reg = pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
 808                 writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR);
 809         }
 810
 811         /* MAIRs (stage-1 only) */
 812         if (stage1) {
 813                 reg = pgtbl_cfg->arm_lpae_s1_cfg.mair[0];
 814                 writel_relaxed(reg, cb_base + ARM_SMMU_CB_S1_MAIR0);
 815                 reg = pgtbl_cfg->arm_lpae_s1_cfg.mair[1];
 816                 writel_relaxed(reg, cb_base + ARM_SMMU_CB_S1_MAIR1);
 817         }
 818
 819         /* SCTLR */
 820         reg = SCTLR_CFIE | SCTLR_CFRE | SCTLR_M | SCTLR_EAE_SBOP;
 821         if (stage1)
 822                 reg |= SCTLR_S1_ASIDPNE;
 823 #ifdef __BIG_ENDIAN
 824         reg |= SCTLR_E;
 825 #endif
 826         writel_relaxed(reg, cb_base + ARM_SMMU_CB_SCTLR);
 827 }
 828
 829 static int arm_smmu_init_domain_context(struct iommu_domain *domain,
 830                                         struct arm_smmu_device *smmu)
 831 {
 832         int irq, start, ret = 0;
 833         unsigned long ias, oas;
 834         struct io_pgtable_ops *pgtbl_ops;
 835         struct io_pgtable_cfg pgtbl_cfg;
 836         enum io_pgtable_fmt fmt;
 837         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 838         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 839
 840         mutex_lock(&smmu_domain->init_mutex);
 841         if (smmu_domain->smmu)
 842                 goto out_unlock;
 843
 844         /* We're bypassing these SIDs, so don't allocate an actual context */
 845         if (domain->type == IOMMU_DOMAIN_DMA) {
 846                 smmu_domain->smmu = smmu;
 847                 goto out_unlock;
 848         }
 849
 850         /*
 851          * Mapping the requested stage onto what we support is surprisingly
 852          * complicated, mainly because the spec allows S1+S2 SMMUs without
 853          * support for nested translation. That means we end up with the
 854          * following table:
 855          *
 856          * Requested        Supported        Actual
 857          *     S1               N              S1
 858          *     S1             S1+S2            S1
 859          *     S1               S2             S2
 860          *     S1               S1             S1
 861          *     N                N              N
 862          *     N              S1+S2            S2
 863          *     N                S2             S2
 864          *     N                S1             S1
 865          *
 866          * Note that you can't actually request stage-2 mappings.
 867          */
 868         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
 869                 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
 870         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
 871                 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
 872
 873         /*
 874          * Choosing a suitable context format is even more fiddly. Until we
 875          * grow some way for the caller to express a preference, and/or move
 876          * the decision into the io-pgtable code where it arguably belongs,
 877          * just aim for the closest thing to the rest of the system, and hope
 878          * that the hardware isn't esoteric enough that we can't assume AArch64
 879          * support to be a superset of AArch32 support...
 880          */
 881         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L)
 882                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L;
 883         if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) &&
 884             (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K |
 885                                ARM_SMMU_FEAT_FMT_AARCH64_16K |
 886                                ARM_SMMU_FEAT_FMT_AARCH64_4K)))
 887                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64;
 888
 889         if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) {
 890                 ret = -EINVAL;
 891                 goto out_unlock;
 892         }
 893
 894         switch (smmu_domain->stage) {
 895         case ARM_SMMU_DOMAIN_S1:
 896                 cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS;
 897                 start = smmu->num_s2_context_banks;
 898                 ias = smmu->va_size;
 899                 oas = smmu->ipa_size;
 900                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
 901                         fmt = ARM_64_LPAE_S1;
 902                 } else {
 903                         fmt = ARM_32_LPAE_S1;
 904                         ias = min(ias, 32UL);
 905                         oas = min(oas, 40UL);
 906                 }
 907                 break;
 908         case ARM_SMMU_DOMAIN_NESTED:
 909                 /*
 910                  * We will likely want to change this if/when KVM gets
 911                  * involved.
 912                  */
 913         case ARM_SMMU_DOMAIN_S2:
 914                 cfg->cbar = CBAR_TYPE_S2_TRANS;
 915                 start = 0;
 916                 ias = smmu->ipa_size;
 917                 oas = smmu->pa_size;
 918                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
 919                         fmt = ARM_64_LPAE_S2;
 920                 } else {
 921                         fmt = ARM_32_LPAE_S2;
 922                         ias = min(ias, 40UL);
 923                         oas = min(oas, 40UL);
 924                 }
 925                 break;
 926         default:
 927                 ret = -EINVAL;
 928                 goto out_unlock;
 929         }
 930
 931         ret = __arm_smmu_alloc_bitmap(smmu->context_map, start,
 932                                       smmu->num_context_banks);
 933         if (ret < 0)
 934                 goto out_unlock;
 935
 936         cfg->cbndx = ret;
 937         if (smmu->version < ARM_SMMU_V2) {
 938                 cfg->irptndx = atomic_inc_return(&smmu->irptndx);
 939                 cfg->irptndx %= smmu->num_context_irqs;
 940         } else {
 941                 cfg->irptndx = cfg->cbndx;
 942         }
 943
 944         pgtbl_cfg = (struct io_pgtable_cfg) {
 945                 .pgsize_bitmap  = smmu->pgsize_bitmap,
 946                 .ias            = ias,
 947                 .oas            = oas,
 948                 .tlb            = &arm_smmu_gather_ops,
 949                 .iommu_dev      = smmu->dev,
 950         };
 951
 952         smmu_domain->smmu = smmu;
 953         pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
 954         if (!pgtbl_ops) {
 955                 ret = -ENOMEM;
 956                 goto out_clear_smmu;
 957         }
 958
 959         /* Update the domain's page sizes to reflect the page table format */
 960         domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
 961
 962         /* Initialise the context bank with our page table cfg */
 963         arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
 964
 965         /*
 966          * Request context fault interrupt. Do this last to avoid the
 967          * handler seeing a half-initialised domain state.
 968          */
 969         irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
 970         ret = devm_request_irq(smmu->dev, irq, arm_smmu_context_fault,
 971                                IRQF_SHARED, "arm-smmu-context-fault", domain);
 972         if (ret < 0) {
 973                 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
 974                         cfg->irptndx, irq);
 975                 cfg->irptndx = INVALID_IRPTNDX;
 976         }
 977
 978         mutex_unlock(&smmu_domain->init_mutex);
 979
 980         /* Publish page table ops for map/unmap */
 981         smmu_domain->pgtbl_ops = pgtbl_ops;
 982         return 0;
 983
 984 out_clear_smmu:
 985         smmu_domain->smmu = NULL;
 986 out_unlock:
 987         mutex_unlock(&smmu_domain->init_mutex);
 988         return ret;
 989 }
 990
 991 static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
 992 {
 993         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 994         struct arm_smmu_device *smmu = smmu_domain->smmu;
 995         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 996         void __iomem *cb_base;
 997         int irq;
 998
 999         if (!smmu || domain->type == IOMMU_DOMAIN_DMA)
1000                 return;
1001
1002         /*
1003          * Disable the context bank and free the page tables before freeing
1004          * it.
1005          */
1006         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
1007         writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
1008
1009         if (cfg->irptndx != INVALID_IRPTNDX) {
1010                 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
1011                 devm_free_irq(smmu->dev, irq, domain);
1012         }
1013
1014         free_io_pgtable_ops(smmu_domain->pgtbl_ops);
1015         __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
1016 }
1017
1018 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
1019 {
1020         struct arm_smmu_domain *smmu_domain;
1021
1022         if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA)
1023                 return NULL;
1024         /*
1025          * Allocate the domain and initialise some of its data structures.
1026          * We can't really do anything meaningful until we've added a
1027          * master.
1028          */
1029         smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
1030         if (!smmu_domain)
1031                 return NULL;
1032
1033         if (type == IOMMU_DOMAIN_DMA &&
1034             iommu_get_dma_cookie(&smmu_domain->domain)) {
1035                 kfree(smmu_domain);
1036                 return NULL;
1037         }
1038
1039         mutex_init(&smmu_domain->init_mutex);
1040         spin_lock_init(&smmu_domain->pgtbl_lock);
1041
1042         return &smmu_domain->domain;
1043 }
1044
1045 static void arm_smmu_domain_free(struct iommu_domain *domain)
1046 {
1047         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1048
1049         /*
1050          * Free the domain resources. We assume that all devices have
1051          * already been detached.
1052          */
1053         iommu_put_dma_cookie(domain);
1054         arm_smmu_destroy_domain_context(domain);
1055         kfree(smmu_domain);
1056 }
1057
1058 static int arm_smmu_master_configure_smrs(struct arm_smmu_device *smmu,
1059                                           struct arm_smmu_master_cfg *cfg)
1060 {
1061         int i;
1062         struct arm_smmu_smr *smrs;
1063         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1064
1065         if (!(smmu->features & ARM_SMMU_FEAT_STREAM_MATCH))
1066                 return 0;
1067
1068         if (cfg->smrs)
1069                 return -EEXIST;
1070
1071         smrs = kmalloc_array(cfg->num_streamids, sizeof(*smrs), GFP_KERNEL);
1072         if (!smrs) {
1073                 dev_err(smmu->dev, "failed to allocate %d SMRs\n",
1074                         cfg->num_streamids);
1075                 return -ENOMEM;
1076         }
1077
1078         /* Allocate the SMRs on the SMMU */
1079         for (i = 0; i < cfg->num_streamids; ++i) {
1080                 int idx = __arm_smmu_alloc_bitmap(smmu->smr_map, 0,
1081                                                   smmu->num_mapping_groups);
1082                 if (idx < 0) {
1083                         dev_err(smmu->dev, "failed to allocate free SMR\n");
1084                         goto err_free_smrs;
1085                 }
1086
1087                 smrs[i] = (struct arm_smmu_smr) {
1088                         .idx    = idx,
1089                         .mask   = 0, /* We don't currently share SMRs */
1090                         .id     = cfg->streamids[i],
1091                 };
1092         }
1093
1094         /* It worked! Now, poke the actual hardware */
1095         for (i = 0; i < cfg->num_streamids; ++i) {
1096                 u32 reg = SMR_VALID | smrs[i].id << SMR_ID_SHIFT |
1097                           smrs[i].mask << SMR_MASK_SHIFT;
1098                 writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_SMR(smrs[i].idx));
1099         }
1100
1101         cfg->smrs = smrs;
1102         return 0;
1103
1104 err_free_smrs:
1105         while (--i >= 0)
1106                 __arm_smmu_free_bitmap(smmu->smr_map, smrs[i].idx);
1107         kfree(smrs);
1108         return -ENOSPC;
1109 }
1110
1111 static void arm_smmu_master_free_smrs(struct arm_smmu_device *smmu,
1112                                       struct arm_smmu_master_cfg *cfg)
1113 {
1114         int i;
1115         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1116         struct arm_smmu_smr *smrs = cfg->smrs;
1117
1118         if (!smrs)
1119                 return;
1120
1121         /* Invalidate the SMRs before freeing back to the allocator */
1122         for (i = 0; i < cfg->num_streamids; ++i) {
1123                 u8 idx = smrs[i].idx;
1124
1125                 writel_relaxed(~SMR_VALID, gr0_base + ARM_SMMU_GR0_SMR(idx));
1126                 __arm_smmu_free_bitmap(smmu->smr_map, idx);
1127         }
1128
1129         cfg->smrs = NULL;
1130         kfree(smrs);
1131 }
1132
1133 static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
1134                                       struct arm_smmu_master_cfg *cfg)
1135 {
1136         int i, ret;
1137         struct arm_smmu_device *smmu = smmu_domain->smmu;
1138         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1139
1140         /*
1141          * FIXME: This won't be needed once we have IOMMU-backed DMA ops
1142          * for all devices behind the SMMU. Note that we need to take
1143          * care configuring SMRs for devices both a platform_device and
1144          * and a PCI device (i.e. a PCI host controller)
1145          */
1146         if (smmu_domain->domain.type == IOMMU_DOMAIN_DMA)
1147                 return 0;
1148
1149         /* Devices in an IOMMU group may already be configured */
1150         ret = arm_smmu_master_configure_smrs(smmu, cfg);
1151         if (ret)
1152                 return ret == -EEXIST ? 0 : ret;
1153
1154         for (i = 0; i < cfg->num_streamids; ++i) {
1155                 u32 idx, s2cr;
1156
1157                 idx = cfg->smrs ? cfg->smrs[i].idx : cfg->streamids[i];
1158                 s2cr = S2CR_TYPE_TRANS | S2CR_PRIVCFG_UNPRIV |
1159                        (smmu_domain->cfg.cbndx << S2CR_CBNDX_SHIFT);
1160                 writel_relaxed(s2cr, gr0_base + ARM_SMMU_GR0_S2CR(idx));
1161         }
1162
1163         return 0;
1164 }
1165
1166 static void arm_smmu_domain_remove_master(struct arm_smmu_domain *smmu_domain,
1167                                           struct arm_smmu_master_cfg *cfg)
1168 {
1169         int i;
1170         struct arm_smmu_device *smmu = smmu_domain->smmu;
1171         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1172
1173         /* An IOMMU group is torn down by the first device to be removed */
1174         if ((smmu->features & ARM_SMMU_FEAT_STREAM_MATCH) && !cfg->smrs)
1175                 return;
1176
1177         /*
1178          * We *must* clear the S2CR first, because freeing the SMR means
1179          * that it can be re-allocated immediately.
1180          */
1181         for (i = 0; i < cfg->num_streamids; ++i) {
1182                 u32 idx = cfg->smrs ? cfg->smrs[i].idx : cfg->streamids[i];
1183                 u32 reg = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS;
1184
1185                 writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_S2CR(idx));
1186         }
1187
1188         arm_smmu_master_free_smrs(smmu, cfg);
1189 }
1190
1191 static void arm_smmu_detach_dev(struct device *dev,
1192                                 struct arm_smmu_master_cfg *cfg)
1193 {
1194         struct iommu_domain *domain = dev->archdata.iommu;
1195         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1196
1197         dev->archdata.iommu = NULL;
1198         arm_smmu_domain_remove_master(smmu_domain, cfg);
1199 }
1200
1201 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1202 {
1203         int ret;
1204         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1205         struct arm_smmu_device *smmu;
1206         struct arm_smmu_master_cfg *cfg;
1207
1208         smmu = find_smmu_for_device(dev);
1209         if (!smmu) {
1210                 dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n");
1211                 return -ENXIO;
1212         }
1213
1214         /* Ensure that the domain is finalised */
1215         ret = arm_smmu_init_domain_context(domain, smmu);
1216         if (ret < 0)
1217                 return ret;
1218
1219         /*
1220          * Sanity check the domain. We don't support domains across
1221          * different SMMUs.
1222          */
1223         if (smmu_domain->smmu != smmu) {
1224                 dev_err(dev,
1225                         "cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n",
1226                         dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev));
1227                 return -EINVAL;
1228         }
1229
1230         /* Looks ok, so add the device to the domain */
1231         cfg = find_smmu_master_cfg(dev);
1232         if (!cfg)
1233                 return -ENODEV;
1234
1235         /* Detach the dev from its current domain */
1236         if (dev->archdata.iommu)
1237                 arm_smmu_detach_dev(dev, cfg);
1238
1239         ret = arm_smmu_domain_add_master(smmu_domain, cfg);
1240         if (!ret)
1241                 dev->archdata.iommu = domain;
1242         return ret;
1243 }
1244
1245 static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
1246                         phys_addr_t paddr, size_t size, int prot)
1247 {
1248         int ret;
1249         unsigned long flags;
1250         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1251         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1252
1253         if (!ops)
1254                 return -ENODEV;
1255
1256         spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1257         ret = ops->map(ops, iova, paddr, size, prot);
1258         spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1259         return ret;
1260 }
1261
1262 static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
1263                              size_t size)
1264 {
1265         size_t ret;
1266         unsigned long flags;
1267         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1268         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1269
1270         if (!ops)
1271                 return 0;
1272
1273         spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1274         ret = ops->unmap(ops, iova, size);
1275         spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1276         return ret;
1277 }
1278
1279 static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
1280                                               dma_addr_t iova)
1281 {
1282         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1283         struct arm_smmu_device *smmu = smmu_domain->smmu;
1284         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1285         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1286         struct device *dev = smmu->dev;
1287         void __iomem *cb_base;
1288         u32 tmp;
1289         u64 phys;
1290         unsigned long va;
1291
1292         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
1293
1294         /* ATS1 registers can only be written atomically */
1295         va = iova & ~0xfffUL;
1296         if (smmu->version == ARM_SMMU_V2)
1297                 smmu_write_atomic_lq(va, cb_base + ARM_SMMU_CB_ATS1PR);
1298         else /* Register is only 32-bit in v1 */
1299                 writel_relaxed(va, cb_base + ARM_SMMU_CB_ATS1PR);
1300
1301         if (readl_poll_timeout_atomic(cb_base + ARM_SMMU_CB_ATSR, tmp,
1302                                       !(tmp & ATSR_ACTIVE), 5, 50)) {
1303                 dev_err(dev,
1304                         "iova to phys timed out on %pad. Falling back to software table walk.\n",
1305                         &iova);
1306                 return ops->iova_to_phys(ops, iova);
1307         }
1308
1309         phys = readq_relaxed(cb_base + ARM_SMMU_CB_PAR);
1310         if (phys & CB_PAR_F) {
1311                 dev_err(dev, "translation fault!\n");
1312                 dev_err(dev, "PAR = 0x%llx\n", phys);
1313                 return 0;
1314         }
1315
1316         return (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff);
1317 }
1318
1319 static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
1320                                         dma_addr_t iova)
1321 {
1322         phys_addr_t ret;
1323         unsigned long flags;
1324         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1325         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1326
1327         if (!ops)
1328                 return 0;
1329
1330         spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1331         if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
1332                         smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1333                 ret = arm_smmu_iova_to_phys_hard(domain, iova);
1334         } else {
1335                 ret = ops->iova_to_phys(ops, iova);
1336         }
1337
1338         spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1339
1340         return ret;
1341 }
1342
1343 static bool arm_smmu_capable(enum iommu_cap cap)
1344 {
1345         switch (cap) {
1346         case IOMMU_CAP_CACHE_COHERENCY:
1347                 /*
1348                  * Return true here as the SMMU can always send out coherent
1349                  * requests.
1350                  */
1351                 return true;
1352         case IOMMU_CAP_INTR_REMAP:
1353                 return true; /* MSIs are just memory writes */
1354         case IOMMU_CAP_NOEXEC:
1355                 return true;
1356         default:
1357                 return false;
1358         }
1359 }
1360
1361 static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
1362 {
1363         *((u16 *)data) = alias;
1364         return 0; /* Continue walking */
1365 }
1366
1367 static void __arm_smmu_release_pci_iommudata(void *data)
1368 {
1369         kfree(data);
1370 }
1371
1372 static int arm_smmu_init_pci_device(struct pci_dev *pdev,
1373                                     struct iommu_group *group)
1374 {
1375         struct arm_smmu_master_cfg *cfg;
1376         u16 sid;
1377         int i;
1378
1379         cfg = iommu_group_get_iommudata(group);
1380         if (!cfg) {
1381                 cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);
1382                 if (!cfg)
1383                         return -ENOMEM;
1384
1385                 iommu_group_set_iommudata(group, cfg,
1386                                           __arm_smmu_release_pci_iommudata);
1387         }
1388
1389         if (cfg->num_streamids >= MAX_MASTER_STREAMIDS)
1390                 return -ENOSPC;
1391
1392         /*
1393          * Assume Stream ID == Requester ID for now.
1394          * We need a way to describe the ID mappings in FDT.
1395          */
1396         pci_for_each_dma_alias(pdev, __arm_smmu_get_pci_sid, &sid);
1397         for (i = 0; i < cfg->num_streamids; ++i)
1398                 if (cfg->streamids[i] == sid)
1399                         break;
1400
1401         /* Avoid duplicate SIDs, as this can lead to SMR conflicts */
1402         if (i == cfg->num_streamids)
1403                 cfg->streamids[cfg->num_streamids++] = sid;
1404
1405         return 0;
1406 }
1407
1408 static int arm_smmu_init_platform_device(struct device *dev,
1409                                          struct iommu_group *group)
1410 {
1411         struct arm_smmu_device *smmu = find_smmu_for_device(dev);
1412         struct arm_smmu_master *master;
1413
1414         if (!smmu)
1415                 return -ENODEV;
1416
1417         master = find_smmu_master(smmu, dev->of_node);
1418         if (!master)
1419                 return -ENODEV;
1420
1421         iommu_group_set_iommudata(group, &master->cfg, NULL);
1422
1423         return 0;
1424 }
1425
1426 static int arm_smmu_add_device(struct device *dev)
1427 {
1428         struct iommu_group *group;
1429
1430         group = iommu_group_get_for_dev(dev);
1431         if (IS_ERR(group))
1432                 return PTR_ERR(group);
1433
1434         iommu_group_put(group);
1435         return 0;
1436 }
1437
1438 static void arm_smmu_remove_device(struct device *dev)
1439 {
1440         iommu_group_remove_device(dev);
1441 }
1442
1443 static struct iommu_group *arm_smmu_device_group(struct device *dev)
1444 {
1445         struct iommu_group *group;
1446         int ret;
1447
1448         if (dev_is_pci(dev))
1449                 group = pci_device_group(dev);
1450         else
1451                 group = generic_device_group(dev);
1452
1453         if (IS_ERR(group))
1454                 return group;
1455
1456         if (dev_is_pci(dev))
1457                 ret = arm_smmu_init_pci_device(to_pci_dev(dev), group);
1458         else
1459                 ret = arm_smmu_init_platform_device(dev, group);
1460
1461         if (ret) {
1462                 iommu_group_put(group);
1463                 group = ERR_PTR(ret);
1464         }
1465
1466         return group;
1467 }
1468
1469 static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
1470                                     enum iommu_attr attr, void *data)
1471 {
1472         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1473
1474         switch (attr) {
1475         case DOMAIN_ATTR_NESTING:
1476                 *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
1477                 return 0;
1478         default:
1479                 return -ENODEV;
1480         }
1481 }
1482
1483 static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
1484                                     enum iommu_attr attr, void *data)
1485 {
1486         int ret = 0;
1487         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1488
1489         mutex_lock(&smmu_domain->init_mutex);
1490
1491         switch (attr) {
1492         case DOMAIN_ATTR_NESTING:
1493                 if (smmu_domain->smmu) {
1494                         ret = -EPERM;
1495                         goto out_unlock;
1496                 }
1497
1498                 if (*(int *)data)
1499                         smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
1500                 else
1501                         smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1502
1503                 break;
1504         default:
1505                 ret = -ENODEV;
1506         }
1507
1508 out_unlock:
1509         mutex_unlock(&smmu_domain->init_mutex);
1510         return ret;
1511 }
1512
1513 static struct iommu_ops arm_smmu_ops = {
1514         .capable                = arm_smmu_capable,
1515         .domain_alloc           = arm_smmu_domain_alloc,
1516         .domain_free            = arm_smmu_domain_free,
1517         .attach_dev             = arm_smmu_attach_dev,
1518         .map                    = arm_smmu_map,
1519         .unmap                  = arm_smmu_unmap,
1520         .map_sg                 = default_iommu_map_sg,
1521         .iova_to_phys           = arm_smmu_iova_to_phys,
1522         .add_device             = arm_smmu_add_device,
1523         .remove_device          = arm_smmu_remove_device,
1524         .device_group           = arm_smmu_device_group,
1525         .domain_get_attr        = arm_smmu_domain_get_attr,
1526         .domain_set_attr        = arm_smmu_domain_set_attr,
1527         .pgsize_bitmap          = -1UL, /* Restricted during device attach */
1528 };
1529
1530 static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
1531 {
1532         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1533         void __iomem *cb_base;
1534         int i = 0;
1535         u32 reg, major;
1536
1537         /* clear global FSR */
1538         reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
1539         writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
1540
1541         /* Mark all SMRn as invalid and all S2CRn as bypass unless overridden */
1542         reg = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS;
1543         for (i = 0; i < smmu->num_mapping_groups; ++i) {
1544                 writel_relaxed(0, gr0_base + ARM_SMMU_GR0_SMR(i));
1545                 writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_S2CR(i));
1546         }
1547
1548         /*
1549          * Before clearing ARM_MMU500_ACTLR_CPRE, need to
1550          * clear CACHE_LOCK bit of ACR first. And, CACHE_LOCK
1551          * bit is only present in MMU-500r2 onwards.
1552          */
1553         reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID7);
1554         major = (reg >> ID7_MAJOR_SHIFT) & ID7_MAJOR_MASK;
1555         if ((smmu->model == ARM_MMU500) && (major >= 2)) {
1556                 reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_sACR);
1557                 reg &= ~ARM_MMU500_ACR_CACHE_LOCK;
1558                 writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_sACR);
1559         }
1560
1561         /* Make sure all context banks are disabled and clear CB_FSR  */
1562         for (i = 0; i < smmu->num_context_banks; ++i) {
1563                 cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, i);
1564                 writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
1565                 writel_relaxed(FSR_FAULT, cb_base + ARM_SMMU_CB_FSR);
1566                 /*
1567                  * Disable MMU-500's not-particularly-beneficial next-page
1568                  * prefetcher for the sake of errata #841119 and #826419.
1569                  */
1570                 if (smmu->model == ARM_MMU500) {
1571                         reg = readl_relaxed(cb_base + ARM_SMMU_CB_ACTLR);
1572                         reg &= ~ARM_MMU500_ACTLR_CPRE;
1573                         writel_relaxed(reg, cb_base + ARM_SMMU_CB_ACTLR);
1574                 }
1575         }
1576
1577         /* Invalidate the TLB, just in case */
1578         writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLH);
1579         writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLNSNH);
1580
1581         reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
1582
1583         /* Enable fault reporting */
1584         reg |= (sCR0_GFRE | sCR0_GFIE | sCR0_GCFGFRE | sCR0_GCFGFIE);
1585
1586         /* Disable TLB broadcasting. */
1587         reg |= (sCR0_VMIDPNE | sCR0_PTM);
1588
1589         /* Enable client access, handling unmatched streams as appropriate */
1590         reg &= ~sCR0_CLIENTPD;
1591         if (disable_bypass)
1592                 reg |= sCR0_USFCFG;
1593         else
1594                 reg &= ~sCR0_USFCFG;
1595
1596         /* Disable forced broadcasting */
1597         reg &= ~sCR0_FB;
1598
1599         /* Don't upgrade barriers */
1600         reg &= ~(sCR0_BSU_MASK << sCR0_BSU_SHIFT);
1601
1602         if (smmu->features & ARM_SMMU_FEAT_VMID16)
1603                 reg |= sCR0_VMID16EN;
1604
1605         /* Push the button */
1606         __arm_smmu_tlb_sync(smmu);
1607         writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
1608 }
1609
1610 static int arm_smmu_id_size_to_bits(int size)
1611 {
1612         switch (size) {
1613         case 0:
1614                 return 32;
1615         case 1:
1616                 return 36;
1617         case 2:
1618                 return 40;
1619         case 3:
1620                 return 42;
1621         case 4:
1622                 return 44;
1623         case 5:
1624         default:
1625                 return 48;
1626         }
1627 }
1628
1629 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1630 {
1631         unsigned long size;
1632         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1633         u32 id;
1634         bool cttw_dt, cttw_reg;
1635
1636         dev_notice(smmu->dev, "probing hardware configuration...\n");
1637         dev_notice(smmu->dev, "SMMUv%d with:\n",
1638                         smmu->version == ARM_SMMU_V2 ? 2 : 1);
1639
1640         /* ID0 */
1641         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID0);
1642
1643         /* Restrict available stages based on module parameter */
1644         if (force_stage == 1)
1645                 id &= ~(ID0_S2TS | ID0_NTS);
1646         else if (force_stage == 2)
1647                 id &= ~(ID0_S1TS | ID0_NTS);
1648
1649         if (id & ID0_S1TS) {
1650                 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
1651                 dev_notice(smmu->dev, "\tstage 1 translation\n");
1652         }
1653
1654         if (id & ID0_S2TS) {
1655                 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
1656                 dev_notice(smmu->dev, "\tstage 2 translation\n");
1657         }
1658
1659         if (id & ID0_NTS) {
1660                 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;
1661                 dev_notice(smmu->dev, "\tnested translation\n");
1662         }
1663
1664         if (!(smmu->features &
1665                 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) {
1666                 dev_err(smmu->dev, "\tno translation support!\n");
1667                 return -ENODEV;
1668         }
1669
1670         if ((id & ID0_S1TS) &&
1671                 ((smmu->version < ARM_SMMU_V2) || !(id & ID0_ATOSNS))) {
1672                 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;
1673                 dev_notice(smmu->dev, "\taddress translation ops\n");
1674         }
1675
1676         /*
1677          * In order for DMA API calls to work properly, we must defer to what
1678          * the DT says about coherency, regardless of what the hardware claims.
1679          * Fortunately, this also opens up a workaround for systems where the
1680          * ID register value has ended up configured incorrectly.
1681          */
1682         cttw_dt = of_dma_is_coherent(smmu->dev->of_node);
1683         cttw_reg = !!(id & ID0_CTTW);
1684         if (cttw_dt)
1685                 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
1686         if (cttw_dt || cttw_reg)
1687                 dev_notice(smmu->dev, "\t%scoherent table walk\n",
1688                            cttw_dt ? "" : "non-");
1689         if (cttw_dt != cttw_reg)
1690                 dev_notice(smmu->dev,
1691                            "\t(IDR0.CTTW overridden by dma-coherent property)\n");
1692
1693         if (id & ID0_SMS) {
1694                 u32 smr, sid, mask;
1695
1696                 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
1697                 smmu->num_mapping_groups = (id >> ID0_NUMSMRG_SHIFT) &
1698                                            ID0_NUMSMRG_MASK;
1699                 if (smmu->num_mapping_groups == 0) {
1700                         dev_err(smmu->dev,
1701                                 "stream-matching supported, but no SMRs present!\n");
1702                         return -ENODEV;
1703                 }
1704
1705                 smr = SMR_MASK_MASK << SMR_MASK_SHIFT;
1706                 smr |= (SMR_ID_MASK << SMR_ID_SHIFT);
1707                 writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0));
1708                 smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0));
1709
1710                 mask = (smr >> SMR_MASK_SHIFT) & SMR_MASK_MASK;
1711                 sid = (smr >> SMR_ID_SHIFT) & SMR_ID_MASK;
1712                 if ((mask & sid) != sid) {
1713                         dev_err(smmu->dev,
1714                                 "SMR mask bits (0x%x) insufficient for ID field (0x%x)\n",
1715                                 mask, sid);
1716                         return -ENODEV;
1717                 }
1718
1719                 dev_notice(smmu->dev,
1720                            "\tstream matching with %u register groups, mask 0x%x",
1721                            smmu->num_mapping_groups, mask);
1722         } else {
1723                 smmu->num_mapping_groups = (id >> ID0_NUMSIDB_SHIFT) &
1724                                            ID0_NUMSIDB_MASK;
1725         }
1726
1727         if (smmu->version < ARM_SMMU_V2 || !(id & ID0_PTFS_NO_AARCH32)) {
1728                 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L;
1729                 if (!(id & ID0_PTFS_NO_AARCH32S))
1730                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S;
1731         }
1732
1733         /* ID1 */
1734         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID1);
1735         smmu->pgshift = (id & ID1_PAGESIZE) ? 16 : 12;
1736
1737         /* Check for size mismatch of SMMU address space from mapped region */
1738         size = 1 << (((id >> ID1_NUMPAGENDXB_SHIFT) & ID1_NUMPAGENDXB_MASK) + 1);
1739         size *= 2 << smmu->pgshift;
1740         if (smmu->size != size)
1741                 dev_warn(smmu->dev,
1742                         "SMMU address space size (0x%lx) differs from mapped region size (0x%lx)!\n",
1743                         size, smmu->size);
1744
1745         smmu->num_s2_context_banks = (id >> ID1_NUMS2CB_SHIFT) & ID1_NUMS2CB_MASK;
1746         smmu->num_context_banks = (id >> ID1_NUMCB_SHIFT) & ID1_NUMCB_MASK;
1747         if (smmu->num_s2_context_banks > smmu->num_context_banks) {
1748                 dev_err(smmu->dev, "impossible number of S2 context banks!\n");
1749                 return -ENODEV;
1750         }
1751         dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",
1752                    smmu->num_context_banks, smmu->num_s2_context_banks);
1753         /*
1754          * Cavium CN88xx erratum #27704.
1755          * Ensure ASID and VMID allocation is unique across all SMMUs in
1756          * the system.
1757          */
1758         if (smmu->model == CAVIUM_SMMUV2) {
1759                 smmu->cavium_id_base =
1760                         atomic_add_return(smmu->num_context_banks,
1761                                           &cavium_smmu_context_count);
1762                 smmu->cavium_id_base -= smmu->num_context_banks;
1763         }
1764
1765         /* ID2 */
1766         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID2);
1767         size = arm_smmu_id_size_to_bits((id >> ID2_IAS_SHIFT) & ID2_IAS_MASK);
1768         smmu->ipa_size = size;
1769
1770         /* The output mask is also applied for bypass */
1771         size = arm_smmu_id_size_to_bits((id >> ID2_OAS_SHIFT) & ID2_OAS_MASK);
1772         smmu->pa_size = size;
1773
1774         if (id & ID2_VMID16)
1775                 smmu->features |= ARM_SMMU_FEAT_VMID16;
1776
1777         /*
1778          * What the page table walker can address actually depends on which
1779          * descriptor format is in use, but since a) we don't know that yet,
1780          * and b) it can vary per context bank, this will have to do...
1781          */
1782         if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))
1783                 dev_warn(smmu->dev,
1784                          "failed to set DMA mask for table walker\n");
1785
1786         if (smmu->version < ARM_SMMU_V2) {
1787                 smmu->va_size = smmu->ipa_size;
1788                 if (smmu->version == ARM_SMMU_V1_64K)
1789                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1790         } else {
1791                 size = (id >> ID2_UBS_SHIFT) & ID2_UBS_MASK;
1792                 smmu->va_size = arm_smmu_id_size_to_bits(size);
1793                 if (id & ID2_PTFS_4K)
1794                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K;
1795                 if (id & ID2_PTFS_16K)
1796                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K;
1797                 if (id & ID2_PTFS_64K)
1798                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1799         }
1800
1801         /* Now we've corralled the various formats, what'll it do? */
1802         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S)
1803                 smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M;
1804         if (smmu->features &
1805             (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K))
1806                 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
1807         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K)
1808                 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
1809         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K)
1810                 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
1811
1812         if (arm_smmu_ops.pgsize_bitmap == -1UL)
1813                 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
1814         else
1815                 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
1816         dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n",
1817                    smmu->pgsize_bitmap);
1818
1819
1820         if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
1821                 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
1822                            smmu->va_size, smmu->ipa_size);
1823
1824         if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)
1825                 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
1826                            smmu->ipa_size, smmu->pa_size);
1827
1828         return 0;
1829 }
1830
1831 struct arm_smmu_match_data {
1832         enum arm_smmu_arch_version version;
1833         enum arm_smmu_implementation model;
1834 };
1835
1836 #define ARM_SMMU_MATCH_DATA(name, ver, imp)     \
1837 static struct arm_smmu_match_data name = { .version = ver, .model = imp }
1838
1839 ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU);
1840 ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU);
1841 ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU);
1842 ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500);
1843 ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2);
1844
1845 static const struct of_device_id arm_smmu_of_match[] = {
1846         { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 },
1847         { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 },
1848         { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 },
1849         { .compatible = "arm,mmu-401", .data = &arm_mmu401 },
1850         { .compatible = "arm,mmu-500", .data = &arm_mmu500 },
1851         { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 },
1852         { },
1853 };
1854 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
1855
1856 static int arm_smmu_device_dt_probe(struct platform_device *pdev)
1857 {
1858         const struct of_device_id *of_id;
1859         const struct arm_smmu_match_data *data;
1860         struct resource *res;
1861         struct arm_smmu_device *smmu;
1862         struct device *dev = &pdev->dev;
1863         struct rb_node *node;
1864         struct of_phandle_iterator it;
1865         struct arm_smmu_phandle_args *masterspec;
1866         int num_irqs, i, err;
1867
1868         smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
1869         if (!smmu) {
1870                 dev_err(dev, "failed to allocate arm_smmu_device\n");
1871                 return -ENOMEM;
1872         }
1873         smmu->dev = dev;
1874
1875         of_id = of_match_node(arm_smmu_of_match, dev->of_node);
1876         data = of_id->data;
1877         smmu->version = data->version;
1878         smmu->model = data->model;
1879
1880         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1881         smmu->base = devm_ioremap_resource(dev, res);
1882         if (IS_ERR(smmu->base))
1883                 return PTR_ERR(smmu->base);
1884         smmu->size = resource_size(res);
1885
1886         if (of_property_read_u32(dev->of_node, "#global-interrupts",
1887                                  &smmu->num_global_irqs)) {
1888                 dev_err(dev, "missing #global-interrupts property\n");
1889                 return -ENODEV;
1890         }
1891
1892         num_irqs = 0;
1893         while ((res = platform_get_resource(pdev, IORESOURCE_IRQ, num_irqs))) {
1894                 num_irqs++;
1895                 if (num_irqs > smmu->num_global_irqs)
1896                         smmu->num_context_irqs++;
1897         }
1898
1899         if (!smmu->num_context_irqs) {
1900                 dev_err(dev, "found %d interrupts but expected at least %d\n",
1901                         num_irqs, smmu->num_global_irqs + 1);
1902                 return -ENODEV;
1903         }
1904
1905         smmu->irqs = devm_kzalloc(dev, sizeof(*smmu->irqs) * num_irqs,
1906                                   GFP_KERNEL);
1907         if (!smmu->irqs) {
1908                 dev_err(dev, "failed to allocate %d irqs\n", num_irqs);
1909                 return -ENOMEM;
1910         }
1911
1912         for (i = 0; i < num_irqs; ++i) {
1913                 int irq = platform_get_irq(pdev, i);
1914
1915                 if (irq < 0) {
1916                         dev_err(dev, "failed to get irq index %d\n", i);
1917                         return -ENODEV;
1918                 }
1919                 smmu->irqs[i] = irq;
1920         }
1921
1922         err = arm_smmu_device_cfg_probe(smmu);
1923         if (err)
1924                 return err;
1925
1926         i = 0;
1927         smmu->masters = RB_ROOT;
1928
1929         err = -ENOMEM;
1930         /* No need to zero the memory for masterspec */
1931         masterspec = kmalloc(sizeof(*masterspec), GFP_KERNEL);
1932         if (!masterspec)
1933                 goto out_put_masters;
1934
1935         of_for_each_phandle(&it, err, dev->of_node,
1936                             "mmu-masters", "#stream-id-cells", 0) {
1937                 int count = of_phandle_iterator_args(&it, masterspec->args,
1938                                                      MAX_MASTER_STREAMIDS);
1939                 masterspec->np          = of_node_get(it.node);
1940                 masterspec->args_count  = count;
1941
1942                 err = register_smmu_master(smmu, dev, masterspec);
1943                 if (err) {
1944                         dev_err(dev, "failed to add master %s\n",
1945                                 masterspec->np->name);
1946                         kfree(masterspec);
1947                         goto out_put_masters;
1948                 }
1949
1950                 i++;
1951         }
1952
1953         dev_notice(dev, "registered %d master devices\n", i);
1954
1955         kfree(masterspec);
1956
1957         parse_driver_options(smmu);
1958
1959         if (smmu->version == ARM_SMMU_V2 &&
1960             smmu->num_context_banks != smmu->num_context_irqs) {
1961                 dev_err(dev,
1962                         "found only %d context interrupt(s) but %d required\n",
1963                         smmu->num_context_irqs, smmu->num_context_banks);
1964                 err = -ENODEV;
1965                 goto out_put_masters;
1966         }
1967
1968         for (i = 0; i < smmu->num_global_irqs; ++i) {
1969                 err = devm_request_irq(smmu->dev, smmu->irqs[i],
1970                                        arm_smmu_global_fault,
1971                                        IRQF_SHARED,
1972                                        "arm-smmu global fault",
1973                                        smmu);
1974                 if (err) {
1975                         dev_err(dev, "failed to request global IRQ %d (%u)\n",
1976                                 i, smmu->irqs[i]);
1977                         goto out_put_masters;
1978                 }
1979         }
1980
1981         INIT_LIST_HEAD(&smmu->list);
1982         spin_lock(&arm_smmu_devices_lock);
1983         list_add(&smmu->list, &arm_smmu_devices);
1984         spin_unlock(&arm_smmu_devices_lock);
1985
1986         arm_smmu_device_reset(smmu);
1987         return 0;
1988
1989 out_put_masters:
1990         for (node = rb_first(&smmu->masters); node; node = rb_next(node)) {
1991                 struct arm_smmu_master *master
1992                         = container_of(node, struct arm_smmu_master, node);
1993                 of_node_put(master->of_node);
1994         }
1995
1996         return err;
1997 }
1998
1999 static int arm_smmu_device_remove(struct platform_device *pdev)
2000 {
2001         int i;
2002         struct device *dev = &pdev->dev;
2003         struct arm_smmu_device *curr, *smmu = NULL;
2004         struct rb_node *node;
2005
2006         spin_lock(&arm_smmu_devices_lock);
2007         list_for_each_entry(curr, &arm_smmu_devices, list) {
2008                 if (curr->dev == dev) {
2009                         smmu = curr;
2010                         list_del(&smmu->list);
2011                         break;
2012                 }
2013         }
2014         spin_unlock(&arm_smmu_devices_lock);
2015
2016         if (!smmu)
2017                 return -ENODEV;
2018
2019         for (node = rb_first(&smmu->masters); node; node = rb_next(node)) {
2020                 struct arm_smmu_master *master
2021                         = container_of(node, struct arm_smmu_master, node);
2022                 of_node_put(master->of_node);
2023         }
2024
2025         if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
2026                 dev_err(dev, "removing device with active domains!\n");
2027
2028         for (i = 0; i < smmu->num_global_irqs; ++i)
2029                 devm_free_irq(smmu->dev, smmu->irqs[i], smmu);
2030
2031         /* Turn the thing off */
2032         writel(sCR0_CLIENTPD, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
2033         return 0;
2034 }
2035
2036 static struct platform_driver arm_smmu_driver = {
2037         .driver = {
2038                 .name           = "arm-smmu",
2039                 .of_match_table = of_match_ptr(arm_smmu_of_match),
2040         },
2041         .probe  = arm_smmu_device_dt_probe,
2042         .remove = arm_smmu_device_remove,
2043 };
2044
2045 static int __init arm_smmu_init(void)
2046 {
2047         struct device_node *np;
2048         int ret;
2049
2050         /*
2051          * Play nice with systems that don't have an ARM SMMU by checking that
2052          * an ARM SMMU exists in the system before proceeding with the driver
2053          * and IOMMU bus operation registration.
2054          */
2055         np = of_find_matching_node(NULL, arm_smmu_of_match);
2056         if (!np)
2057                 return 0;
2058
2059         of_node_put(np);
2060
2061         ret = platform_driver_register(&arm_smmu_driver);
2062         if (ret)
2063                 return ret;
2064
2065         /* Oh, for a proper bus abstraction */
2066         if (!iommu_present(&platform_bus_type))
2067                 bus_set_iommu(&platform_bus_type, &arm_smmu_ops);
2068
2069 #ifdef CONFIG_ARM_AMBA
2070         if (!iommu_present(&amba_bustype))
2071                 bus_set_iommu(&amba_bustype, &arm_smmu_ops);
2072 #endif
2073
2074 #ifdef CONFIG_PCI
2075         if (!iommu_present(&pci_bus_type)) {
2076                 pci_request_acs();
2077                 bus_set_iommu(&pci_bus_type, &arm_smmu_ops);
2078         }
2079 #endif
2080
2081         return 0;
2082 }
2083
2084 static void __exit arm_smmu_exit(void)
2085 {
2086         return platform_driver_unregister(&arm_smmu_driver);
2087 }
2088
2089 subsys_initcall(arm_smmu_init);
2090 module_exit(arm_smmu_exit);
2091
2092 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations");
2093 MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>");
2094 MODULE_LICENSE("GPL v2");