arch/arm/kernel/smp_tlb.c

   1 /*
   2  *  linux/arch/arm/kernel/smp_tlb.c
   3  *
   4  *  Copyright (C) 2002 ARM Limited, All Rights Reserved.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 as
   8  * published by the Free Software Foundation.
   9  */
  10 #include <linux/preempt.h>
  11 #include <linux/smp.h>
  12
  13 #include <asm/smp_plat.h>
  14 #include <asm/tlbflush.h>
  15 #include <asm/mmu_context.h>
  16
  17 /**********************************************************************/
  18
  19 /*
  20  * TLB operations
  21  */
  22 struct tlb_args {
  23         struct vm_area_struct *ta_vma;
  24         unsigned long ta_start;
  25         unsigned long ta_end;
  26 };
  27
  28 static inline void ipi_flush_tlb_all(void *ignored)
  29 {
  30         local_flush_tlb_all();
  31 }
  32
  33 static inline void ipi_flush_tlb_mm(void *arg)
  34 {
  35         struct mm_struct *mm = (struct mm_struct *)arg;
  36
  37         local_flush_tlb_mm(mm);
  38 }
  39
  40 static inline void ipi_flush_tlb_page(void *arg)
  41 {
  42         struct tlb_args *ta = (struct tlb_args *)arg;
  43
  44         local_flush_tlb_page(ta->ta_vma, ta->ta_start);
  45 }
  46
  47 static inline void ipi_flush_tlb_kernel_page(void *arg)
  48 {
  49         struct tlb_args *ta = (struct tlb_args *)arg;
  50
  51         local_flush_tlb_kernel_page(ta->ta_start);
  52 }
  53
  54 static inline void ipi_flush_tlb_range(void *arg)
  55 {
  56         struct tlb_args *ta = (struct tlb_args *)arg;
  57
  58         local_flush_tlb_range(ta->ta_vma, ta->ta_start, ta->ta_end);
  59 }
  60
  61 static inline void ipi_flush_tlb_kernel_range(void *arg)
  62 {
  63         struct tlb_args *ta = (struct tlb_args *)arg;
  64
  65         local_flush_tlb_kernel_range(ta->ta_start, ta->ta_end);
  66 }
  67
  68 static inline void ipi_flush_bp_all(void *ignored)
  69 {
  70         local_flush_bp_all();
  71 }
  72
  73 #ifdef CONFIG_ARM_ERRATA_798181
  74 bool (*erratum_a15_798181_handler)(void);
  75
  76 static bool erratum_a15_798181_partial(void)
  77 {
  78         asm("mcr p15, 0, %0, c8, c3, 1" : : "r" (0));
  79         dsb(ish);
  80         return false;
  81 }
  82
  83 static bool erratum_a15_798181_broadcast(void)
  84 {
  85         asm("mcr p15, 0, %0, c8, c3, 1" : : "r" (0));
  86         dsb(ish);
  87         return true;
  88 }
  89
  90 void erratum_a15_798181_init(void)
  91 {
  92         unsigned int midr = read_cpuid_id();
  93         unsigned int revidr = read_cpuid(CPUID_REVIDR);
  94
  95         /* Brahma-B15 r0p0..r0p2 affected
  96          * Cortex-A15 r0p0..r3p3 w/o ECO fix affected
  97          * Fixes applied to A15 with respect to the revision and revidr are:
  98          *
  99          * r0p0-r2p1: No fixes applied
 100          * r2p2,r2p3:
 101          *      REVIDR[4]: 798181 Moving a virtual page that is being accessed
 102          *                 by an active process can lead to unexpected behavior
 103          *      REVIDR[9]: Not defined
 104          * r2p4,r3p0,r3p1,r3p2:
 105          *      REVIDR[4]: 798181 Moving a virtual page that is being accessed
 106          *                 by an active process can lead to unexpected behavior
 107          *      REVIDR[9]: 798181 Moving a virtual page that is being accessed
 108          *                 by an active process can lead to unexpected behavior
 109          *                 - This is an update to a previously released ECO.
 110          * r3p3:
 111          *      REVIDR[4]: Reserved
 112          *      REVIDR[9]: 798181 Moving a virtual page that is being accessed
 113          *                 by an active process can lead to unexpected behavior
 114          *                 - This is an update to a previously released ECO.
 115          *
 116          * Handling:
 117          *      REVIDR[9] set -> No WA
 118          *      REVIDR[4] set, REVIDR[9] cleared -> Partial WA
 119          *      Both cleared -> Full WA
 120          */
 121         if ((midr & 0xff0ffff0) == 0x420f00f0 && midr <= 0x420f00f2) {
 122                 erratum_a15_798181_handler = erratum_a15_798181_broadcast;
 123         } else if ((midr & 0xff0ffff0) == 0x410fc0f0 && midr < 0x412fc0f2) {
 124                 erratum_a15_798181_handler = erratum_a15_798181_broadcast;
 125         } else if ((midr & 0xff0ffff0) == 0x410fc0f0 && midr < 0x412fc0f4) {
 126                 if (revidr & 0x10)
 127                         erratum_a15_798181_handler =
 128                                 erratum_a15_798181_partial;
 129                 else
 130                         erratum_a15_798181_handler =
 131                                 erratum_a15_798181_broadcast;
 132         } else if ((midr & 0xff0ffff0) == 0x410fc0f0 && midr < 0x413fc0f3) {
 133                 if ((revidr & 0x210) == 0)
 134                         erratum_a15_798181_handler =
 135                                 erratum_a15_798181_broadcast;
 136                 else if (revidr & 0x10)
 137                         erratum_a15_798181_handler =
 138                                 erratum_a15_798181_partial;
 139         } else if ((midr & 0xff0ffff0) == 0x410fc0f0 && midr < 0x414fc0f0) {
 140                 if ((revidr & 0x200) == 0)
 141                         erratum_a15_798181_handler =
 142                                 erratum_a15_798181_partial;
 143         }
 144 }
 145 #endif
 146
 147 static void ipi_flush_tlb_a15_erratum(void *arg)
 148 {
 149         dmb();
 150 }
 151
 152 static void broadcast_tlb_a15_erratum(void)
 153 {
 154         if (!erratum_a15_798181())
 155                 return;
 156
 157         smp_call_function(ipi_flush_tlb_a15_erratum, NULL, 1);
 158 }
 159
 160 static void broadcast_tlb_mm_a15_erratum(struct mm_struct *mm)
 161 {
 162         int this_cpu;
 163         cpumask_t mask = { CPU_BITS_NONE };
 164
 165         if (!erratum_a15_798181())
 166                 return;
 167
 168         this_cpu = get_cpu();
 169         a15_erratum_get_cpumask(this_cpu, mm, &mask);
 170         smp_call_function_many(&mask, ipi_flush_tlb_a15_erratum, NULL, 1);
 171         put_cpu();
 172 }
 173
 174 void flush_tlb_all(void)
 175 {
 176         if (tlb_ops_need_broadcast())
 177                 on_each_cpu(ipi_flush_tlb_all, NULL, 1);
 178         else
 179                 __flush_tlb_all();
 180         broadcast_tlb_a15_erratum();
 181 }
 182
 183 void flush_tlb_mm(struct mm_struct *mm)
 184 {
 185         if (tlb_ops_need_broadcast())
 186                 on_each_cpu_mask(mm_cpumask(mm), ipi_flush_tlb_mm, mm, 1);
 187         else
 188                 __flush_tlb_mm(mm);
 189         broadcast_tlb_mm_a15_erratum(mm);
 190 }
 191
 192 void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr)
 193 {
 194         if (tlb_ops_need_broadcast()) {
 195                 struct tlb_args ta;
 196                 ta.ta_vma = vma;
 197                 ta.ta_start = uaddr;
 198                 on_each_cpu_mask(mm_cpumask(vma->vm_mm), ipi_flush_tlb_page,
 199                                         &ta, 1);
 200         } else
 201                 __flush_tlb_page(vma, uaddr);
 202         broadcast_tlb_mm_a15_erratum(vma->vm_mm);
 203 }
 204
 205 void flush_tlb_kernel_page(unsigned long kaddr)
 206 {
 207         if (tlb_ops_need_broadcast()) {
 208                 struct tlb_args ta;
 209                 ta.ta_start = kaddr;
 210                 on_each_cpu(ipi_flush_tlb_kernel_page, &ta, 1);
 211         } else
 212                 __flush_tlb_kernel_page(kaddr);
 213         broadcast_tlb_a15_erratum();
 214 }
 215
 216 void flush_tlb_range(struct vm_area_struct *vma,
 217                      unsigned long start, unsigned long end)
 218 {
 219         if (tlb_ops_need_broadcast()) {
 220                 struct tlb_args ta;
 221                 ta.ta_vma = vma;
 222                 ta.ta_start = start;
 223                 ta.ta_end = end;
 224                 on_each_cpu_mask(mm_cpumask(vma->vm_mm), ipi_flush_tlb_range,
 225                                         &ta, 1);
 226         } else
 227                 local_flush_tlb_range(vma, start, end);
 228         broadcast_tlb_mm_a15_erratum(vma->vm_mm);
 229 }
 230
 231 void flush_tlb_kernel_range(unsigned long start, unsigned long end)
 232 {
 233         if (tlb_ops_need_broadcast()) {
 234                 struct tlb_args ta;
 235                 ta.ta_start = start;
 236                 ta.ta_end = end;
 237                 on_each_cpu(ipi_flush_tlb_kernel_range, &ta, 1);
 238         } else
 239                 local_flush_tlb_kernel_range(start, end);
 240         broadcast_tlb_a15_erratum();
 241 }
 242
 243 void flush_bp_all(void)
 244 {
 245         if (tlb_ops_need_broadcast())
 246                 on_each_cpu(ipi_flush_bp_all, NULL, 1);
 247         else
 248                 __flush_bp_all();
 249 }