2 * ARC700 VIPT Cache Management
4 * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com)
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
10 * vineetg: May 2011: for Non-aliasing VIPT D-cache following can be NOPs
11 * -flush_cache_dup_mm (fork)
12 * -likewise for flush_cache_mm (exit/execve)
13 * -likewise for flush_cache_range,flush_cache_page (munmap, exit, COW-break)
16 * -Now that MMU can support larger pg sz (16K), the determiniation of
17 * aliasing shd not be based on assumption of 8k pg
20 * -optimised version of flush_icache_range( ) for making I/D coherent
21 * when vaddr is available (agnostic of num of aliases)
24 * -Added documentation about I-cache aliasing on ARC700 and the way it
25 * was handled up until MMU V2.
26 * -Spotted a three year old bug when killing the 4 aliases, which needs
27 * bottom 2 bits, so we need to do paddr | {0x00, 0x01, 0x02, 0x03}
28 * instead of paddr | {0x00, 0x01, 0x10, 0x11}
29 * (Rajesh you owe me one now)
32 * -Off-by-one error when computing num_of_lines to flush
33 * This broke signal handling with bionic which uses synthetic sigret stub
36 * -GCC can't generate ZOL for core cache flush loops.
37 * Conv them into iterations based as opposed to while (start < end) types
40 * -In I-cache flush routine we used to chk for aliasing for every line INV.
41 * Instead now we setup routines per cache geometry and invoke them
42 * via function pointers.
45 * -Cache Line flush routines used to flush an extra line beyond end addr
46 * because check was while (end >= start) instead of (end > start)
47 * =Some call sites had to work around by doing -1, -4 etc to end param
48 * =Some callers didnt care. This was spec bad in case of INV routines
49 * which would discard valid data (cause of the horrible ext2 bug
52 * vineetg: June 11th 2008: Fixed flush_icache_range( )
53 * -Since ARC700 caches are not coherent (I$ doesnt snoop D$) both need
54 * to be flushed, which it was not doing.
55 * -load_module( ) passes vmalloc addr (Kernel Virtual Addr) to the API,
56 * however ARC cache maintenance OPs require PHY addr. Thus need to do
58 * -Also added optimisation there, that for range > PAGE SIZE we flush the
59 * entire cache in one shot rather than line by line. For e.g. a module
60 * with Code sz 600k, old code flushed 600k worth of cache (line-by-line),
61 * while cache is only 16 or 32k.
64 #include <linux/module.h>
66 #include <linux/sched.h>
67 #include <linux/cache.h>
68 #include <linux/mmu_context.h>
69 #include <linux/syscalls.h>
70 #include <linux/uaccess.h>
71 #include <asm/cacheflush.h>
72 #include <asm/cachectl.h>
73 #include <asm/setup.h>
75 char *arc_cache_mumbojumbo(int cpu_id, char *buf, int len)
78 unsigned int c = smp_processor_id();
80 #define PR_CACHE(p, enb, str) \
83 n += scnprintf(buf + n, len - n, str"\t\t: N/A\n"); \
85 n += scnprintf(buf + n, len - n, \
86 str"\t\t: (%uK) VIPT, %dway set-asc, %ub Line %s\n", \
87 TO_KB((p)->sz), (p)->assoc, (p)->line_len, \
88 enb ? "" : "DISABLED (kernel-build)"); \
91 PR_CACHE(&cpuinfo_arc700[c].icache, __CONFIG_ARC_HAS_ICACHE, "I-Cache");
92 PR_CACHE(&cpuinfo_arc700[c].dcache, __CONFIG_ARC_HAS_DCACHE, "D-Cache");
98 * Read the Cache Build Confuration Registers, Decode them and save into
99 * the cpuinfo structure for later use.
100 * No Validation done here, simply read/convert the BCRs
102 void __cpuinit read_decode_cache_bcr(void)
104 struct bcr_cache ibcr, dbcr;
105 struct cpuinfo_arc_cache *p_ic, *p_dc;
106 unsigned int cpu = smp_processor_id();
108 p_ic = &cpuinfo_arc700[cpu].icache;
109 READ_BCR(ARC_REG_IC_BCR, ibcr);
111 if (ibcr.config == 0x3)
113 p_ic->line_len = 8 << ibcr.line_len;
114 p_ic->sz = 0x200 << ibcr.sz;
115 p_ic->ver = ibcr.ver;
117 p_dc = &cpuinfo_arc700[cpu].dcache;
118 READ_BCR(ARC_REG_DC_BCR, dbcr);
120 if (dbcr.config == 0x2)
122 p_dc->line_len = 16 << dbcr.line_len;
123 p_dc->sz = 0x200 << dbcr.sz;
124 p_dc->ver = dbcr.ver;
128 * 1. Validate the Cache Geomtery (compile time config matches hardware)
129 * 2. If I-cache suffers from aliasing, setup work arounds (difft flush rtn)
130 * (aliasing D-cache configurations are not supported YET)
131 * 3. Enable the Caches, setup default flush mode for D-Cache
132 * 3. Calculate the SHMLBA used by user space
134 void __cpuinit arc_cache_init(void)
137 unsigned int cpu = smp_processor_id();
138 struct cpuinfo_arc_cache *ic = &cpuinfo_arc700[cpu].icache;
139 struct cpuinfo_arc_cache *dc = &cpuinfo_arc700[cpu].dcache;
140 int way_pg_ratio = way_pg_ratio;
143 printk(arc_cache_mumbojumbo(0, str, sizeof(str)));
148 #ifdef CONFIG_ARC_HAS_ICACHE
149 /* 1. Confirm some of I-cache params which Linux assumes */
150 if ((ic->assoc != ARC_ICACHE_WAYS) ||
151 (ic->line_len != ARC_ICACHE_LINE_LEN)) {
152 panic("Cache H/W doesn't match kernel Config");
154 #if (CONFIG_ARC_MMU_VER > 2)
157 panic("Cache ver doesn't match MMU ver\n");
159 /* For ISS - suggest the toggles to use */
160 pr_err("Use -prop=icache_version=3,-prop=dcache_version=3\n");
166 /* Enable/disable I-Cache */
167 temp = read_aux_reg(ARC_REG_IC_CTRL);
169 #ifdef CONFIG_ARC_HAS_ICACHE
170 temp &= ~IC_CTRL_CACHE_DISABLE;
172 temp |= IC_CTRL_CACHE_DISABLE;
175 write_aux_reg(ARC_REG_IC_CTRL, temp);
181 #ifdef CONFIG_ARC_HAS_DCACHE
182 if ((dc->assoc != ARC_DCACHE_WAYS) ||
183 (dc->line_len != ARC_DCACHE_LINE_LEN)) {
184 panic("Cache H/W doesn't match kernel Config");
187 /* check for D-Cache aliasing */
188 if ((dc->sz / ARC_DCACHE_WAYS) > PAGE_SIZE)
189 panic("D$ aliasing not handled right now\n");
192 /* Set the default Invalidate Mode to "simpy discard dirty lines"
193 * as this is more frequent then flush before invalidate
194 * Ofcourse we toggle this default behviour when desired
196 temp = read_aux_reg(ARC_REG_DC_CTRL);
197 temp &= ~DC_CTRL_INV_MODE_FLUSH;
199 #ifdef CONFIG_ARC_HAS_DCACHE
200 /* Enable D-Cache: Clear Bit 0 */
201 write_aux_reg(ARC_REG_DC_CTRL, temp & ~IC_CTRL_CACHE_DISABLE);
204 write_aux_reg(ARC_REG_DC_FLSH, 0x1);
205 /* Disable D cache */
206 write_aux_reg(ARC_REG_DC_CTRL, temp | IC_CTRL_CACHE_DISABLE);
214 #define OP_FLUSH_N_INV 0x3
216 #ifdef CONFIG_ARC_HAS_DCACHE
218 /***************************************************************
219 * Machine specific helpers for Entire D-Cache or Per Line ops
222 static inline void wait_for_flush(void)
224 while (read_aux_reg(ARC_REG_DC_CTRL) & DC_CTRL_FLUSH_STATUS)
229 * Operation on Entire D-Cache
230 * @cacheop = {OP_INV, OP_FLUSH, OP_FLUSH_N_INV}
231 * Note that constant propagation ensures all the checks are gone
234 static inline void __dc_entire_op(const int cacheop)
236 unsigned long flags, tmp = tmp;
239 local_irq_save(flags);
241 if (cacheop == OP_FLUSH_N_INV) {
242 /* Dcache provides 2 cmd: FLUSH or INV
243 * INV inturn has sub-modes: DISCARD or FLUSH-BEFORE
244 * flush-n-inv is achieved by INV cmd but with IM=1
245 * Default INV sub-mode is DISCARD, which needs to be toggled
247 tmp = read_aux_reg(ARC_REG_DC_CTRL);
248 write_aux_reg(ARC_REG_DC_CTRL, tmp | DC_CTRL_INV_MODE_FLUSH);
251 if (cacheop & OP_INV) /* Inv or flush-n-inv use same cmd reg */
252 aux = ARC_REG_DC_IVDC;
254 aux = ARC_REG_DC_FLSH;
256 write_aux_reg(aux, 0x1);
258 if (cacheop & OP_FLUSH) /* flush / flush-n-inv both wait */
261 /* Switch back the DISCARD ONLY Invalidate mode */
262 if (cacheop == OP_FLUSH_N_INV)
263 write_aux_reg(ARC_REG_DC_CTRL, tmp & ~DC_CTRL_INV_MODE_FLUSH);
265 local_irq_restore(flags);
269 * Per Line Operation on D-Cache
270 * Doesn't deal with type-of-op/IRQ-disabling/waiting-for-flush-to-complete
271 * It's sole purpose is to help gcc generate ZOL
273 static inline void __dc_line_loop(unsigned long paddr, unsigned long sz,
278 /* Ensure we properly floor/ceil the non-line aligned/sized requests
279 * and have @paddr - aligned to cache line and integral @num_lines.
280 * This however can be avoided for page sized since:
281 * -@paddr will be cache-line aligned already (being page aligned)
282 * -@sz will be integral multiple of line size (being page sized).
284 if (!(__builtin_constant_p(sz) && sz == PAGE_SIZE)) {
285 sz += paddr & ~DCACHE_LINE_MASK;
286 paddr &= DCACHE_LINE_MASK;
289 num_lines = DIV_ROUND_UP(sz, ARC_DCACHE_LINE_LEN);
291 while (num_lines-- > 0) {
292 #if (CONFIG_ARC_MMU_VER > 2)
294 * Just as for I$, in MMU v3, D$ ops also require
295 * "tag" bits in DC_PTAG, "index" bits in FLDL,IVDL ops
296 * But we pass phy addr for both. This works since Linux
297 * doesn't support aliasing configs for D$, yet.
298 * Thus paddr is enough to provide both tag and index.
300 write_aux_reg(ARC_REG_DC_PTAG, paddr);
302 write_aux_reg(aux_reg, paddr);
303 paddr += ARC_DCACHE_LINE_LEN;
308 * D-Cache : Per Line INV (discard or wback+discard) or FLUSH (wback)
310 static inline void __dc_line_op(unsigned long paddr, unsigned long sz,
313 unsigned long flags, tmp = tmp;
316 local_irq_save(flags);
318 if (cacheop == OP_FLUSH_N_INV) {
320 * Dcache provides 2 cmd: FLUSH or INV
321 * INV inturn has sub-modes: DISCARD or FLUSH-BEFORE
322 * flush-n-inv is achieved by INV cmd but with IM=1
323 * Default INV sub-mode is DISCARD, which needs to be toggled
325 tmp = read_aux_reg(ARC_REG_DC_CTRL);
326 write_aux_reg(ARC_REG_DC_CTRL, tmp | DC_CTRL_INV_MODE_FLUSH);
329 if (cacheop & OP_INV) /* Inv / flush-n-inv use same cmd reg */
330 aux = ARC_REG_DC_IVDL;
332 aux = ARC_REG_DC_FLDL;
334 __dc_line_loop(paddr, sz, aux);
336 if (cacheop & OP_FLUSH) /* flush / flush-n-inv both wait */
339 /* Switch back the DISCARD ONLY Invalidate mode */
340 if (cacheop == OP_FLUSH_N_INV)
341 write_aux_reg(ARC_REG_DC_CTRL, tmp & ~DC_CTRL_INV_MODE_FLUSH);
343 local_irq_restore(flags);
348 #define __dc_entire_op(cacheop)
349 #define __dc_line_op(paddr, sz, cacheop)
351 #endif /* CONFIG_ARC_HAS_DCACHE */
354 #ifdef CONFIG_ARC_HAS_ICACHE
357 * I-Cache Aliasing in ARC700 VIPT caches
359 * ARC VIPT I-cache uses vaddr to index into cache and paddr to match the tag.
360 * The orig Cache Management Module "CDU" only required paddr to invalidate a
361 * certain line since it sufficed as index in Non-Aliasing VIPT cache-geometry.
362 * Infact for distinct V1,V2,P: all of {V1-P},{V2-P},{P-P} would end up fetching
363 * the exact same line.
365 * However for larger Caches (way-size > page-size) - i.e. in Aliasing config,
366 * paddr alone could not be used to correctly index the cache.
369 * MMU v1/v2 (Fixed Page Size 8k)
371 * The solution was to provide CDU with these additonal vaddr bits. These
372 * would be bits [x:13], x would depend on cache-geometry, 13 comes from
373 * standard page size of 8k.
374 * H/w folks chose [17:13] to be a future safe range, and moreso these 5 bits
375 * of vaddr could easily be "stuffed" in the paddr as bits [4:0] since the
376 * orig 5 bits of paddr were anyways ignored by CDU line ops, as they
377 * represent the offset within cache-line. The adv of using this "clumsy"
378 * interface for additional info was no new reg was needed in CDU programming
381 * 17:13 represented the max num of bits passable, actual bits needed were
382 * fewer, based on the num-of-aliases possible.
383 * -for 2 alias possibility, only bit 13 needed (32K cache)
384 * -for 4 alias possibility, bits 14:13 needed (64K cache)
389 * This ver of MMU supports variable page sizes (1k-16k): although Linux will
390 * only support 8k (default), 16k and 4k.
391 * However from hardware perspective, smaller page sizes aggrevate aliasing
392 * meaning more vaddr bits needed to disambiguate the cache-line-op ;
393 * the existing scheme of piggybacking won't work for certain configurations.
394 * Two new registers IC_PTAG and DC_PTAG inttoduced.
395 * "tag" bits are provided in PTAG, index bits in existing IVIL/IVDL/FLDL regs
398 /***********************************************************
399 * Machine specific helper for per line I-Cache invalidate.
401 static void __ic_line_inv_vaddr(unsigned long paddr, unsigned long vaddr,
408 * Ensure we properly floor/ceil the non-line aligned/sized requests:
409 * However page sized flushes can be compile time optimised.
410 * -@paddr will be cache-line aligned already (being page aligned)
411 * -@sz will be integral multiple of line size (being page sized).
413 if (!(__builtin_constant_p(sz) && sz == PAGE_SIZE)) {
414 sz += paddr & ~ICACHE_LINE_MASK;
415 paddr &= ICACHE_LINE_MASK;
416 vaddr &= ICACHE_LINE_MASK;
419 num_lines = DIV_ROUND_UP(sz, ARC_ICACHE_LINE_LEN);
421 #if (CONFIG_ARC_MMU_VER <= 2)
422 /* bits 17:13 of vaddr go as bits 4:0 of paddr */
423 paddr |= (vaddr >> PAGE_SHIFT) & 0x1F;
426 local_irq_save(flags);
427 while (num_lines-- > 0) {
428 #if (CONFIG_ARC_MMU_VER > 2)
429 /* tag comes from phy addr */
430 write_aux_reg(ARC_REG_IC_PTAG, paddr);
432 /* index bits come from vaddr */
433 write_aux_reg(ARC_REG_IC_IVIL, vaddr);
434 vaddr += ARC_ICACHE_LINE_LEN;
436 /* paddr contains stuffed vaddrs bits */
437 write_aux_reg(ARC_REG_IC_IVIL, paddr);
439 paddr += ARC_ICACHE_LINE_LEN;
441 local_irq_restore(flags);
446 #define __ic_line_inv_vaddr(pstart, vstart, sz)
448 #endif /* CONFIG_ARC_HAS_ICACHE */
451 /***********************************************************
455 void flush_dcache_page(struct page *page)
457 /* Make a note that dcache is not yet flushed for this page */
458 set_bit(PG_arch_1, &page->flags);
460 EXPORT_SYMBOL(flush_dcache_page);
463 void dma_cache_wback_inv(unsigned long start, unsigned long sz)
465 __dc_line_op(start, sz, OP_FLUSH_N_INV);
467 EXPORT_SYMBOL(dma_cache_wback_inv);
469 void dma_cache_inv(unsigned long start, unsigned long sz)
471 __dc_line_op(start, sz, OP_INV);
473 EXPORT_SYMBOL(dma_cache_inv);
475 void dma_cache_wback(unsigned long start, unsigned long sz)
477 __dc_line_op(start, sz, OP_FLUSH);
479 EXPORT_SYMBOL(dma_cache_wback);
482 * This is API for making I/D Caches consistent when modifying
483 * kernel code (loadable modules, kprobes, kgdb...)
484 * This is called on insmod, with kernel virtual address for CODE of
485 * the module. ARC cache maintenance ops require PHY address thus we
486 * need to convert vmalloc addr to PHY addr
488 void flush_icache_range(unsigned long kstart, unsigned long kend)
490 unsigned int tot_sz, off, sz;
491 unsigned long phy, pfn;
493 /* printk("Kernel Cache Cohenercy: %lx to %lx\n",kstart, kend); */
495 /* This is not the right API for user virtual address */
496 if (kstart < TASK_SIZE) {
497 BUG_ON("Flush icache range for user virtual addr space");
501 /* Shortcut for bigger flush ranges.
502 * Here we don't care if this was kernel virtual or phy addr
504 tot_sz = kend - kstart;
505 if (tot_sz > PAGE_SIZE) {
510 /* Case: Kernel Phy addr (0x8000_0000 onwards) */
511 if (likely(kstart > PAGE_OFFSET)) {
513 * The 2nd arg despite being paddr will be used to index icache
514 * This is OK since no alternate virtual mappings will exist
515 * given the callers for this case: kprobe/kgdb in built-in
518 __sync_icache_dcache(kstart, kstart, kend - kstart);
523 * Case: Kernel Vaddr (0x7000_0000 to 0x7fff_ffff)
524 * (1) ARC Cache Maintenance ops only take Phy addr, hence special
525 * handling of kernel vaddr.
527 * (2) Despite @tot_sz being < PAGE_SIZE (bigger cases handled already),
528 * it still needs to handle a 2 page scenario, where the range
529 * straddles across 2 virtual pages and hence need for loop
532 off = kstart % PAGE_SIZE;
533 pfn = vmalloc_to_pfn((void *)kstart);
534 phy = (pfn << PAGE_SHIFT) + off;
535 sz = min_t(unsigned int, tot_sz, PAGE_SIZE - off);
536 __sync_icache_dcache(phy, kstart, sz);
543 * General purpose helper to make I and D cache lines consistent.
544 * @paddr is phy addr of region
545 * @vaddr is typically user or kernel vaddr (vmalloc)
546 * Howver in one instance, flush_icache_range() by kprobe (for a breakpt in
547 * builtin kernel code) @vaddr will be paddr only, meaning CDU operation will
548 * use a paddr to index the cache (despite VIPT). This is fine since since a
549 * built-in kernel page will not have any virtual mappings (not even kernel)
550 * kprobe on loadable module is different as it will have kvaddr.
552 void __sync_icache_dcache(unsigned long paddr, unsigned long vaddr, int len)
556 local_irq_save(flags);
557 __ic_line_inv_vaddr(paddr, vaddr, len);
558 __dc_line_op(paddr, len, OP_FLUSH);
559 local_irq_restore(flags);
562 /* wrapper to compile time eliminate alignment checks in flush loop */
563 void __inv_icache_page(unsigned long paddr, unsigned long vaddr)
565 __ic_line_inv_vaddr(paddr, vaddr, PAGE_SIZE);
568 void __flush_dcache_page(unsigned long paddr)
570 __dc_line_op(paddr, PAGE_SIZE, OP_FLUSH_N_INV);
573 void flush_icache_all(void)
577 local_irq_save(flags);
579 write_aux_reg(ARC_REG_IC_IVIC, 1);
581 /* lr will not complete till the icache inv operation is not over */
582 read_aux_reg(ARC_REG_IC_CTRL);
583 local_irq_restore(flags);
586 noinline void flush_cache_all(void)
590 local_irq_save(flags);
593 __dc_entire_op(OP_FLUSH_N_INV);
595 local_irq_restore(flags);
599 /**********************************************************************
600 * Explicit Cache flush request from user space via syscall
601 * Needed for JITs which generate code on the fly
603 SYSCALL_DEFINE3(cacheflush, uint32_t, start, uint32_t, sz, uint32_t, flags)
605 /* TBD: optimize this */