2 * Copyright 2012 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
22 * Authors: Alex Deucher
24 #include <linux/firmware.h>
25 #include <linux/slab.h>
26 #include <linux/module.h>
29 #include "radeon_asic.h"
32 #include "cik_blit_shaders.h"
33 #include "radeon_ucode.h"
34 #include "clearstate_ci.h"
36 MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
37 MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
38 MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
39 MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
40 MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
41 MODULE_FIRMWARE("radeon/BONAIRE_mc2.bin");
42 MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
43 MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
44 MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
46 MODULE_FIRMWARE("radeon/bonaire_pfp.bin");
47 MODULE_FIRMWARE("radeon/bonaire_me.bin");
48 MODULE_FIRMWARE("radeon/bonaire_ce.bin");
49 MODULE_FIRMWARE("radeon/bonaire_mec.bin");
50 MODULE_FIRMWARE("radeon/bonaire_mc.bin");
51 MODULE_FIRMWARE("radeon/bonaire_rlc.bin");
52 MODULE_FIRMWARE("radeon/bonaire_sdma.bin");
53 MODULE_FIRMWARE("radeon/bonaire_smc.bin");
55 MODULE_FIRMWARE("radeon/HAWAII_pfp.bin");
56 MODULE_FIRMWARE("radeon/HAWAII_me.bin");
57 MODULE_FIRMWARE("radeon/HAWAII_ce.bin");
58 MODULE_FIRMWARE("radeon/HAWAII_mec.bin");
59 MODULE_FIRMWARE("radeon/HAWAII_mc.bin");
60 MODULE_FIRMWARE("radeon/HAWAII_mc2.bin");
61 MODULE_FIRMWARE("radeon/HAWAII_rlc.bin");
62 MODULE_FIRMWARE("radeon/HAWAII_sdma.bin");
63 MODULE_FIRMWARE("radeon/HAWAII_smc.bin");
65 MODULE_FIRMWARE("radeon/hawaii_pfp.bin");
66 MODULE_FIRMWARE("radeon/hawaii_me.bin");
67 MODULE_FIRMWARE("radeon/hawaii_ce.bin");
68 MODULE_FIRMWARE("radeon/hawaii_mec.bin");
69 MODULE_FIRMWARE("radeon/hawaii_mc.bin");
70 MODULE_FIRMWARE("radeon/hawaii_rlc.bin");
71 MODULE_FIRMWARE("radeon/hawaii_sdma.bin");
72 MODULE_FIRMWARE("radeon/hawaii_smc.bin");
74 MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
75 MODULE_FIRMWARE("radeon/KAVERI_me.bin");
76 MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
77 MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
78 MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
79 MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
81 MODULE_FIRMWARE("radeon/kaveri_pfp.bin");
82 MODULE_FIRMWARE("radeon/kaveri_me.bin");
83 MODULE_FIRMWARE("radeon/kaveri_ce.bin");
84 MODULE_FIRMWARE("radeon/kaveri_mec.bin");
85 MODULE_FIRMWARE("radeon/kaveri_mec2.bin");
86 MODULE_FIRMWARE("radeon/kaveri_rlc.bin");
87 MODULE_FIRMWARE("radeon/kaveri_sdma.bin");
89 MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
90 MODULE_FIRMWARE("radeon/KABINI_me.bin");
91 MODULE_FIRMWARE("radeon/KABINI_ce.bin");
92 MODULE_FIRMWARE("radeon/KABINI_mec.bin");
93 MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
94 MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
96 MODULE_FIRMWARE("radeon/kabini_pfp.bin");
97 MODULE_FIRMWARE("radeon/kabini_me.bin");
98 MODULE_FIRMWARE("radeon/kabini_ce.bin");
99 MODULE_FIRMWARE("radeon/kabini_mec.bin");
100 MODULE_FIRMWARE("radeon/kabini_rlc.bin");
101 MODULE_FIRMWARE("radeon/kabini_sdma.bin");
103 MODULE_FIRMWARE("radeon/MULLINS_pfp.bin");
104 MODULE_FIRMWARE("radeon/MULLINS_me.bin");
105 MODULE_FIRMWARE("radeon/MULLINS_ce.bin");
106 MODULE_FIRMWARE("radeon/MULLINS_mec.bin");
107 MODULE_FIRMWARE("radeon/MULLINS_rlc.bin");
108 MODULE_FIRMWARE("radeon/MULLINS_sdma.bin");
110 MODULE_FIRMWARE("radeon/mullins_pfp.bin");
111 MODULE_FIRMWARE("radeon/mullins_me.bin");
112 MODULE_FIRMWARE("radeon/mullins_ce.bin");
113 MODULE_FIRMWARE("radeon/mullins_mec.bin");
114 MODULE_FIRMWARE("radeon/mullins_rlc.bin");
115 MODULE_FIRMWARE("radeon/mullins_sdma.bin");
117 extern int r600_ih_ring_alloc(struct radeon_device *rdev);
118 extern void r600_ih_ring_fini(struct radeon_device *rdev);
119 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
120 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
121 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
122 extern void sumo_rlc_fini(struct radeon_device *rdev);
123 extern int sumo_rlc_init(struct radeon_device *rdev);
124 extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
125 extern void si_rlc_reset(struct radeon_device *rdev);
126 extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
127 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
128 extern int cik_sdma_resume(struct radeon_device *rdev);
129 extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
130 extern void cik_sdma_fini(struct radeon_device *rdev);
131 extern void vce_v2_0_enable_mgcg(struct radeon_device *rdev, bool enable);
132 static void cik_rlc_stop(struct radeon_device *rdev);
133 static void cik_pcie_gen3_enable(struct radeon_device *rdev);
134 static void cik_program_aspm(struct radeon_device *rdev);
135 static void cik_init_pg(struct radeon_device *rdev);
136 static void cik_init_cg(struct radeon_device *rdev);
137 static void cik_fini_pg(struct radeon_device *rdev);
138 static void cik_fini_cg(struct radeon_device *rdev);
139 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
142 /* get temperature in millidegrees */
143 int ci_get_temp(struct radeon_device *rdev)
148 temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
154 actual_temp = temp & 0x1ff;
156 actual_temp = actual_temp * 1000;
161 /* get temperature in millidegrees */
162 int kv_get_temp(struct radeon_device *rdev)
167 temp = RREG32_SMC(0xC0300E0C);
170 actual_temp = (temp / 8) - 49;
174 actual_temp = actual_temp * 1000;
180 * Indirect registers accessor
182 u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
187 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
188 WREG32(PCIE_INDEX, reg);
189 (void)RREG32(PCIE_INDEX);
190 r = RREG32(PCIE_DATA);
191 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
195 void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
199 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
200 WREG32(PCIE_INDEX, reg);
201 (void)RREG32(PCIE_INDEX);
202 WREG32(PCIE_DATA, v);
203 (void)RREG32(PCIE_DATA);
204 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
207 static const u32 spectre_rlc_save_restore_register_list[] =
209 (0x0e00 << 16) | (0xc12c >> 2),
211 (0x0e00 << 16) | (0xc140 >> 2),
213 (0x0e00 << 16) | (0xc150 >> 2),
215 (0x0e00 << 16) | (0xc15c >> 2),
217 (0x0e00 << 16) | (0xc168 >> 2),
219 (0x0e00 << 16) | (0xc170 >> 2),
221 (0x0e00 << 16) | (0xc178 >> 2),
223 (0x0e00 << 16) | (0xc204 >> 2),
225 (0x0e00 << 16) | (0xc2b4 >> 2),
227 (0x0e00 << 16) | (0xc2b8 >> 2),
229 (0x0e00 << 16) | (0xc2bc >> 2),
231 (0x0e00 << 16) | (0xc2c0 >> 2),
233 (0x0e00 << 16) | (0x8228 >> 2),
235 (0x0e00 << 16) | (0x829c >> 2),
237 (0x0e00 << 16) | (0x869c >> 2),
239 (0x0600 << 16) | (0x98f4 >> 2),
241 (0x0e00 << 16) | (0x98f8 >> 2),
243 (0x0e00 << 16) | (0x9900 >> 2),
245 (0x0e00 << 16) | (0xc260 >> 2),
247 (0x0e00 << 16) | (0x90e8 >> 2),
249 (0x0e00 << 16) | (0x3c000 >> 2),
251 (0x0e00 << 16) | (0x3c00c >> 2),
253 (0x0e00 << 16) | (0x8c1c >> 2),
255 (0x0e00 << 16) | (0x9700 >> 2),
257 (0x0e00 << 16) | (0xcd20 >> 2),
259 (0x4e00 << 16) | (0xcd20 >> 2),
261 (0x5e00 << 16) | (0xcd20 >> 2),
263 (0x6e00 << 16) | (0xcd20 >> 2),
265 (0x7e00 << 16) | (0xcd20 >> 2),
267 (0x8e00 << 16) | (0xcd20 >> 2),
269 (0x9e00 << 16) | (0xcd20 >> 2),
271 (0xae00 << 16) | (0xcd20 >> 2),
273 (0xbe00 << 16) | (0xcd20 >> 2),
275 (0x0e00 << 16) | (0x89bc >> 2),
277 (0x0e00 << 16) | (0x8900 >> 2),
280 (0x0e00 << 16) | (0xc130 >> 2),
282 (0x0e00 << 16) | (0xc134 >> 2),
284 (0x0e00 << 16) | (0xc1fc >> 2),
286 (0x0e00 << 16) | (0xc208 >> 2),
288 (0x0e00 << 16) | (0xc264 >> 2),
290 (0x0e00 << 16) | (0xc268 >> 2),
292 (0x0e00 << 16) | (0xc26c >> 2),
294 (0x0e00 << 16) | (0xc270 >> 2),
296 (0x0e00 << 16) | (0xc274 >> 2),
298 (0x0e00 << 16) | (0xc278 >> 2),
300 (0x0e00 << 16) | (0xc27c >> 2),
302 (0x0e00 << 16) | (0xc280 >> 2),
304 (0x0e00 << 16) | (0xc284 >> 2),
306 (0x0e00 << 16) | (0xc288 >> 2),
308 (0x0e00 << 16) | (0xc28c >> 2),
310 (0x0e00 << 16) | (0xc290 >> 2),
312 (0x0e00 << 16) | (0xc294 >> 2),
314 (0x0e00 << 16) | (0xc298 >> 2),
316 (0x0e00 << 16) | (0xc29c >> 2),
318 (0x0e00 << 16) | (0xc2a0 >> 2),
320 (0x0e00 << 16) | (0xc2a4 >> 2),
322 (0x0e00 << 16) | (0xc2a8 >> 2),
324 (0x0e00 << 16) | (0xc2ac >> 2),
326 (0x0e00 << 16) | (0xc2b0 >> 2),
328 (0x0e00 << 16) | (0x301d0 >> 2),
330 (0x0e00 << 16) | (0x30238 >> 2),
332 (0x0e00 << 16) | (0x30250 >> 2),
334 (0x0e00 << 16) | (0x30254 >> 2),
336 (0x0e00 << 16) | (0x30258 >> 2),
338 (0x0e00 << 16) | (0x3025c >> 2),
340 (0x4e00 << 16) | (0xc900 >> 2),
342 (0x5e00 << 16) | (0xc900 >> 2),
344 (0x6e00 << 16) | (0xc900 >> 2),
346 (0x7e00 << 16) | (0xc900 >> 2),
348 (0x8e00 << 16) | (0xc900 >> 2),
350 (0x9e00 << 16) | (0xc900 >> 2),
352 (0xae00 << 16) | (0xc900 >> 2),
354 (0xbe00 << 16) | (0xc900 >> 2),
356 (0x4e00 << 16) | (0xc904 >> 2),
358 (0x5e00 << 16) | (0xc904 >> 2),
360 (0x6e00 << 16) | (0xc904 >> 2),
362 (0x7e00 << 16) | (0xc904 >> 2),
364 (0x8e00 << 16) | (0xc904 >> 2),
366 (0x9e00 << 16) | (0xc904 >> 2),
368 (0xae00 << 16) | (0xc904 >> 2),
370 (0xbe00 << 16) | (0xc904 >> 2),
372 (0x4e00 << 16) | (0xc908 >> 2),
374 (0x5e00 << 16) | (0xc908 >> 2),
376 (0x6e00 << 16) | (0xc908 >> 2),
378 (0x7e00 << 16) | (0xc908 >> 2),
380 (0x8e00 << 16) | (0xc908 >> 2),
382 (0x9e00 << 16) | (0xc908 >> 2),
384 (0xae00 << 16) | (0xc908 >> 2),
386 (0xbe00 << 16) | (0xc908 >> 2),
388 (0x4e00 << 16) | (0xc90c >> 2),
390 (0x5e00 << 16) | (0xc90c >> 2),
392 (0x6e00 << 16) | (0xc90c >> 2),
394 (0x7e00 << 16) | (0xc90c >> 2),
396 (0x8e00 << 16) | (0xc90c >> 2),
398 (0x9e00 << 16) | (0xc90c >> 2),
400 (0xae00 << 16) | (0xc90c >> 2),
402 (0xbe00 << 16) | (0xc90c >> 2),
404 (0x4e00 << 16) | (0xc910 >> 2),
406 (0x5e00 << 16) | (0xc910 >> 2),
408 (0x6e00 << 16) | (0xc910 >> 2),
410 (0x7e00 << 16) | (0xc910 >> 2),
412 (0x8e00 << 16) | (0xc910 >> 2),
414 (0x9e00 << 16) | (0xc910 >> 2),
416 (0xae00 << 16) | (0xc910 >> 2),
418 (0xbe00 << 16) | (0xc910 >> 2),
420 (0x0e00 << 16) | (0xc99c >> 2),
422 (0x0e00 << 16) | (0x9834 >> 2),
424 (0x0000 << 16) | (0x30f00 >> 2),
426 (0x0001 << 16) | (0x30f00 >> 2),
428 (0x0000 << 16) | (0x30f04 >> 2),
430 (0x0001 << 16) | (0x30f04 >> 2),
432 (0x0000 << 16) | (0x30f08 >> 2),
434 (0x0001 << 16) | (0x30f08 >> 2),
436 (0x0000 << 16) | (0x30f0c >> 2),
438 (0x0001 << 16) | (0x30f0c >> 2),
440 (0x0600 << 16) | (0x9b7c >> 2),
442 (0x0e00 << 16) | (0x8a14 >> 2),
444 (0x0e00 << 16) | (0x8a18 >> 2),
446 (0x0600 << 16) | (0x30a00 >> 2),
448 (0x0e00 << 16) | (0x8bf0 >> 2),
450 (0x0e00 << 16) | (0x8bcc >> 2),
452 (0x0e00 << 16) | (0x8b24 >> 2),
454 (0x0e00 << 16) | (0x30a04 >> 2),
456 (0x0600 << 16) | (0x30a10 >> 2),
458 (0x0600 << 16) | (0x30a14 >> 2),
460 (0x0600 << 16) | (0x30a18 >> 2),
462 (0x0600 << 16) | (0x30a2c >> 2),
464 (0x0e00 << 16) | (0xc700 >> 2),
466 (0x0e00 << 16) | (0xc704 >> 2),
468 (0x0e00 << 16) | (0xc708 >> 2),
470 (0x0e00 << 16) | (0xc768 >> 2),
472 (0x0400 << 16) | (0xc770 >> 2),
474 (0x0400 << 16) | (0xc774 >> 2),
476 (0x0400 << 16) | (0xc778 >> 2),
478 (0x0400 << 16) | (0xc77c >> 2),
480 (0x0400 << 16) | (0xc780 >> 2),
482 (0x0400 << 16) | (0xc784 >> 2),
484 (0x0400 << 16) | (0xc788 >> 2),
486 (0x0400 << 16) | (0xc78c >> 2),
488 (0x0400 << 16) | (0xc798 >> 2),
490 (0x0400 << 16) | (0xc79c >> 2),
492 (0x0400 << 16) | (0xc7a0 >> 2),
494 (0x0400 << 16) | (0xc7a4 >> 2),
496 (0x0400 << 16) | (0xc7a8 >> 2),
498 (0x0400 << 16) | (0xc7ac >> 2),
500 (0x0400 << 16) | (0xc7b0 >> 2),
502 (0x0400 << 16) | (0xc7b4 >> 2),
504 (0x0e00 << 16) | (0x9100 >> 2),
506 (0x0e00 << 16) | (0x3c010 >> 2),
508 (0x0e00 << 16) | (0x92a8 >> 2),
510 (0x0e00 << 16) | (0x92ac >> 2),
512 (0x0e00 << 16) | (0x92b4 >> 2),
514 (0x0e00 << 16) | (0x92b8 >> 2),
516 (0x0e00 << 16) | (0x92bc >> 2),
518 (0x0e00 << 16) | (0x92c0 >> 2),
520 (0x0e00 << 16) | (0x92c4 >> 2),
522 (0x0e00 << 16) | (0x92c8 >> 2),
524 (0x0e00 << 16) | (0x92cc >> 2),
526 (0x0e00 << 16) | (0x92d0 >> 2),
528 (0x0e00 << 16) | (0x8c00 >> 2),
530 (0x0e00 << 16) | (0x8c04 >> 2),
532 (0x0e00 << 16) | (0x8c20 >> 2),
534 (0x0e00 << 16) | (0x8c38 >> 2),
536 (0x0e00 << 16) | (0x8c3c >> 2),
538 (0x0e00 << 16) | (0xae00 >> 2),
540 (0x0e00 << 16) | (0x9604 >> 2),
542 (0x0e00 << 16) | (0xac08 >> 2),
544 (0x0e00 << 16) | (0xac0c >> 2),
546 (0x0e00 << 16) | (0xac10 >> 2),
548 (0x0e00 << 16) | (0xac14 >> 2),
550 (0x0e00 << 16) | (0xac58 >> 2),
552 (0x0e00 << 16) | (0xac68 >> 2),
554 (0x0e00 << 16) | (0xac6c >> 2),
556 (0x0e00 << 16) | (0xac70 >> 2),
558 (0x0e00 << 16) | (0xac74 >> 2),
560 (0x0e00 << 16) | (0xac78 >> 2),
562 (0x0e00 << 16) | (0xac7c >> 2),
564 (0x0e00 << 16) | (0xac80 >> 2),
566 (0x0e00 << 16) | (0xac84 >> 2),
568 (0x0e00 << 16) | (0xac88 >> 2),
570 (0x0e00 << 16) | (0xac8c >> 2),
572 (0x0e00 << 16) | (0x970c >> 2),
574 (0x0e00 << 16) | (0x9714 >> 2),
576 (0x0e00 << 16) | (0x9718 >> 2),
578 (0x0e00 << 16) | (0x971c >> 2),
580 (0x0e00 << 16) | (0x31068 >> 2),
582 (0x4e00 << 16) | (0x31068 >> 2),
584 (0x5e00 << 16) | (0x31068 >> 2),
586 (0x6e00 << 16) | (0x31068 >> 2),
588 (0x7e00 << 16) | (0x31068 >> 2),
590 (0x8e00 << 16) | (0x31068 >> 2),
592 (0x9e00 << 16) | (0x31068 >> 2),
594 (0xae00 << 16) | (0x31068 >> 2),
596 (0xbe00 << 16) | (0x31068 >> 2),
598 (0x0e00 << 16) | (0xcd10 >> 2),
600 (0x0e00 << 16) | (0xcd14 >> 2),
602 (0x0e00 << 16) | (0x88b0 >> 2),
604 (0x0e00 << 16) | (0x88b4 >> 2),
606 (0x0e00 << 16) | (0x88b8 >> 2),
608 (0x0e00 << 16) | (0x88bc >> 2),
610 (0x0400 << 16) | (0x89c0 >> 2),
612 (0x0e00 << 16) | (0x88c4 >> 2),
614 (0x0e00 << 16) | (0x88c8 >> 2),
616 (0x0e00 << 16) | (0x88d0 >> 2),
618 (0x0e00 << 16) | (0x88d4 >> 2),
620 (0x0e00 << 16) | (0x88d8 >> 2),
622 (0x0e00 << 16) | (0x8980 >> 2),
624 (0x0e00 << 16) | (0x30938 >> 2),
626 (0x0e00 << 16) | (0x3093c >> 2),
628 (0x0e00 << 16) | (0x30940 >> 2),
630 (0x0e00 << 16) | (0x89a0 >> 2),
632 (0x0e00 << 16) | (0x30900 >> 2),
634 (0x0e00 << 16) | (0x30904 >> 2),
636 (0x0e00 << 16) | (0x89b4 >> 2),
638 (0x0e00 << 16) | (0x3c210 >> 2),
640 (0x0e00 << 16) | (0x3c214 >> 2),
642 (0x0e00 << 16) | (0x3c218 >> 2),
644 (0x0e00 << 16) | (0x8904 >> 2),
647 (0x0e00 << 16) | (0x8c28 >> 2),
648 (0x0e00 << 16) | (0x8c2c >> 2),
649 (0x0e00 << 16) | (0x8c30 >> 2),
650 (0x0e00 << 16) | (0x8c34 >> 2),
651 (0x0e00 << 16) | (0x9600 >> 2),
654 static const u32 kalindi_rlc_save_restore_register_list[] =
656 (0x0e00 << 16) | (0xc12c >> 2),
658 (0x0e00 << 16) | (0xc140 >> 2),
660 (0x0e00 << 16) | (0xc150 >> 2),
662 (0x0e00 << 16) | (0xc15c >> 2),
664 (0x0e00 << 16) | (0xc168 >> 2),
666 (0x0e00 << 16) | (0xc170 >> 2),
668 (0x0e00 << 16) | (0xc204 >> 2),
670 (0x0e00 << 16) | (0xc2b4 >> 2),
672 (0x0e00 << 16) | (0xc2b8 >> 2),
674 (0x0e00 << 16) | (0xc2bc >> 2),
676 (0x0e00 << 16) | (0xc2c0 >> 2),
678 (0x0e00 << 16) | (0x8228 >> 2),
680 (0x0e00 << 16) | (0x829c >> 2),
682 (0x0e00 << 16) | (0x869c >> 2),
684 (0x0600 << 16) | (0x98f4 >> 2),
686 (0x0e00 << 16) | (0x98f8 >> 2),
688 (0x0e00 << 16) | (0x9900 >> 2),
690 (0x0e00 << 16) | (0xc260 >> 2),
692 (0x0e00 << 16) | (0x90e8 >> 2),
694 (0x0e00 << 16) | (0x3c000 >> 2),
696 (0x0e00 << 16) | (0x3c00c >> 2),
698 (0x0e00 << 16) | (0x8c1c >> 2),
700 (0x0e00 << 16) | (0x9700 >> 2),
702 (0x0e00 << 16) | (0xcd20 >> 2),
704 (0x4e00 << 16) | (0xcd20 >> 2),
706 (0x5e00 << 16) | (0xcd20 >> 2),
708 (0x6e00 << 16) | (0xcd20 >> 2),
710 (0x7e00 << 16) | (0xcd20 >> 2),
712 (0x0e00 << 16) | (0x89bc >> 2),
714 (0x0e00 << 16) | (0x8900 >> 2),
717 (0x0e00 << 16) | (0xc130 >> 2),
719 (0x0e00 << 16) | (0xc134 >> 2),
721 (0x0e00 << 16) | (0xc1fc >> 2),
723 (0x0e00 << 16) | (0xc208 >> 2),
725 (0x0e00 << 16) | (0xc264 >> 2),
727 (0x0e00 << 16) | (0xc268 >> 2),
729 (0x0e00 << 16) | (0xc26c >> 2),
731 (0x0e00 << 16) | (0xc270 >> 2),
733 (0x0e00 << 16) | (0xc274 >> 2),
735 (0x0e00 << 16) | (0xc28c >> 2),
737 (0x0e00 << 16) | (0xc290 >> 2),
739 (0x0e00 << 16) | (0xc294 >> 2),
741 (0x0e00 << 16) | (0xc298 >> 2),
743 (0x0e00 << 16) | (0xc2a0 >> 2),
745 (0x0e00 << 16) | (0xc2a4 >> 2),
747 (0x0e00 << 16) | (0xc2a8 >> 2),
749 (0x0e00 << 16) | (0xc2ac >> 2),
751 (0x0e00 << 16) | (0x301d0 >> 2),
753 (0x0e00 << 16) | (0x30238 >> 2),
755 (0x0e00 << 16) | (0x30250 >> 2),
757 (0x0e00 << 16) | (0x30254 >> 2),
759 (0x0e00 << 16) | (0x30258 >> 2),
761 (0x0e00 << 16) | (0x3025c >> 2),
763 (0x4e00 << 16) | (0xc900 >> 2),
765 (0x5e00 << 16) | (0xc900 >> 2),
767 (0x6e00 << 16) | (0xc900 >> 2),
769 (0x7e00 << 16) | (0xc900 >> 2),
771 (0x4e00 << 16) | (0xc904 >> 2),
773 (0x5e00 << 16) | (0xc904 >> 2),
775 (0x6e00 << 16) | (0xc904 >> 2),
777 (0x7e00 << 16) | (0xc904 >> 2),
779 (0x4e00 << 16) | (0xc908 >> 2),
781 (0x5e00 << 16) | (0xc908 >> 2),
783 (0x6e00 << 16) | (0xc908 >> 2),
785 (0x7e00 << 16) | (0xc908 >> 2),
787 (0x4e00 << 16) | (0xc90c >> 2),
789 (0x5e00 << 16) | (0xc90c >> 2),
791 (0x6e00 << 16) | (0xc90c >> 2),
793 (0x7e00 << 16) | (0xc90c >> 2),
795 (0x4e00 << 16) | (0xc910 >> 2),
797 (0x5e00 << 16) | (0xc910 >> 2),
799 (0x6e00 << 16) | (0xc910 >> 2),
801 (0x7e00 << 16) | (0xc910 >> 2),
803 (0x0e00 << 16) | (0xc99c >> 2),
805 (0x0e00 << 16) | (0x9834 >> 2),
807 (0x0000 << 16) | (0x30f00 >> 2),
809 (0x0000 << 16) | (0x30f04 >> 2),
811 (0x0000 << 16) | (0x30f08 >> 2),
813 (0x0000 << 16) | (0x30f0c >> 2),
815 (0x0600 << 16) | (0x9b7c >> 2),
817 (0x0e00 << 16) | (0x8a14 >> 2),
819 (0x0e00 << 16) | (0x8a18 >> 2),
821 (0x0600 << 16) | (0x30a00 >> 2),
823 (0x0e00 << 16) | (0x8bf0 >> 2),
825 (0x0e00 << 16) | (0x8bcc >> 2),
827 (0x0e00 << 16) | (0x8b24 >> 2),
829 (0x0e00 << 16) | (0x30a04 >> 2),
831 (0x0600 << 16) | (0x30a10 >> 2),
833 (0x0600 << 16) | (0x30a14 >> 2),
835 (0x0600 << 16) | (0x30a18 >> 2),
837 (0x0600 << 16) | (0x30a2c >> 2),
839 (0x0e00 << 16) | (0xc700 >> 2),
841 (0x0e00 << 16) | (0xc704 >> 2),
843 (0x0e00 << 16) | (0xc708 >> 2),
845 (0x0e00 << 16) | (0xc768 >> 2),
847 (0x0400 << 16) | (0xc770 >> 2),
849 (0x0400 << 16) | (0xc774 >> 2),
851 (0x0400 << 16) | (0xc798 >> 2),
853 (0x0400 << 16) | (0xc79c >> 2),
855 (0x0e00 << 16) | (0x9100 >> 2),
857 (0x0e00 << 16) | (0x3c010 >> 2),
859 (0x0e00 << 16) | (0x8c00 >> 2),
861 (0x0e00 << 16) | (0x8c04 >> 2),
863 (0x0e00 << 16) | (0x8c20 >> 2),
865 (0x0e00 << 16) | (0x8c38 >> 2),
867 (0x0e00 << 16) | (0x8c3c >> 2),
869 (0x0e00 << 16) | (0xae00 >> 2),
871 (0x0e00 << 16) | (0x9604 >> 2),
873 (0x0e00 << 16) | (0xac08 >> 2),
875 (0x0e00 << 16) | (0xac0c >> 2),
877 (0x0e00 << 16) | (0xac10 >> 2),
879 (0x0e00 << 16) | (0xac14 >> 2),
881 (0x0e00 << 16) | (0xac58 >> 2),
883 (0x0e00 << 16) | (0xac68 >> 2),
885 (0x0e00 << 16) | (0xac6c >> 2),
887 (0x0e00 << 16) | (0xac70 >> 2),
889 (0x0e00 << 16) | (0xac74 >> 2),
891 (0x0e00 << 16) | (0xac78 >> 2),
893 (0x0e00 << 16) | (0xac7c >> 2),
895 (0x0e00 << 16) | (0xac80 >> 2),
897 (0x0e00 << 16) | (0xac84 >> 2),
899 (0x0e00 << 16) | (0xac88 >> 2),
901 (0x0e00 << 16) | (0xac8c >> 2),
903 (0x0e00 << 16) | (0x970c >> 2),
905 (0x0e00 << 16) | (0x9714 >> 2),
907 (0x0e00 << 16) | (0x9718 >> 2),
909 (0x0e00 << 16) | (0x971c >> 2),
911 (0x0e00 << 16) | (0x31068 >> 2),
913 (0x4e00 << 16) | (0x31068 >> 2),
915 (0x5e00 << 16) | (0x31068 >> 2),
917 (0x6e00 << 16) | (0x31068 >> 2),
919 (0x7e00 << 16) | (0x31068 >> 2),
921 (0x0e00 << 16) | (0xcd10 >> 2),
923 (0x0e00 << 16) | (0xcd14 >> 2),
925 (0x0e00 << 16) | (0x88b0 >> 2),
927 (0x0e00 << 16) | (0x88b4 >> 2),
929 (0x0e00 << 16) | (0x88b8 >> 2),
931 (0x0e00 << 16) | (0x88bc >> 2),
933 (0x0400 << 16) | (0x89c0 >> 2),
935 (0x0e00 << 16) | (0x88c4 >> 2),
937 (0x0e00 << 16) | (0x88c8 >> 2),
939 (0x0e00 << 16) | (0x88d0 >> 2),
941 (0x0e00 << 16) | (0x88d4 >> 2),
943 (0x0e00 << 16) | (0x88d8 >> 2),
945 (0x0e00 << 16) | (0x8980 >> 2),
947 (0x0e00 << 16) | (0x30938 >> 2),
949 (0x0e00 << 16) | (0x3093c >> 2),
951 (0x0e00 << 16) | (0x30940 >> 2),
953 (0x0e00 << 16) | (0x89a0 >> 2),
955 (0x0e00 << 16) | (0x30900 >> 2),
957 (0x0e00 << 16) | (0x30904 >> 2),
959 (0x0e00 << 16) | (0x89b4 >> 2),
961 (0x0e00 << 16) | (0x3e1fc >> 2),
963 (0x0e00 << 16) | (0x3c210 >> 2),
965 (0x0e00 << 16) | (0x3c214 >> 2),
967 (0x0e00 << 16) | (0x3c218 >> 2),
969 (0x0e00 << 16) | (0x8904 >> 2),
972 (0x0e00 << 16) | (0x8c28 >> 2),
973 (0x0e00 << 16) | (0x8c2c >> 2),
974 (0x0e00 << 16) | (0x8c30 >> 2),
975 (0x0e00 << 16) | (0x8c34 >> 2),
976 (0x0e00 << 16) | (0x9600 >> 2),
979 static const u32 bonaire_golden_spm_registers[] =
981 0x30800, 0xe0ffffff, 0xe0000000
984 static const u32 bonaire_golden_common_registers[] =
986 0xc770, 0xffffffff, 0x00000800,
987 0xc774, 0xffffffff, 0x00000800,
988 0xc798, 0xffffffff, 0x00007fbf,
989 0xc79c, 0xffffffff, 0x00007faf
992 static const u32 bonaire_golden_registers[] =
994 0x3354, 0x00000333, 0x00000333,
995 0x3350, 0x000c0fc0, 0x00040200,
996 0x9a10, 0x00010000, 0x00058208,
997 0x3c000, 0xffff1fff, 0x00140000,
998 0x3c200, 0xfdfc0fff, 0x00000100,
999 0x3c234, 0x40000000, 0x40000200,
1000 0x9830, 0xffffffff, 0x00000000,
1001 0x9834, 0xf00fffff, 0x00000400,
1002 0x9838, 0x0002021c, 0x00020200,
1003 0xc78, 0x00000080, 0x00000000,
1004 0x5bb0, 0x000000f0, 0x00000070,
1005 0x5bc0, 0xf0311fff, 0x80300000,
1006 0x98f8, 0x73773777, 0x12010001,
1007 0x350c, 0x00810000, 0x408af000,
1008 0x7030, 0x31000111, 0x00000011,
1009 0x2f48, 0x73773777, 0x12010001,
1010 0x220c, 0x00007fb6, 0x0021a1b1,
1011 0x2210, 0x00007fb6, 0x002021b1,
1012 0x2180, 0x00007fb6, 0x00002191,
1013 0x2218, 0x00007fb6, 0x002121b1,
1014 0x221c, 0x00007fb6, 0x002021b1,
1015 0x21dc, 0x00007fb6, 0x00002191,
1016 0x21e0, 0x00007fb6, 0x00002191,
1017 0x3628, 0x0000003f, 0x0000000a,
1018 0x362c, 0x0000003f, 0x0000000a,
1019 0x2ae4, 0x00073ffe, 0x000022a2,
1020 0x240c, 0x000007ff, 0x00000000,
1021 0x8a14, 0xf000003f, 0x00000007,
1022 0x8bf0, 0x00002001, 0x00000001,
1023 0x8b24, 0xffffffff, 0x00ffffff,
1024 0x30a04, 0x0000ff0f, 0x00000000,
1025 0x28a4c, 0x07ffffff, 0x06000000,
1026 0x4d8, 0x00000fff, 0x00000100,
1027 0x3e78, 0x00000001, 0x00000002,
1028 0x9100, 0x03000000, 0x0362c688,
1029 0x8c00, 0x000000ff, 0x00000001,
1030 0xe40, 0x00001fff, 0x00001fff,
1031 0x9060, 0x0000007f, 0x00000020,
1032 0x9508, 0x00010000, 0x00010000,
1033 0xac14, 0x000003ff, 0x000000f3,
1034 0xac0c, 0xffffffff, 0x00001032
1037 static const u32 bonaire_mgcg_cgcg_init[] =
1039 0xc420, 0xffffffff, 0xfffffffc,
1040 0x30800, 0xffffffff, 0xe0000000,
1041 0x3c2a0, 0xffffffff, 0x00000100,
1042 0x3c208, 0xffffffff, 0x00000100,
1043 0x3c2c0, 0xffffffff, 0xc0000100,
1044 0x3c2c8, 0xffffffff, 0xc0000100,
1045 0x3c2c4, 0xffffffff, 0xc0000100,
1046 0x55e4, 0xffffffff, 0x00600100,
1047 0x3c280, 0xffffffff, 0x00000100,
1048 0x3c214, 0xffffffff, 0x06000100,
1049 0x3c220, 0xffffffff, 0x00000100,
1050 0x3c218, 0xffffffff, 0x06000100,
1051 0x3c204, 0xffffffff, 0x00000100,
1052 0x3c2e0, 0xffffffff, 0x00000100,
1053 0x3c224, 0xffffffff, 0x00000100,
1054 0x3c200, 0xffffffff, 0x00000100,
1055 0x3c230, 0xffffffff, 0x00000100,
1056 0x3c234, 0xffffffff, 0x00000100,
1057 0x3c250, 0xffffffff, 0x00000100,
1058 0x3c254, 0xffffffff, 0x00000100,
1059 0x3c258, 0xffffffff, 0x00000100,
1060 0x3c25c, 0xffffffff, 0x00000100,
1061 0x3c260, 0xffffffff, 0x00000100,
1062 0x3c27c, 0xffffffff, 0x00000100,
1063 0x3c278, 0xffffffff, 0x00000100,
1064 0x3c210, 0xffffffff, 0x06000100,
1065 0x3c290, 0xffffffff, 0x00000100,
1066 0x3c274, 0xffffffff, 0x00000100,
1067 0x3c2b4, 0xffffffff, 0x00000100,
1068 0x3c2b0, 0xffffffff, 0x00000100,
1069 0x3c270, 0xffffffff, 0x00000100,
1070 0x30800, 0xffffffff, 0xe0000000,
1071 0x3c020, 0xffffffff, 0x00010000,
1072 0x3c024, 0xffffffff, 0x00030002,
1073 0x3c028, 0xffffffff, 0x00040007,
1074 0x3c02c, 0xffffffff, 0x00060005,
1075 0x3c030, 0xffffffff, 0x00090008,
1076 0x3c034, 0xffffffff, 0x00010000,
1077 0x3c038, 0xffffffff, 0x00030002,
1078 0x3c03c, 0xffffffff, 0x00040007,
1079 0x3c040, 0xffffffff, 0x00060005,
1080 0x3c044, 0xffffffff, 0x00090008,
1081 0x3c048, 0xffffffff, 0x00010000,
1082 0x3c04c, 0xffffffff, 0x00030002,
1083 0x3c050, 0xffffffff, 0x00040007,
1084 0x3c054, 0xffffffff, 0x00060005,
1085 0x3c058, 0xffffffff, 0x00090008,
1086 0x3c05c, 0xffffffff, 0x00010000,
1087 0x3c060, 0xffffffff, 0x00030002,
1088 0x3c064, 0xffffffff, 0x00040007,
1089 0x3c068, 0xffffffff, 0x00060005,
1090 0x3c06c, 0xffffffff, 0x00090008,
1091 0x3c070, 0xffffffff, 0x00010000,
1092 0x3c074, 0xffffffff, 0x00030002,
1093 0x3c078, 0xffffffff, 0x00040007,
1094 0x3c07c, 0xffffffff, 0x00060005,
1095 0x3c080, 0xffffffff, 0x00090008,
1096 0x3c084, 0xffffffff, 0x00010000,
1097 0x3c088, 0xffffffff, 0x00030002,
1098 0x3c08c, 0xffffffff, 0x00040007,
1099 0x3c090, 0xffffffff, 0x00060005,
1100 0x3c094, 0xffffffff, 0x00090008,
1101 0x3c098, 0xffffffff, 0x00010000,
1102 0x3c09c, 0xffffffff, 0x00030002,
1103 0x3c0a0, 0xffffffff, 0x00040007,
1104 0x3c0a4, 0xffffffff, 0x00060005,
1105 0x3c0a8, 0xffffffff, 0x00090008,
1106 0x3c000, 0xffffffff, 0x96e00200,
1107 0x8708, 0xffffffff, 0x00900100,
1108 0xc424, 0xffffffff, 0x0020003f,
1109 0x38, 0xffffffff, 0x0140001c,
1110 0x3c, 0x000f0000, 0x000f0000,
1111 0x220, 0xffffffff, 0xC060000C,
1112 0x224, 0xc0000fff, 0x00000100,
1113 0xf90, 0xffffffff, 0x00000100,
1114 0xf98, 0x00000101, 0x00000000,
1115 0x20a8, 0xffffffff, 0x00000104,
1116 0x55e4, 0xff000fff, 0x00000100,
1117 0x30cc, 0xc0000fff, 0x00000104,
1118 0xc1e4, 0x00000001, 0x00000001,
1119 0xd00c, 0xff000ff0, 0x00000100,
1120 0xd80c, 0xff000ff0, 0x00000100
1123 static const u32 spectre_golden_spm_registers[] =
1125 0x30800, 0xe0ffffff, 0xe0000000
1128 static const u32 spectre_golden_common_registers[] =
1130 0xc770, 0xffffffff, 0x00000800,
1131 0xc774, 0xffffffff, 0x00000800,
1132 0xc798, 0xffffffff, 0x00007fbf,
1133 0xc79c, 0xffffffff, 0x00007faf
1136 static const u32 spectre_golden_registers[] =
1138 0x3c000, 0xffff1fff, 0x96940200,
1139 0x3c00c, 0xffff0001, 0xff000000,
1140 0x3c200, 0xfffc0fff, 0x00000100,
1141 0x6ed8, 0x00010101, 0x00010000,
1142 0x9834, 0xf00fffff, 0x00000400,
1143 0x9838, 0xfffffffc, 0x00020200,
1144 0x5bb0, 0x000000f0, 0x00000070,
1145 0x5bc0, 0xf0311fff, 0x80300000,
1146 0x98f8, 0x73773777, 0x12010001,
1147 0x9b7c, 0x00ff0000, 0x00fc0000,
1148 0x2f48, 0x73773777, 0x12010001,
1149 0x8a14, 0xf000003f, 0x00000007,
1150 0x8b24, 0xffffffff, 0x00ffffff,
1151 0x28350, 0x3f3f3fff, 0x00000082,
1152 0x28354, 0x0000003f, 0x00000000,
1153 0x3e78, 0x00000001, 0x00000002,
1154 0x913c, 0xffff03df, 0x00000004,
1155 0xc768, 0x00000008, 0x00000008,
1156 0x8c00, 0x000008ff, 0x00000800,
1157 0x9508, 0x00010000, 0x00010000,
1158 0xac0c, 0xffffffff, 0x54763210,
1159 0x214f8, 0x01ff01ff, 0x00000002,
1160 0x21498, 0x007ff800, 0x00200000,
1161 0x2015c, 0xffffffff, 0x00000f40,
1162 0x30934, 0xffffffff, 0x00000001
1165 static const u32 spectre_mgcg_cgcg_init[] =
1167 0xc420, 0xffffffff, 0xfffffffc,
1168 0x30800, 0xffffffff, 0xe0000000,
1169 0x3c2a0, 0xffffffff, 0x00000100,
1170 0x3c208, 0xffffffff, 0x00000100,
1171 0x3c2c0, 0xffffffff, 0x00000100,
1172 0x3c2c8, 0xffffffff, 0x00000100,
1173 0x3c2c4, 0xffffffff, 0x00000100,
1174 0x55e4, 0xffffffff, 0x00600100,
1175 0x3c280, 0xffffffff, 0x00000100,
1176 0x3c214, 0xffffffff, 0x06000100,
1177 0x3c220, 0xffffffff, 0x00000100,
1178 0x3c218, 0xffffffff, 0x06000100,
1179 0x3c204, 0xffffffff, 0x00000100,
1180 0x3c2e0, 0xffffffff, 0x00000100,
1181 0x3c224, 0xffffffff, 0x00000100,
1182 0x3c200, 0xffffffff, 0x00000100,
1183 0x3c230, 0xffffffff, 0x00000100,
1184 0x3c234, 0xffffffff, 0x00000100,
1185 0x3c250, 0xffffffff, 0x00000100,
1186 0x3c254, 0xffffffff, 0x00000100,
1187 0x3c258, 0xffffffff, 0x00000100,
1188 0x3c25c, 0xffffffff, 0x00000100,
1189 0x3c260, 0xffffffff, 0x00000100,
1190 0x3c27c, 0xffffffff, 0x00000100,
1191 0x3c278, 0xffffffff, 0x00000100,
1192 0x3c210, 0xffffffff, 0x06000100,
1193 0x3c290, 0xffffffff, 0x00000100,
1194 0x3c274, 0xffffffff, 0x00000100,
1195 0x3c2b4, 0xffffffff, 0x00000100,
1196 0x3c2b0, 0xffffffff, 0x00000100,
1197 0x3c270, 0xffffffff, 0x00000100,
1198 0x30800, 0xffffffff, 0xe0000000,
1199 0x3c020, 0xffffffff, 0x00010000,
1200 0x3c024, 0xffffffff, 0x00030002,
1201 0x3c028, 0xffffffff, 0x00040007,
1202 0x3c02c, 0xffffffff, 0x00060005,
1203 0x3c030, 0xffffffff, 0x00090008,
1204 0x3c034, 0xffffffff, 0x00010000,
1205 0x3c038, 0xffffffff, 0x00030002,
1206 0x3c03c, 0xffffffff, 0x00040007,
1207 0x3c040, 0xffffffff, 0x00060005,
1208 0x3c044, 0xffffffff, 0x00090008,
1209 0x3c048, 0xffffffff, 0x00010000,
1210 0x3c04c, 0xffffffff, 0x00030002,
1211 0x3c050, 0xffffffff, 0x00040007,
1212 0x3c054, 0xffffffff, 0x00060005,
1213 0x3c058, 0xffffffff, 0x00090008,
1214 0x3c05c, 0xffffffff, 0x00010000,
1215 0x3c060, 0xffffffff, 0x00030002,
1216 0x3c064, 0xffffffff, 0x00040007,
1217 0x3c068, 0xffffffff, 0x00060005,
1218 0x3c06c, 0xffffffff, 0x00090008,
1219 0x3c070, 0xffffffff, 0x00010000,
1220 0x3c074, 0xffffffff, 0x00030002,
1221 0x3c078, 0xffffffff, 0x00040007,
1222 0x3c07c, 0xffffffff, 0x00060005,
1223 0x3c080, 0xffffffff, 0x00090008,
1224 0x3c084, 0xffffffff, 0x00010000,
1225 0x3c088, 0xffffffff, 0x00030002,
1226 0x3c08c, 0xffffffff, 0x00040007,
1227 0x3c090, 0xffffffff, 0x00060005,
1228 0x3c094, 0xffffffff, 0x00090008,
1229 0x3c098, 0xffffffff, 0x00010000,
1230 0x3c09c, 0xffffffff, 0x00030002,
1231 0x3c0a0, 0xffffffff, 0x00040007,
1232 0x3c0a4, 0xffffffff, 0x00060005,
1233 0x3c0a8, 0xffffffff, 0x00090008,
1234 0x3c0ac, 0xffffffff, 0x00010000,
1235 0x3c0b0, 0xffffffff, 0x00030002,
1236 0x3c0b4, 0xffffffff, 0x00040007,
1237 0x3c0b8, 0xffffffff, 0x00060005,
1238 0x3c0bc, 0xffffffff, 0x00090008,
1239 0x3c000, 0xffffffff, 0x96e00200,
1240 0x8708, 0xffffffff, 0x00900100,
1241 0xc424, 0xffffffff, 0x0020003f,
1242 0x38, 0xffffffff, 0x0140001c,
1243 0x3c, 0x000f0000, 0x000f0000,
1244 0x220, 0xffffffff, 0xC060000C,
1245 0x224, 0xc0000fff, 0x00000100,
1246 0xf90, 0xffffffff, 0x00000100,
1247 0xf98, 0x00000101, 0x00000000,
1248 0x20a8, 0xffffffff, 0x00000104,
1249 0x55e4, 0xff000fff, 0x00000100,
1250 0x30cc, 0xc0000fff, 0x00000104,
1251 0xc1e4, 0x00000001, 0x00000001,
1252 0xd00c, 0xff000ff0, 0x00000100,
1253 0xd80c, 0xff000ff0, 0x00000100
1256 static const u32 kalindi_golden_spm_registers[] =
1258 0x30800, 0xe0ffffff, 0xe0000000
1261 static const u32 kalindi_golden_common_registers[] =
1263 0xc770, 0xffffffff, 0x00000800,
1264 0xc774, 0xffffffff, 0x00000800,
1265 0xc798, 0xffffffff, 0x00007fbf,
1266 0xc79c, 0xffffffff, 0x00007faf
1269 static const u32 kalindi_golden_registers[] =
1271 0x3c000, 0xffffdfff, 0x6e944040,
1272 0x55e4, 0xff607fff, 0xfc000100,
1273 0x3c220, 0xff000fff, 0x00000100,
1274 0x3c224, 0xff000fff, 0x00000100,
1275 0x3c200, 0xfffc0fff, 0x00000100,
1276 0x6ed8, 0x00010101, 0x00010000,
1277 0x9830, 0xffffffff, 0x00000000,
1278 0x9834, 0xf00fffff, 0x00000400,
1279 0x5bb0, 0x000000f0, 0x00000070,
1280 0x5bc0, 0xf0311fff, 0x80300000,
1281 0x98f8, 0x73773777, 0x12010001,
1282 0x98fc, 0xffffffff, 0x00000010,
1283 0x9b7c, 0x00ff0000, 0x00fc0000,
1284 0x8030, 0x00001f0f, 0x0000100a,
1285 0x2f48, 0x73773777, 0x12010001,
1286 0x2408, 0x000fffff, 0x000c007f,
1287 0x8a14, 0xf000003f, 0x00000007,
1288 0x8b24, 0x3fff3fff, 0x00ffcfff,
1289 0x30a04, 0x0000ff0f, 0x00000000,
1290 0x28a4c, 0x07ffffff, 0x06000000,
1291 0x4d8, 0x00000fff, 0x00000100,
1292 0x3e78, 0x00000001, 0x00000002,
1293 0xc768, 0x00000008, 0x00000008,
1294 0x8c00, 0x000000ff, 0x00000003,
1295 0x214f8, 0x01ff01ff, 0x00000002,
1296 0x21498, 0x007ff800, 0x00200000,
1297 0x2015c, 0xffffffff, 0x00000f40,
1298 0x88c4, 0x001f3ae3, 0x00000082,
1299 0x88d4, 0x0000001f, 0x00000010,
1300 0x30934, 0xffffffff, 0x00000000
1303 static const u32 kalindi_mgcg_cgcg_init[] =
1305 0xc420, 0xffffffff, 0xfffffffc,
1306 0x30800, 0xffffffff, 0xe0000000,
1307 0x3c2a0, 0xffffffff, 0x00000100,
1308 0x3c208, 0xffffffff, 0x00000100,
1309 0x3c2c0, 0xffffffff, 0x00000100,
1310 0x3c2c8, 0xffffffff, 0x00000100,
1311 0x3c2c4, 0xffffffff, 0x00000100,
1312 0x55e4, 0xffffffff, 0x00600100,
1313 0x3c280, 0xffffffff, 0x00000100,
1314 0x3c214, 0xffffffff, 0x06000100,
1315 0x3c220, 0xffffffff, 0x00000100,
1316 0x3c218, 0xffffffff, 0x06000100,
1317 0x3c204, 0xffffffff, 0x00000100,
1318 0x3c2e0, 0xffffffff, 0x00000100,
1319 0x3c224, 0xffffffff, 0x00000100,
1320 0x3c200, 0xffffffff, 0x00000100,
1321 0x3c230, 0xffffffff, 0x00000100,
1322 0x3c234, 0xffffffff, 0x00000100,
1323 0x3c250, 0xffffffff, 0x00000100,
1324 0x3c254, 0xffffffff, 0x00000100,
1325 0x3c258, 0xffffffff, 0x00000100,
1326 0x3c25c, 0xffffffff, 0x00000100,
1327 0x3c260, 0xffffffff, 0x00000100,
1328 0x3c27c, 0xffffffff, 0x00000100,
1329 0x3c278, 0xffffffff, 0x00000100,
1330 0x3c210, 0xffffffff, 0x06000100,
1331 0x3c290, 0xffffffff, 0x00000100,
1332 0x3c274, 0xffffffff, 0x00000100,
1333 0x3c2b4, 0xffffffff, 0x00000100,
1334 0x3c2b0, 0xffffffff, 0x00000100,
1335 0x3c270, 0xffffffff, 0x00000100,
1336 0x30800, 0xffffffff, 0xe0000000,
1337 0x3c020, 0xffffffff, 0x00010000,
1338 0x3c024, 0xffffffff, 0x00030002,
1339 0x3c028, 0xffffffff, 0x00040007,
1340 0x3c02c, 0xffffffff, 0x00060005,
1341 0x3c030, 0xffffffff, 0x00090008,
1342 0x3c034, 0xffffffff, 0x00010000,
1343 0x3c038, 0xffffffff, 0x00030002,
1344 0x3c03c, 0xffffffff, 0x00040007,
1345 0x3c040, 0xffffffff, 0x00060005,
1346 0x3c044, 0xffffffff, 0x00090008,
1347 0x3c000, 0xffffffff, 0x96e00200,
1348 0x8708, 0xffffffff, 0x00900100,
1349 0xc424, 0xffffffff, 0x0020003f,
1350 0x38, 0xffffffff, 0x0140001c,
1351 0x3c, 0x000f0000, 0x000f0000,
1352 0x220, 0xffffffff, 0xC060000C,
1353 0x224, 0xc0000fff, 0x00000100,
1354 0x20a8, 0xffffffff, 0x00000104,
1355 0x55e4, 0xff000fff, 0x00000100,
1356 0x30cc, 0xc0000fff, 0x00000104,
1357 0xc1e4, 0x00000001, 0x00000001,
1358 0xd00c, 0xff000ff0, 0x00000100,
1359 0xd80c, 0xff000ff0, 0x00000100
1362 static const u32 hawaii_golden_spm_registers[] =
1364 0x30800, 0xe0ffffff, 0xe0000000
1367 static const u32 hawaii_golden_common_registers[] =
1369 0x30800, 0xffffffff, 0xe0000000,
1370 0x28350, 0xffffffff, 0x3a00161a,
1371 0x28354, 0xffffffff, 0x0000002e,
1372 0x9a10, 0xffffffff, 0x00018208,
1373 0x98f8, 0xffffffff, 0x12011003
1376 static const u32 hawaii_golden_registers[] =
1378 0x3354, 0x00000333, 0x00000333,
1379 0x9a10, 0x00010000, 0x00058208,
1380 0x9830, 0xffffffff, 0x00000000,
1381 0x9834, 0xf00fffff, 0x00000400,
1382 0x9838, 0x0002021c, 0x00020200,
1383 0xc78, 0x00000080, 0x00000000,
1384 0x5bb0, 0x000000f0, 0x00000070,
1385 0x5bc0, 0xf0311fff, 0x80300000,
1386 0x350c, 0x00810000, 0x408af000,
1387 0x7030, 0x31000111, 0x00000011,
1388 0x2f48, 0x73773777, 0x12010001,
1389 0x2120, 0x0000007f, 0x0000001b,
1390 0x21dc, 0x00007fb6, 0x00002191,
1391 0x3628, 0x0000003f, 0x0000000a,
1392 0x362c, 0x0000003f, 0x0000000a,
1393 0x2ae4, 0x00073ffe, 0x000022a2,
1394 0x240c, 0x000007ff, 0x00000000,
1395 0x8bf0, 0x00002001, 0x00000001,
1396 0x8b24, 0xffffffff, 0x00ffffff,
1397 0x30a04, 0x0000ff0f, 0x00000000,
1398 0x28a4c, 0x07ffffff, 0x06000000,
1399 0x3e78, 0x00000001, 0x00000002,
1400 0xc768, 0x00000008, 0x00000008,
1401 0xc770, 0x00000f00, 0x00000800,
1402 0xc774, 0x00000f00, 0x00000800,
1403 0xc798, 0x00ffffff, 0x00ff7fbf,
1404 0xc79c, 0x00ffffff, 0x00ff7faf,
1405 0x8c00, 0x000000ff, 0x00000800,
1406 0xe40, 0x00001fff, 0x00001fff,
1407 0x9060, 0x0000007f, 0x00000020,
1408 0x9508, 0x00010000, 0x00010000,
1409 0xae00, 0x00100000, 0x000ff07c,
1410 0xac14, 0x000003ff, 0x0000000f,
1411 0xac10, 0xffffffff, 0x7564fdec,
1412 0xac0c, 0xffffffff, 0x3120b9a8,
1413 0xac08, 0x20000000, 0x0f9c0000
1416 static const u32 hawaii_mgcg_cgcg_init[] =
1418 0xc420, 0xffffffff, 0xfffffffd,
1419 0x30800, 0xffffffff, 0xe0000000,
1420 0x3c2a0, 0xffffffff, 0x00000100,
1421 0x3c208, 0xffffffff, 0x00000100,
1422 0x3c2c0, 0xffffffff, 0x00000100,
1423 0x3c2c8, 0xffffffff, 0x00000100,
1424 0x3c2c4, 0xffffffff, 0x00000100,
1425 0x55e4, 0xffffffff, 0x00200100,
1426 0x3c280, 0xffffffff, 0x00000100,
1427 0x3c214, 0xffffffff, 0x06000100,
1428 0x3c220, 0xffffffff, 0x00000100,
1429 0x3c218, 0xffffffff, 0x06000100,
1430 0x3c204, 0xffffffff, 0x00000100,
1431 0x3c2e0, 0xffffffff, 0x00000100,
1432 0x3c224, 0xffffffff, 0x00000100,
1433 0x3c200, 0xffffffff, 0x00000100,
1434 0x3c230, 0xffffffff, 0x00000100,
1435 0x3c234, 0xffffffff, 0x00000100,
1436 0x3c250, 0xffffffff, 0x00000100,
1437 0x3c254, 0xffffffff, 0x00000100,
1438 0x3c258, 0xffffffff, 0x00000100,
1439 0x3c25c, 0xffffffff, 0x00000100,
1440 0x3c260, 0xffffffff, 0x00000100,
1441 0x3c27c, 0xffffffff, 0x00000100,
1442 0x3c278, 0xffffffff, 0x00000100,
1443 0x3c210, 0xffffffff, 0x06000100,
1444 0x3c290, 0xffffffff, 0x00000100,
1445 0x3c274, 0xffffffff, 0x00000100,
1446 0x3c2b4, 0xffffffff, 0x00000100,
1447 0x3c2b0, 0xffffffff, 0x00000100,
1448 0x3c270, 0xffffffff, 0x00000100,
1449 0x30800, 0xffffffff, 0xe0000000,
1450 0x3c020, 0xffffffff, 0x00010000,
1451 0x3c024, 0xffffffff, 0x00030002,
1452 0x3c028, 0xffffffff, 0x00040007,
1453 0x3c02c, 0xffffffff, 0x00060005,
1454 0x3c030, 0xffffffff, 0x00090008,
1455 0x3c034, 0xffffffff, 0x00010000,
1456 0x3c038, 0xffffffff, 0x00030002,
1457 0x3c03c, 0xffffffff, 0x00040007,
1458 0x3c040, 0xffffffff, 0x00060005,
1459 0x3c044, 0xffffffff, 0x00090008,
1460 0x3c048, 0xffffffff, 0x00010000,
1461 0x3c04c, 0xffffffff, 0x00030002,
1462 0x3c050, 0xffffffff, 0x00040007,
1463 0x3c054, 0xffffffff, 0x00060005,
1464 0x3c058, 0xffffffff, 0x00090008,
1465 0x3c05c, 0xffffffff, 0x00010000,
1466 0x3c060, 0xffffffff, 0x00030002,
1467 0x3c064, 0xffffffff, 0x00040007,
1468 0x3c068, 0xffffffff, 0x00060005,
1469 0x3c06c, 0xffffffff, 0x00090008,
1470 0x3c070, 0xffffffff, 0x00010000,
1471 0x3c074, 0xffffffff, 0x00030002,
1472 0x3c078, 0xffffffff, 0x00040007,
1473 0x3c07c, 0xffffffff, 0x00060005,
1474 0x3c080, 0xffffffff, 0x00090008,
1475 0x3c084, 0xffffffff, 0x00010000,
1476 0x3c088, 0xffffffff, 0x00030002,
1477 0x3c08c, 0xffffffff, 0x00040007,
1478 0x3c090, 0xffffffff, 0x00060005,
1479 0x3c094, 0xffffffff, 0x00090008,
1480 0x3c098, 0xffffffff, 0x00010000,
1481 0x3c09c, 0xffffffff, 0x00030002,
1482 0x3c0a0, 0xffffffff, 0x00040007,
1483 0x3c0a4, 0xffffffff, 0x00060005,
1484 0x3c0a8, 0xffffffff, 0x00090008,
1485 0x3c0ac, 0xffffffff, 0x00010000,
1486 0x3c0b0, 0xffffffff, 0x00030002,
1487 0x3c0b4, 0xffffffff, 0x00040007,
1488 0x3c0b8, 0xffffffff, 0x00060005,
1489 0x3c0bc, 0xffffffff, 0x00090008,
1490 0x3c0c0, 0xffffffff, 0x00010000,
1491 0x3c0c4, 0xffffffff, 0x00030002,
1492 0x3c0c8, 0xffffffff, 0x00040007,
1493 0x3c0cc, 0xffffffff, 0x00060005,
1494 0x3c0d0, 0xffffffff, 0x00090008,
1495 0x3c0d4, 0xffffffff, 0x00010000,
1496 0x3c0d8, 0xffffffff, 0x00030002,
1497 0x3c0dc, 0xffffffff, 0x00040007,
1498 0x3c0e0, 0xffffffff, 0x00060005,
1499 0x3c0e4, 0xffffffff, 0x00090008,
1500 0x3c0e8, 0xffffffff, 0x00010000,
1501 0x3c0ec, 0xffffffff, 0x00030002,
1502 0x3c0f0, 0xffffffff, 0x00040007,
1503 0x3c0f4, 0xffffffff, 0x00060005,
1504 0x3c0f8, 0xffffffff, 0x00090008,
1505 0xc318, 0xffffffff, 0x00020200,
1506 0x3350, 0xffffffff, 0x00000200,
1507 0x15c0, 0xffffffff, 0x00000400,
1508 0x55e8, 0xffffffff, 0x00000000,
1509 0x2f50, 0xffffffff, 0x00000902,
1510 0x3c000, 0xffffffff, 0x96940200,
1511 0x8708, 0xffffffff, 0x00900100,
1512 0xc424, 0xffffffff, 0x0020003f,
1513 0x38, 0xffffffff, 0x0140001c,
1514 0x3c, 0x000f0000, 0x000f0000,
1515 0x220, 0xffffffff, 0xc060000c,
1516 0x224, 0xc0000fff, 0x00000100,
1517 0xf90, 0xffffffff, 0x00000100,
1518 0xf98, 0x00000101, 0x00000000,
1519 0x20a8, 0xffffffff, 0x00000104,
1520 0x55e4, 0xff000fff, 0x00000100,
1521 0x30cc, 0xc0000fff, 0x00000104,
1522 0xc1e4, 0x00000001, 0x00000001,
1523 0xd00c, 0xff000ff0, 0x00000100,
1524 0xd80c, 0xff000ff0, 0x00000100
1527 static const u32 godavari_golden_registers[] =
1529 0x55e4, 0xff607fff, 0xfc000100,
1530 0x6ed8, 0x00010101, 0x00010000,
1531 0x9830, 0xffffffff, 0x00000000,
1532 0x98302, 0xf00fffff, 0x00000400,
1533 0x6130, 0xffffffff, 0x00010000,
1534 0x5bb0, 0x000000f0, 0x00000070,
1535 0x5bc0, 0xf0311fff, 0x80300000,
1536 0x98f8, 0x73773777, 0x12010001,
1537 0x98fc, 0xffffffff, 0x00000010,
1538 0x8030, 0x00001f0f, 0x0000100a,
1539 0x2f48, 0x73773777, 0x12010001,
1540 0x2408, 0x000fffff, 0x000c007f,
1541 0x8a14, 0xf000003f, 0x00000007,
1542 0x8b24, 0xffffffff, 0x00ff0fff,
1543 0x30a04, 0x0000ff0f, 0x00000000,
1544 0x28a4c, 0x07ffffff, 0x06000000,
1545 0x4d8, 0x00000fff, 0x00000100,
1546 0xd014, 0x00010000, 0x00810001,
1547 0xd814, 0x00010000, 0x00810001,
1548 0x3e78, 0x00000001, 0x00000002,
1549 0xc768, 0x00000008, 0x00000008,
1550 0xc770, 0x00000f00, 0x00000800,
1551 0xc774, 0x00000f00, 0x00000800,
1552 0xc798, 0x00ffffff, 0x00ff7fbf,
1553 0xc79c, 0x00ffffff, 0x00ff7faf,
1554 0x8c00, 0x000000ff, 0x00000001,
1555 0x214f8, 0x01ff01ff, 0x00000002,
1556 0x21498, 0x007ff800, 0x00200000,
1557 0x2015c, 0xffffffff, 0x00000f40,
1558 0x88c4, 0x001f3ae3, 0x00000082,
1559 0x88d4, 0x0000001f, 0x00000010,
1560 0x30934, 0xffffffff, 0x00000000
1564 static void cik_init_golden_registers(struct radeon_device *rdev)
1566 switch (rdev->family) {
1568 radeon_program_register_sequence(rdev,
1569 bonaire_mgcg_cgcg_init,
1570 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
1571 radeon_program_register_sequence(rdev,
1572 bonaire_golden_registers,
1573 (const u32)ARRAY_SIZE(bonaire_golden_registers));
1574 radeon_program_register_sequence(rdev,
1575 bonaire_golden_common_registers,
1576 (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
1577 radeon_program_register_sequence(rdev,
1578 bonaire_golden_spm_registers,
1579 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
1582 radeon_program_register_sequence(rdev,
1583 kalindi_mgcg_cgcg_init,
1584 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1585 radeon_program_register_sequence(rdev,
1586 kalindi_golden_registers,
1587 (const u32)ARRAY_SIZE(kalindi_golden_registers));
1588 radeon_program_register_sequence(rdev,
1589 kalindi_golden_common_registers,
1590 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1591 radeon_program_register_sequence(rdev,
1592 kalindi_golden_spm_registers,
1593 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1596 radeon_program_register_sequence(rdev,
1597 kalindi_mgcg_cgcg_init,
1598 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1599 radeon_program_register_sequence(rdev,
1600 godavari_golden_registers,
1601 (const u32)ARRAY_SIZE(godavari_golden_registers));
1602 radeon_program_register_sequence(rdev,
1603 kalindi_golden_common_registers,
1604 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1605 radeon_program_register_sequence(rdev,
1606 kalindi_golden_spm_registers,
1607 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1610 radeon_program_register_sequence(rdev,
1611 spectre_mgcg_cgcg_init,
1612 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
1613 radeon_program_register_sequence(rdev,
1614 spectre_golden_registers,
1615 (const u32)ARRAY_SIZE(spectre_golden_registers));
1616 radeon_program_register_sequence(rdev,
1617 spectre_golden_common_registers,
1618 (const u32)ARRAY_SIZE(spectre_golden_common_registers));
1619 radeon_program_register_sequence(rdev,
1620 spectre_golden_spm_registers,
1621 (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
1624 radeon_program_register_sequence(rdev,
1625 hawaii_mgcg_cgcg_init,
1626 (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
1627 radeon_program_register_sequence(rdev,
1628 hawaii_golden_registers,
1629 (const u32)ARRAY_SIZE(hawaii_golden_registers));
1630 radeon_program_register_sequence(rdev,
1631 hawaii_golden_common_registers,
1632 (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
1633 radeon_program_register_sequence(rdev,
1634 hawaii_golden_spm_registers,
1635 (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
1643 * cik_get_xclk - get the xclk
1645 * @rdev: radeon_device pointer
1647 * Returns the reference clock used by the gfx engine
1650 u32 cik_get_xclk(struct radeon_device *rdev)
1652 u32 reference_clock = rdev->clock.spll.reference_freq;
1654 if (rdev->flags & RADEON_IS_IGP) {
1655 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
1656 return reference_clock / 2;
1658 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
1659 return reference_clock / 4;
1661 return reference_clock;
1665 * cik_mm_rdoorbell - read a doorbell dword
1667 * @rdev: radeon_device pointer
1668 * @index: doorbell index
1670 * Returns the value in the doorbell aperture at the
1671 * requested doorbell index (CIK).
1673 u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index)
1675 if (index < rdev->doorbell.num_doorbells) {
1676 return readl(rdev->doorbell.ptr + index);
1678 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
1684 * cik_mm_wdoorbell - write a doorbell dword
1686 * @rdev: radeon_device pointer
1687 * @index: doorbell index
1688 * @v: value to write
1690 * Writes @v to the doorbell aperture at the
1691 * requested doorbell index (CIK).
1693 void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v)
1695 if (index < rdev->doorbell.num_doorbells) {
1696 writel(v, rdev->doorbell.ptr + index);
1698 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
1702 #define BONAIRE_IO_MC_REGS_SIZE 36
1704 static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
1706 {0x00000070, 0x04400000},
1707 {0x00000071, 0x80c01803},
1708 {0x00000072, 0x00004004},
1709 {0x00000073, 0x00000100},
1710 {0x00000074, 0x00ff0000},
1711 {0x00000075, 0x34000000},
1712 {0x00000076, 0x08000014},
1713 {0x00000077, 0x00cc08ec},
1714 {0x00000078, 0x00000400},
1715 {0x00000079, 0x00000000},
1716 {0x0000007a, 0x04090000},
1717 {0x0000007c, 0x00000000},
1718 {0x0000007e, 0x4408a8e8},
1719 {0x0000007f, 0x00000304},
1720 {0x00000080, 0x00000000},
1721 {0x00000082, 0x00000001},
1722 {0x00000083, 0x00000002},
1723 {0x00000084, 0xf3e4f400},
1724 {0x00000085, 0x052024e3},
1725 {0x00000087, 0x00000000},
1726 {0x00000088, 0x01000000},
1727 {0x0000008a, 0x1c0a0000},
1728 {0x0000008b, 0xff010000},
1729 {0x0000008d, 0xffffefff},
1730 {0x0000008e, 0xfff3efff},
1731 {0x0000008f, 0xfff3efbf},
1732 {0x00000092, 0xf7ffffff},
1733 {0x00000093, 0xffffff7f},
1734 {0x00000095, 0x00101101},
1735 {0x00000096, 0x00000fff},
1736 {0x00000097, 0x00116fff},
1737 {0x00000098, 0x60010000},
1738 {0x00000099, 0x10010000},
1739 {0x0000009a, 0x00006000},
1740 {0x0000009b, 0x00001000},
1741 {0x0000009f, 0x00b48000}
1744 #define HAWAII_IO_MC_REGS_SIZE 22
1746 static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] =
1748 {0x0000007d, 0x40000000},
1749 {0x0000007e, 0x40180304},
1750 {0x0000007f, 0x0000ff00},
1751 {0x00000081, 0x00000000},
1752 {0x00000083, 0x00000800},
1753 {0x00000086, 0x00000000},
1754 {0x00000087, 0x00000100},
1755 {0x00000088, 0x00020100},
1756 {0x00000089, 0x00000000},
1757 {0x0000008b, 0x00040000},
1758 {0x0000008c, 0x00000100},
1759 {0x0000008e, 0xff010000},
1760 {0x00000090, 0xffffefff},
1761 {0x00000091, 0xfff3efff},
1762 {0x00000092, 0xfff3efbf},
1763 {0x00000093, 0xf7ffffff},
1764 {0x00000094, 0xffffff7f},
1765 {0x00000095, 0x00000fff},
1766 {0x00000096, 0x00116fff},
1767 {0x00000097, 0x60010000},
1768 {0x00000098, 0x10010000},
1769 {0x0000009f, 0x00c79000}
1774 * cik_srbm_select - select specific register instances
1776 * @rdev: radeon_device pointer
1777 * @me: selected ME (micro engine)
1782 * Switches the currently active registers instances. Some
1783 * registers are instanced per VMID, others are instanced per
1784 * me/pipe/queue combination.
1786 static void cik_srbm_select(struct radeon_device *rdev,
1787 u32 me, u32 pipe, u32 queue, u32 vmid)
1789 u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
1792 QUEUEID(queue & 0x7));
1793 WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
1798 * ci_mc_load_microcode - load MC ucode into the hw
1800 * @rdev: radeon_device pointer
1802 * Load the GDDR MC ucode into the hw (CIK).
1803 * Returns 0 on success, error on failure.
1805 int ci_mc_load_microcode(struct radeon_device *rdev)
1807 const __be32 *fw_data = NULL;
1808 const __le32 *new_fw_data = NULL;
1809 u32 running, blackout = 0;
1810 u32 *io_mc_regs = NULL;
1811 const __le32 *new_io_mc_regs = NULL;
1812 int i, regs_size, ucode_size;
1818 const struct mc_firmware_header_v1_0 *hdr =
1819 (const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data;
1821 radeon_ucode_print_mc_hdr(&hdr->header);
1823 regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
1824 new_io_mc_regs = (const __le32 *)
1825 (rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
1826 ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
1827 new_fw_data = (const __le32 *)
1828 (rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1830 ucode_size = rdev->mc_fw->size / 4;
1832 switch (rdev->family) {
1834 io_mc_regs = (u32 *)&bonaire_io_mc_regs;
1835 regs_size = BONAIRE_IO_MC_REGS_SIZE;
1838 io_mc_regs = (u32 *)&hawaii_io_mc_regs;
1839 regs_size = HAWAII_IO_MC_REGS_SIZE;
1844 fw_data = (const __be32 *)rdev->mc_fw->data;
1847 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1851 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1852 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1855 /* reset the engine and set to writable */
1856 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1857 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1859 /* load mc io regs */
1860 for (i = 0; i < regs_size; i++) {
1862 WREG32(MC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++));
1863 WREG32(MC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++));
1865 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1866 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1869 /* load the MC ucode */
1870 for (i = 0; i < ucode_size; i++) {
1872 WREG32(MC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++));
1874 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1877 /* put the engine back into the active state */
1878 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1879 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1880 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1882 /* wait for training to complete */
1883 for (i = 0; i < rdev->usec_timeout; i++) {
1884 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1888 for (i = 0; i < rdev->usec_timeout; i++) {
1889 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1895 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1902 * cik_init_microcode - load ucode images from disk
1904 * @rdev: radeon_device pointer
1906 * Use the firmware interface to load the ucode images into
1907 * the driver (not loaded into hw).
1908 * Returns 0 on success, error on failure.
1910 static int cik_init_microcode(struct radeon_device *rdev)
1912 const char *chip_name;
1913 const char *new_chip_name;
1914 size_t pfp_req_size, me_req_size, ce_req_size,
1915 mec_req_size, rlc_req_size, mc_req_size = 0,
1916 sdma_req_size, smc_req_size = 0, mc2_req_size = 0;
1924 switch (rdev->family) {
1926 chip_name = "BONAIRE";
1927 new_chip_name = "bonaire";
1928 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1929 me_req_size = CIK_ME_UCODE_SIZE * 4;
1930 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1931 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1932 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1933 mc_req_size = BONAIRE_MC_UCODE_SIZE * 4;
1934 mc2_req_size = BONAIRE_MC2_UCODE_SIZE * 4;
1935 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1936 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
1940 chip_name = "HAWAII";
1941 new_chip_name = "hawaii";
1942 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1943 me_req_size = CIK_ME_UCODE_SIZE * 4;
1944 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1945 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1946 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1947 mc_req_size = HAWAII_MC_UCODE_SIZE * 4;
1948 mc2_req_size = HAWAII_MC2_UCODE_SIZE * 4;
1949 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1950 smc_req_size = ALIGN(HAWAII_SMC_UCODE_SIZE, 4);
1954 chip_name = "KAVERI";
1955 new_chip_name = "kaveri";
1956 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1957 me_req_size = CIK_ME_UCODE_SIZE * 4;
1958 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1959 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1960 rlc_req_size = KV_RLC_UCODE_SIZE * 4;
1961 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1965 chip_name = "KABINI";
1966 new_chip_name = "kabini";
1967 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1968 me_req_size = CIK_ME_UCODE_SIZE * 4;
1969 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1970 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1971 rlc_req_size = KB_RLC_UCODE_SIZE * 4;
1972 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1976 chip_name = "MULLINS";
1977 new_chip_name = "mullins";
1978 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1979 me_req_size = CIK_ME_UCODE_SIZE * 4;
1980 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1981 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1982 rlc_req_size = ML_RLC_UCODE_SIZE * 4;
1983 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1989 DRM_INFO("Loading %s Microcode\n", new_chip_name);
1991 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", new_chip_name);
1992 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1994 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1995 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1998 if (rdev->pfp_fw->size != pfp_req_size) {
2000 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2001 rdev->pfp_fw->size, fw_name);
2006 err = radeon_ucode_validate(rdev->pfp_fw);
2009 "cik_fw: validation failed for firmware \"%s\"\n",
2017 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", new_chip_name);
2018 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
2020 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
2021 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
2024 if (rdev->me_fw->size != me_req_size) {
2026 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2027 rdev->me_fw->size, fw_name);
2031 err = radeon_ucode_validate(rdev->me_fw);
2034 "cik_fw: validation failed for firmware \"%s\"\n",
2042 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", new_chip_name);
2043 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
2045 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
2046 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
2049 if (rdev->ce_fw->size != ce_req_size) {
2051 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2052 rdev->ce_fw->size, fw_name);
2056 err = radeon_ucode_validate(rdev->ce_fw);
2059 "cik_fw: validation failed for firmware \"%s\"\n",
2067 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", new_chip_name);
2068 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
2070 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
2071 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
2074 if (rdev->mec_fw->size != mec_req_size) {
2076 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2077 rdev->mec_fw->size, fw_name);
2081 err = radeon_ucode_validate(rdev->mec_fw);
2084 "cik_fw: validation failed for firmware \"%s\"\n",
2092 if (rdev->family == CHIP_KAVERI) {
2093 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec2.bin", new_chip_name);
2094 err = request_firmware(&rdev->mec2_fw, fw_name, rdev->dev);
2098 err = radeon_ucode_validate(rdev->mec2_fw);
2107 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", new_chip_name);
2108 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
2110 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
2111 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
2114 if (rdev->rlc_fw->size != rlc_req_size) {
2116 "cik_rlc: Bogus length %zu in firmware \"%s\"\n",
2117 rdev->rlc_fw->size, fw_name);
2121 err = radeon_ucode_validate(rdev->rlc_fw);
2124 "cik_fw: validation failed for firmware \"%s\"\n",
2132 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", new_chip_name);
2133 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
2135 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
2136 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
2139 if (rdev->sdma_fw->size != sdma_req_size) {
2141 "cik_sdma: Bogus length %zu in firmware \"%s\"\n",
2142 rdev->sdma_fw->size, fw_name);
2146 err = radeon_ucode_validate(rdev->sdma_fw);
2149 "cik_fw: validation failed for firmware \"%s\"\n",
2157 /* No SMC, MC ucode on APUs */
2158 if (!(rdev->flags & RADEON_IS_IGP)) {
2159 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", new_chip_name);
2160 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2162 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name);
2163 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2165 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
2166 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2170 if ((rdev->mc_fw->size != mc_req_size) &&
2171 (rdev->mc_fw->size != mc2_req_size)){
2173 "cik_mc: Bogus length %zu in firmware \"%s\"\n",
2174 rdev->mc_fw->size, fw_name);
2177 DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);
2179 err = radeon_ucode_validate(rdev->mc_fw);
2182 "cik_fw: validation failed for firmware \"%s\"\n",
2190 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", new_chip_name);
2191 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
2193 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
2194 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
2197 "smc: error loading firmware \"%s\"\n",
2199 release_firmware(rdev->smc_fw);
2200 rdev->smc_fw = NULL;
2202 } else if (rdev->smc_fw->size != smc_req_size) {
2204 "cik_smc: Bogus length %zu in firmware \"%s\"\n",
2205 rdev->smc_fw->size, fw_name);
2209 err = radeon_ucode_validate(rdev->smc_fw);
2212 "cik_fw: validation failed for firmware \"%s\"\n",
2222 rdev->new_fw = false;
2223 } else if (new_fw < num_fw) {
2224 printk(KERN_ERR "ci_fw: mixing new and old firmware!\n");
2227 rdev->new_fw = true;
2234 "cik_cp: Failed to load firmware \"%s\"\n",
2236 release_firmware(rdev->pfp_fw);
2237 rdev->pfp_fw = NULL;
2238 release_firmware(rdev->me_fw);
2240 release_firmware(rdev->ce_fw);
2242 release_firmware(rdev->mec_fw);
2243 rdev->mec_fw = NULL;
2244 release_firmware(rdev->mec2_fw);
2245 rdev->mec2_fw = NULL;
2246 release_firmware(rdev->rlc_fw);
2247 rdev->rlc_fw = NULL;
2248 release_firmware(rdev->sdma_fw);
2249 rdev->sdma_fw = NULL;
2250 release_firmware(rdev->mc_fw);
2252 release_firmware(rdev->smc_fw);
2253 rdev->smc_fw = NULL;
2262 * cik_tiling_mode_table_init - init the hw tiling table
2264 * @rdev: radeon_device pointer
2266 * Starting with SI, the tiling setup is done globally in a
2267 * set of 32 tiling modes. Rather than selecting each set of
2268 * parameters per surface as on older asics, we just select
2269 * which index in the tiling table we want to use, and the
2270 * surface uses those parameters (CIK).
2272 static void cik_tiling_mode_table_init(struct radeon_device *rdev)
2274 const u32 num_tile_mode_states = 32;
2275 const u32 num_secondary_tile_mode_states = 16;
2276 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
2277 u32 num_pipe_configs;
2278 u32 num_rbs = rdev->config.cik.max_backends_per_se *
2279 rdev->config.cik.max_shader_engines;
2281 switch (rdev->config.cik.mem_row_size_in_kb) {
2283 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2287 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2290 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2294 num_pipe_configs = rdev->config.cik.max_tile_pipes;
2295 if (num_pipe_configs > 8)
2296 num_pipe_configs = 16;
2298 if (num_pipe_configs == 16) {
2299 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2300 switch (reg_offset) {
2302 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2303 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2304 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2305 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2308 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2309 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2310 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2311 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2314 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2315 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2316 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2317 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2320 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2321 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2322 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2323 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2326 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2327 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2328 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2329 TILE_SPLIT(split_equal_to_row_size));
2332 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2333 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2334 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2337 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2338 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2339 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2340 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2343 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2344 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2345 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2346 TILE_SPLIT(split_equal_to_row_size));
2349 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2350 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
2353 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2354 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2355 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2358 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2359 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2360 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2361 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2364 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2365 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2366 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2367 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2370 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2371 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2372 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2373 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2376 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2377 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2378 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2381 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2382 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2383 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2384 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2387 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2388 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2389 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2390 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2393 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2394 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2395 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2396 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2399 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2400 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2401 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2404 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2405 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2406 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2407 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2410 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2411 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2412 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2413 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2416 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2417 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2418 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2419 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2425 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2426 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2428 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2429 switch (reg_offset) {
2431 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2432 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2433 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2434 NUM_BANKS(ADDR_SURF_16_BANK));
2437 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2438 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2439 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2440 NUM_BANKS(ADDR_SURF_16_BANK));
2443 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2444 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2445 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2446 NUM_BANKS(ADDR_SURF_16_BANK));
2449 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2450 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2451 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2452 NUM_BANKS(ADDR_SURF_16_BANK));
2455 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2456 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2457 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2458 NUM_BANKS(ADDR_SURF_8_BANK));
2461 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2462 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2463 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2464 NUM_BANKS(ADDR_SURF_4_BANK));
2467 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2468 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2469 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2470 NUM_BANKS(ADDR_SURF_2_BANK));
2473 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2474 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2475 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2476 NUM_BANKS(ADDR_SURF_16_BANK));
2479 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2480 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2481 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2482 NUM_BANKS(ADDR_SURF_16_BANK));
2485 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2486 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2487 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2488 NUM_BANKS(ADDR_SURF_16_BANK));
2491 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2492 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2493 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2494 NUM_BANKS(ADDR_SURF_8_BANK));
2497 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2498 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2499 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2500 NUM_BANKS(ADDR_SURF_4_BANK));
2503 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2504 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2505 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2506 NUM_BANKS(ADDR_SURF_2_BANK));
2509 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2510 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2511 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2512 NUM_BANKS(ADDR_SURF_2_BANK));
2518 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
2519 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2521 } else if (num_pipe_configs == 8) {
2522 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2523 switch (reg_offset) {
2525 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2526 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2527 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2528 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2531 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2532 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2533 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2534 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2537 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2538 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2539 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2540 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2543 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2544 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2545 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2546 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2549 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2550 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2551 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2552 TILE_SPLIT(split_equal_to_row_size));
2555 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2556 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2557 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2560 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2561 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2562 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2563 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2566 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2567 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2568 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2569 TILE_SPLIT(split_equal_to_row_size));
2572 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2573 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2576 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2577 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2578 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2581 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2582 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2583 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2584 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2587 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2588 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2589 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2590 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2593 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2594 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2595 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2596 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2599 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2600 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2601 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2604 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2605 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2606 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2607 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2610 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2611 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2612 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2613 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2616 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2617 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2618 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2619 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2622 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2623 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2624 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2627 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2628 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2629 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2630 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2633 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2634 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2635 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2636 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2639 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2640 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2641 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2642 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2648 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2649 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2651 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2652 switch (reg_offset) {
2654 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2655 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2656 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2657 NUM_BANKS(ADDR_SURF_16_BANK));
2660 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2661 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2662 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2663 NUM_BANKS(ADDR_SURF_16_BANK));
2666 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2667 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2668 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2669 NUM_BANKS(ADDR_SURF_16_BANK));
2672 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2673 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2674 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2675 NUM_BANKS(ADDR_SURF_16_BANK));
2678 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2679 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2680 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2681 NUM_BANKS(ADDR_SURF_8_BANK));
2684 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2685 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2686 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2687 NUM_BANKS(ADDR_SURF_4_BANK));
2690 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2691 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2692 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2693 NUM_BANKS(ADDR_SURF_2_BANK));
2696 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2697 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2698 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2699 NUM_BANKS(ADDR_SURF_16_BANK));
2702 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2703 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2704 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2705 NUM_BANKS(ADDR_SURF_16_BANK));
2708 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2709 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2710 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2711 NUM_BANKS(ADDR_SURF_16_BANK));
2714 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2715 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2716 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2717 NUM_BANKS(ADDR_SURF_16_BANK));
2720 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2721 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2722 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2723 NUM_BANKS(ADDR_SURF_8_BANK));
2726 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2727 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2728 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2729 NUM_BANKS(ADDR_SURF_4_BANK));
2732 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2733 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2734 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2735 NUM_BANKS(ADDR_SURF_2_BANK));
2741 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
2742 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2744 } else if (num_pipe_configs == 4) {
2746 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2747 switch (reg_offset) {
2749 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2750 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2751 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2752 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2755 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2756 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2757 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2758 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2761 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2762 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2763 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2764 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2767 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2768 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2769 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2770 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2773 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2774 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2775 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2776 TILE_SPLIT(split_equal_to_row_size));
2779 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2780 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2781 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2784 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2785 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2786 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2787 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2790 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2791 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2792 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2793 TILE_SPLIT(split_equal_to_row_size));
2796 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2797 PIPE_CONFIG(ADDR_SURF_P4_16x16));
2800 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2801 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2802 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2805 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2806 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2807 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2808 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2811 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2812 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2813 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2814 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2817 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2818 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2819 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2820 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2823 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2824 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2825 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2828 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2829 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2830 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2831 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2834 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2835 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2836 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2837 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2840 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2841 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2842 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2843 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2846 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2847 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2848 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2851 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2852 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2853 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2854 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2857 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2858 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2859 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2860 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2863 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2864 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2865 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2866 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2872 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2873 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2875 } else if (num_rbs < 4) {
2876 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2877 switch (reg_offset) {
2879 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2880 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2881 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2882 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2885 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2886 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2887 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2888 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2891 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2892 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2893 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2894 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2897 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2898 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2899 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2900 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2903 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2904 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2905 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2906 TILE_SPLIT(split_equal_to_row_size));
2909 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2910 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2911 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2914 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2915 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2916 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2917 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2920 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2921 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2922 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2923 TILE_SPLIT(split_equal_to_row_size));
2926 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2927 PIPE_CONFIG(ADDR_SURF_P4_8x16));
2930 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2931 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2932 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2935 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2936 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2937 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2938 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2941 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2942 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2943 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2944 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2947 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2948 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2949 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2950 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2953 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2954 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2955 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2958 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2959 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2960 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2961 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2964 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2965 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2966 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2967 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2970 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2971 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2972 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2973 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2976 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2977 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2978 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2981 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2982 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2983 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2984 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2987 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2988 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2989 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2990 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2993 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2994 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2995 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2996 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3002 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
3003 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3006 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
3007 switch (reg_offset) {
3009 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3010 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3011 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3012 NUM_BANKS(ADDR_SURF_16_BANK));
3015 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3016 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3017 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3018 NUM_BANKS(ADDR_SURF_16_BANK));
3021 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3022 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3023 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3024 NUM_BANKS(ADDR_SURF_16_BANK));
3027 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3028 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3029 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3030 NUM_BANKS(ADDR_SURF_16_BANK));
3033 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3034 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3035 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3036 NUM_BANKS(ADDR_SURF_16_BANK));
3039 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3040 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3041 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3042 NUM_BANKS(ADDR_SURF_8_BANK));
3045 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3046 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3047 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3048 NUM_BANKS(ADDR_SURF_4_BANK));
3051 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3052 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3053 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3054 NUM_BANKS(ADDR_SURF_16_BANK));
3057 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3058 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3059 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3060 NUM_BANKS(ADDR_SURF_16_BANK));
3063 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3064 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3065 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3066 NUM_BANKS(ADDR_SURF_16_BANK));
3069 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3070 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3071 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3072 NUM_BANKS(ADDR_SURF_16_BANK));
3075 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3076 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3077 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3078 NUM_BANKS(ADDR_SURF_16_BANK));
3081 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3082 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3083 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3084 NUM_BANKS(ADDR_SURF_8_BANK));
3087 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3088 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3089 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3090 NUM_BANKS(ADDR_SURF_4_BANK));
3096 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
3097 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3099 } else if (num_pipe_configs == 2) {
3100 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
3101 switch (reg_offset) {
3103 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3104 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3105 PIPE_CONFIG(ADDR_SURF_P2) |
3106 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
3109 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3110 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3111 PIPE_CONFIG(ADDR_SURF_P2) |
3112 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
3115 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3116 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3117 PIPE_CONFIG(ADDR_SURF_P2) |
3118 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
3121 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3122 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3123 PIPE_CONFIG(ADDR_SURF_P2) |
3124 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
3127 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3128 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3129 PIPE_CONFIG(ADDR_SURF_P2) |
3130 TILE_SPLIT(split_equal_to_row_size));
3133 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3134 PIPE_CONFIG(ADDR_SURF_P2) |
3135 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3138 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3139 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3140 PIPE_CONFIG(ADDR_SURF_P2) |
3141 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
3144 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3145 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3146 PIPE_CONFIG(ADDR_SURF_P2) |
3147 TILE_SPLIT(split_equal_to_row_size));
3150 gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
3151 PIPE_CONFIG(ADDR_SURF_P2);
3154 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3155 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3156 PIPE_CONFIG(ADDR_SURF_P2));
3159 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3160 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3161 PIPE_CONFIG(ADDR_SURF_P2) |
3162 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3165 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3166 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3167 PIPE_CONFIG(ADDR_SURF_P2) |
3168 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3171 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3172 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3173 PIPE_CONFIG(ADDR_SURF_P2) |
3174 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3177 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3178 PIPE_CONFIG(ADDR_SURF_P2) |
3179 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
3182 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3183 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3184 PIPE_CONFIG(ADDR_SURF_P2) |
3185 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3188 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3189 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3190 PIPE_CONFIG(ADDR_SURF_P2) |
3191 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3194 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3195 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3196 PIPE_CONFIG(ADDR_SURF_P2) |
3197 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3200 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3201 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3202 PIPE_CONFIG(ADDR_SURF_P2));
3205 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3206 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3207 PIPE_CONFIG(ADDR_SURF_P2) |
3208 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3211 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3212 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3213 PIPE_CONFIG(ADDR_SURF_P2) |
3214 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3217 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3218 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3219 PIPE_CONFIG(ADDR_SURF_P2) |
3220 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3226 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
3227 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3229 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
3230 switch (reg_offset) {
3232 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3233 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3234 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3235 NUM_BANKS(ADDR_SURF_16_BANK));
3238 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3239 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3240 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3241 NUM_BANKS(ADDR_SURF_16_BANK));
3244 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3245 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3246 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3247 NUM_BANKS(ADDR_SURF_16_BANK));
3250 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3251 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3252 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3253 NUM_BANKS(ADDR_SURF_16_BANK));
3256 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3257 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3258 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3259 NUM_BANKS(ADDR_SURF_16_BANK));
3262 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3263 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3264 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3265 NUM_BANKS(ADDR_SURF_16_BANK));
3268 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3269 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3270 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3271 NUM_BANKS(ADDR_SURF_8_BANK));
3274 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3275 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3276 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3277 NUM_BANKS(ADDR_SURF_16_BANK));
3280 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3281 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3282 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3283 NUM_BANKS(ADDR_SURF_16_BANK));
3286 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3287 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3288 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3289 NUM_BANKS(ADDR_SURF_16_BANK));
3292 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3293 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3294 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3295 NUM_BANKS(ADDR_SURF_16_BANK));
3298 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3299 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3300 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3301 NUM_BANKS(ADDR_SURF_16_BANK));
3304 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3305 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3306 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3307 NUM_BANKS(ADDR_SURF_16_BANK));
3310 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3311 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3312 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3313 NUM_BANKS(ADDR_SURF_8_BANK));
3319 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
3320 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3323 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
3327 * cik_select_se_sh - select which SE, SH to address
3329 * @rdev: radeon_device pointer
3330 * @se_num: shader engine to address
3331 * @sh_num: sh block to address
3333 * Select which SE, SH combinations to address. Certain
3334 * registers are instanced per SE or SH. 0xffffffff means
3335 * broadcast to all SEs or SHs (CIK).
3337 static void cik_select_se_sh(struct radeon_device *rdev,
3338 u32 se_num, u32 sh_num)
3340 u32 data = INSTANCE_BROADCAST_WRITES;
3342 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
3343 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
3344 else if (se_num == 0xffffffff)
3345 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
3346 else if (sh_num == 0xffffffff)
3347 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
3349 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
3350 WREG32(GRBM_GFX_INDEX, data);
3354 * cik_create_bitmask - create a bitmask
3356 * @bit_width: length of the mask
3358 * create a variable length bit mask (CIK).
3359 * Returns the bitmask.
3361 static u32 cik_create_bitmask(u32 bit_width)
3365 for (i = 0; i < bit_width; i++) {
3373 * cik_get_rb_disabled - computes the mask of disabled RBs
3375 * @rdev: radeon_device pointer
3376 * @max_rb_num: max RBs (render backends) for the asic
3377 * @se_num: number of SEs (shader engines) for the asic
3378 * @sh_per_se: number of SH blocks per SE for the asic
3380 * Calculates the bitmask of disabled RBs (CIK).
3381 * Returns the disabled RB bitmask.
3383 static u32 cik_get_rb_disabled(struct radeon_device *rdev,
3384 u32 max_rb_num_per_se,
3389 data = RREG32(CC_RB_BACKEND_DISABLE);
3391 data &= BACKEND_DISABLE_MASK;
3394 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3396 data >>= BACKEND_DISABLE_SHIFT;
3398 mask = cik_create_bitmask(max_rb_num_per_se / sh_per_se);
3404 * cik_setup_rb - setup the RBs on the asic
3406 * @rdev: radeon_device pointer
3407 * @se_num: number of SEs (shader engines) for the asic
3408 * @sh_per_se: number of SH blocks per SE for the asic
3409 * @max_rb_num: max RBs (render backends) for the asic
3411 * Configures per-SE/SH RB registers (CIK).
3413 static void cik_setup_rb(struct radeon_device *rdev,
3414 u32 se_num, u32 sh_per_se,
3415 u32 max_rb_num_per_se)
3419 u32 disabled_rbs = 0;
3420 u32 enabled_rbs = 0;
3422 for (i = 0; i < se_num; i++) {
3423 for (j = 0; j < sh_per_se; j++) {
3424 cik_select_se_sh(rdev, i, j);
3425 data = cik_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
3426 if (rdev->family == CHIP_HAWAII)
3427 disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH);
3429 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
3432 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3435 for (i = 0; i < max_rb_num_per_se * se_num; i++) {
3436 if (!(disabled_rbs & mask))
3437 enabled_rbs |= mask;
3441 rdev->config.cik.backend_enable_mask = enabled_rbs;
3443 for (i = 0; i < se_num; i++) {
3444 cik_select_se_sh(rdev, i, 0xffffffff);
3446 for (j = 0; j < sh_per_se; j++) {
3447 switch (enabled_rbs & 3) {
3450 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3);
3452 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0);
3455 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3458 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3462 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3467 WREG32(PA_SC_RASTER_CONFIG, data);
3469 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3473 * cik_gpu_init - setup the 3D engine
3475 * @rdev: radeon_device pointer
3477 * Configures the 3D engine and tiling configuration
3478 * registers so that the 3D engine is usable.
3480 static void cik_gpu_init(struct radeon_device *rdev)
3482 u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
3483 u32 mc_shared_chmap, mc_arb_ramcfg;
3484 u32 hdp_host_path_cntl;
3488 switch (rdev->family) {
3490 rdev->config.cik.max_shader_engines = 2;
3491 rdev->config.cik.max_tile_pipes = 4;
3492 rdev->config.cik.max_cu_per_sh = 7;
3493 rdev->config.cik.max_sh_per_se = 1;
3494 rdev->config.cik.max_backends_per_se = 2;
3495 rdev->config.cik.max_texture_channel_caches = 4;
3496 rdev->config.cik.max_gprs = 256;
3497 rdev->config.cik.max_gs_threads = 32;
3498 rdev->config.cik.max_hw_contexts = 8;
3500 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3501 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3502 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3503 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3504 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3507 rdev->config.cik.max_shader_engines = 4;
3508 rdev->config.cik.max_tile_pipes = 16;
3509 rdev->config.cik.max_cu_per_sh = 11;
3510 rdev->config.cik.max_sh_per_se = 1;
3511 rdev->config.cik.max_backends_per_se = 4;
3512 rdev->config.cik.max_texture_channel_caches = 16;
3513 rdev->config.cik.max_gprs = 256;
3514 rdev->config.cik.max_gs_threads = 32;
3515 rdev->config.cik.max_hw_contexts = 8;
3517 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3518 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3519 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3520 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3521 gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN;
3524 rdev->config.cik.max_shader_engines = 1;
3525 rdev->config.cik.max_tile_pipes = 4;
3526 if ((rdev->pdev->device == 0x1304) ||
3527 (rdev->pdev->device == 0x1305) ||
3528 (rdev->pdev->device == 0x130C) ||
3529 (rdev->pdev->device == 0x130F) ||
3530 (rdev->pdev->device == 0x1310) ||
3531 (rdev->pdev->device == 0x1311) ||
3532 (rdev->pdev->device == 0x131C)) {
3533 rdev->config.cik.max_cu_per_sh = 8;
3534 rdev->config.cik.max_backends_per_se = 2;
3535 } else if ((rdev->pdev->device == 0x1309) ||
3536 (rdev->pdev->device == 0x130A) ||
3537 (rdev->pdev->device == 0x130D) ||
3538 (rdev->pdev->device == 0x1313) ||
3539 (rdev->pdev->device == 0x131D)) {
3540 rdev->config.cik.max_cu_per_sh = 6;
3541 rdev->config.cik.max_backends_per_se = 2;
3542 } else if ((rdev->pdev->device == 0x1306) ||
3543 (rdev->pdev->device == 0x1307) ||
3544 (rdev->pdev->device == 0x130B) ||
3545 (rdev->pdev->device == 0x130E) ||
3546 (rdev->pdev->device == 0x1315) ||
3547 (rdev->pdev->device == 0x1318) ||
3548 (rdev->pdev->device == 0x131B)) {
3549 rdev->config.cik.max_cu_per_sh = 4;
3550 rdev->config.cik.max_backends_per_se = 1;
3552 rdev->config.cik.max_cu_per_sh = 3;
3553 rdev->config.cik.max_backends_per_se = 1;
3555 rdev->config.cik.max_sh_per_se = 1;
3556 rdev->config.cik.max_texture_channel_caches = 4;
3557 rdev->config.cik.max_gprs = 256;
3558 rdev->config.cik.max_gs_threads = 16;
3559 rdev->config.cik.max_hw_contexts = 8;
3561 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3562 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3563 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3564 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3565 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3570 rdev->config.cik.max_shader_engines = 1;
3571 rdev->config.cik.max_tile_pipes = 2;
3572 rdev->config.cik.max_cu_per_sh = 2;
3573 rdev->config.cik.max_sh_per_se = 1;
3574 rdev->config.cik.max_backends_per_se = 1;
3575 rdev->config.cik.max_texture_channel_caches = 2;
3576 rdev->config.cik.max_gprs = 256;
3577 rdev->config.cik.max_gs_threads = 16;
3578 rdev->config.cik.max_hw_contexts = 8;
3580 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3581 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3582 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3583 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3584 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3588 /* Initialize HDP */
3589 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3590 WREG32((0x2c14 + j), 0x00000000);
3591 WREG32((0x2c18 + j), 0x00000000);
3592 WREG32((0x2c1c + j), 0x00000000);
3593 WREG32((0x2c20 + j), 0x00000000);
3594 WREG32((0x2c24 + j), 0x00000000);
3597 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3599 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3601 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3602 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3604 rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
3605 rdev->config.cik.mem_max_burst_length_bytes = 256;
3606 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3607 rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3608 if (rdev->config.cik.mem_row_size_in_kb > 4)
3609 rdev->config.cik.mem_row_size_in_kb = 4;
3610 /* XXX use MC settings? */
3611 rdev->config.cik.shader_engine_tile_size = 32;
3612 rdev->config.cik.num_gpus = 1;
3613 rdev->config.cik.multi_gpu_tile_size = 64;
3615 /* fix up row size */
3616 gb_addr_config &= ~ROW_SIZE_MASK;
3617 switch (rdev->config.cik.mem_row_size_in_kb) {
3620 gb_addr_config |= ROW_SIZE(0);
3623 gb_addr_config |= ROW_SIZE(1);
3626 gb_addr_config |= ROW_SIZE(2);
3630 /* setup tiling info dword. gb_addr_config is not adequate since it does
3631 * not have bank info, so create a custom tiling dword.
3632 * bits 3:0 num_pipes
3633 * bits 7:4 num_banks
3634 * bits 11:8 group_size
3635 * bits 15:12 row_size
3637 rdev->config.cik.tile_config = 0;
3638 switch (rdev->config.cik.num_tile_pipes) {
3640 rdev->config.cik.tile_config |= (0 << 0);
3643 rdev->config.cik.tile_config |= (1 << 0);
3646 rdev->config.cik.tile_config |= (2 << 0);
3650 /* XXX what about 12? */
3651 rdev->config.cik.tile_config |= (3 << 0);
3654 rdev->config.cik.tile_config |=
3655 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
3656 rdev->config.cik.tile_config |=
3657 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3658 rdev->config.cik.tile_config |=
3659 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3661 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3662 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3663 WREG32(DMIF_ADDR_CALC, gb_addr_config);
3664 WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
3665 WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
3666 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3667 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3668 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3670 cik_tiling_mode_table_init(rdev);
3672 cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
3673 rdev->config.cik.max_sh_per_se,
3674 rdev->config.cik.max_backends_per_se);
3676 rdev->config.cik.active_cus = 0;
3677 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
3678 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
3679 rdev->config.cik.active_cus +=
3680 hweight32(cik_get_cu_active_bitmap(rdev, i, j));
3684 /* set HW defaults for 3D engine */
3685 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3687 WREG32(SX_DEBUG_1, 0x20);
3689 WREG32(TA_CNTL_AUX, 0x00010000);
3691 tmp = RREG32(SPI_CONFIG_CNTL);
3693 WREG32(SPI_CONFIG_CNTL, tmp);
3695 WREG32(SQ_CONFIG, 1);
3697 WREG32(DB_DEBUG, 0);
3699 tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
3701 WREG32(DB_DEBUG2, tmp);
3703 tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
3705 WREG32(DB_DEBUG3, tmp);
3707 tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
3709 WREG32(CB_HW_CONTROL, tmp);
3711 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3713 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
3714 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
3715 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
3716 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
3718 WREG32(VGT_NUM_INSTANCES, 1);
3720 WREG32(CP_PERFMON_CNTL, 0);
3722 WREG32(SQ_CONFIG, 0);
3724 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3725 FORCE_EOV_MAX_REZ_CNT(255)));
3727 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3728 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3730 WREG32(VGT_GS_VERTEX_REUSE, 16);
3731 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3733 tmp = RREG32(HDP_MISC_CNTL);
3734 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3735 WREG32(HDP_MISC_CNTL, tmp);
3737 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3738 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3740 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3741 WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
3747 * GPU scratch registers helpers function.
3750 * cik_scratch_init - setup driver info for CP scratch regs
3752 * @rdev: radeon_device pointer
3754 * Set up the number and offset of the CP scratch registers.
3755 * NOTE: use of CP scratch registers is a legacy inferface and
3756 * is not used by default on newer asics (r6xx+). On newer asics,
3757 * memory buffers are used for fences rather than scratch regs.
3759 static void cik_scratch_init(struct radeon_device *rdev)
3763 rdev->scratch.num_reg = 7;
3764 rdev->scratch.reg_base = SCRATCH_REG0;
3765 for (i = 0; i < rdev->scratch.num_reg; i++) {
3766 rdev->scratch.free[i] = true;
3767 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3772 * cik_ring_test - basic gfx ring test
3774 * @rdev: radeon_device pointer
3775 * @ring: radeon_ring structure holding ring information
3777 * Allocate a scratch register and write to it using the gfx ring (CIK).
3778 * Provides a basic gfx ring test to verify that the ring is working.
3779 * Used by cik_cp_gfx_resume();
3780 * Returns 0 on success, error on failure.
3782 int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
3789 r = radeon_scratch_get(rdev, &scratch);
3791 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
3794 WREG32(scratch, 0xCAFEDEAD);
3795 r = radeon_ring_lock(rdev, ring, 3);
3797 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
3798 radeon_scratch_free(rdev, scratch);
3801 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3802 radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
3803 radeon_ring_write(ring, 0xDEADBEEF);
3804 radeon_ring_unlock_commit(rdev, ring, false);
3806 for (i = 0; i < rdev->usec_timeout; i++) {
3807 tmp = RREG32(scratch);
3808 if (tmp == 0xDEADBEEF)
3812 if (i < rdev->usec_timeout) {
3813 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3815 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
3816 ring->idx, scratch, tmp);
3819 radeon_scratch_free(rdev, scratch);
3824 * cik_hdp_flush_cp_ring_emit - emit an hdp flush on the cp
3826 * @rdev: radeon_device pointer
3827 * @ridx: radeon ring index
3829 * Emits an hdp flush on the cp.
3831 static void cik_hdp_flush_cp_ring_emit(struct radeon_device *rdev,
3834 struct radeon_ring *ring = &rdev->ring[ridx];
3837 switch (ring->idx) {
3838 case CAYMAN_RING_TYPE_CP1_INDEX:
3839 case CAYMAN_RING_TYPE_CP2_INDEX:
3843 ref_and_mask = CP2 << ring->pipe;
3846 ref_and_mask = CP6 << ring->pipe;
3852 case RADEON_RING_TYPE_GFX_INDEX:
3857 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
3858 radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */
3859 WAIT_REG_MEM_FUNCTION(3) | /* == */
3860 WAIT_REG_MEM_ENGINE(1))); /* pfp */
3861 radeon_ring_write(ring, GPU_HDP_FLUSH_REQ >> 2);
3862 radeon_ring_write(ring, GPU_HDP_FLUSH_DONE >> 2);
3863 radeon_ring_write(ring, ref_and_mask);
3864 radeon_ring_write(ring, ref_and_mask);
3865 radeon_ring_write(ring, 0x20); /* poll interval */
3869 * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
3871 * @rdev: radeon_device pointer
3872 * @fence: radeon fence object
3874 * Emits a fence sequnce number on the gfx ring and flushes
3877 void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
3878 struct radeon_fence *fence)
3880 struct radeon_ring *ring = &rdev->ring[fence->ring];
3881 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3883 /* EVENT_WRITE_EOP - flush caches, send int */
3884 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3885 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3887 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3889 radeon_ring_write(ring, addr & 0xfffffffc);
3890 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3891 radeon_ring_write(ring, fence->seq);
3892 radeon_ring_write(ring, 0);
3896 * cik_fence_compute_ring_emit - emit a fence on the compute ring
3898 * @rdev: radeon_device pointer
3899 * @fence: radeon fence object
3901 * Emits a fence sequnce number on the compute ring and flushes
3904 void cik_fence_compute_ring_emit(struct radeon_device *rdev,
3905 struct radeon_fence *fence)
3907 struct radeon_ring *ring = &rdev->ring[fence->ring];
3908 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3910 /* RELEASE_MEM - flush caches, send int */
3911 radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
3912 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3914 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3916 radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
3917 radeon_ring_write(ring, addr & 0xfffffffc);
3918 radeon_ring_write(ring, upper_32_bits(addr));
3919 radeon_ring_write(ring, fence->seq);
3920 radeon_ring_write(ring, 0);
3924 * cik_semaphore_ring_emit - emit a semaphore on the CP ring
3926 * @rdev: radeon_device pointer
3927 * @ring: radeon ring buffer object
3928 * @semaphore: radeon semaphore object
3929 * @emit_wait: Is this a sempahore wait?
3931 * Emits a semaphore signal/wait packet to the CP ring and prevents the PFP
3932 * from running ahead of semaphore waits.
3934 bool cik_semaphore_ring_emit(struct radeon_device *rdev,
3935 struct radeon_ring *ring,
3936 struct radeon_semaphore *semaphore,
3939 uint64_t addr = semaphore->gpu_addr;
3940 unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3942 radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
3943 radeon_ring_write(ring, lower_32_bits(addr));
3944 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
3946 if (emit_wait && ring->idx == RADEON_RING_TYPE_GFX_INDEX) {
3947 /* Prevent the PFP from running ahead of the semaphore wait */
3948 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
3949 radeon_ring_write(ring, 0x0);
3956 * cik_copy_cpdma - copy pages using the CP DMA engine
3958 * @rdev: radeon_device pointer
3959 * @src_offset: src GPU address
3960 * @dst_offset: dst GPU address
3961 * @num_gpu_pages: number of GPU pages to xfer
3962 * @resv: reservation object to sync to
3964 * Copy GPU paging using the CP DMA engine (CIK+).
3965 * Used by the radeon ttm implementation to move pages if
3966 * registered as the asic copy callback.
3968 struct radeon_fence *cik_copy_cpdma(struct radeon_device *rdev,
3969 uint64_t src_offset, uint64_t dst_offset,
3970 unsigned num_gpu_pages,
3971 struct reservation_object *resv)
3973 struct radeon_semaphore *sem = NULL;
3974 struct radeon_fence *fence;
3975 int ring_index = rdev->asic->copy.blit_ring_index;
3976 struct radeon_ring *ring = &rdev->ring[ring_index];
3977 u32 size_in_bytes, cur_size_in_bytes, control;
3981 r = radeon_semaphore_create(rdev, &sem);
3983 DRM_ERROR("radeon: moving bo (%d).\n", r);
3987 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
3988 num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
3989 r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18);
3991 DRM_ERROR("radeon: moving bo (%d).\n", r);
3992 radeon_semaphore_free(rdev, &sem, NULL);
3996 radeon_semaphore_sync_resv(sem, resv, false);
3997 radeon_semaphore_sync_rings(rdev, sem, ring->idx);
3999 for (i = 0; i < num_loops; i++) {
4000 cur_size_in_bytes = size_in_bytes;
4001 if (cur_size_in_bytes > 0x1fffff)
4002 cur_size_in_bytes = 0x1fffff;
4003 size_in_bytes -= cur_size_in_bytes;
4005 if (size_in_bytes == 0)
4006 control |= PACKET3_DMA_DATA_CP_SYNC;
4007 radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
4008 radeon_ring_write(ring, control);
4009 radeon_ring_write(ring, lower_32_bits(src_offset));
4010 radeon_ring_write(ring, upper_32_bits(src_offset));
4011 radeon_ring_write(ring, lower_32_bits(dst_offset));
4012 radeon_ring_write(ring, upper_32_bits(dst_offset));
4013 radeon_ring_write(ring, cur_size_in_bytes);
4014 src_offset += cur_size_in_bytes;
4015 dst_offset += cur_size_in_bytes;
4018 r = radeon_fence_emit(rdev, &fence, ring->idx);
4020 radeon_ring_unlock_undo(rdev, ring);
4021 radeon_semaphore_free(rdev, &sem, NULL);
4025 radeon_ring_unlock_commit(rdev, ring, false);
4026 radeon_semaphore_free(rdev, &sem, fence);
4035 * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
4037 * @rdev: radeon_device pointer
4038 * @ib: radeon indirect buffer object
4040 * Emits an DE (drawing engine) or CE (constant engine) IB
4041 * on the gfx ring. IBs are usually generated by userspace
4042 * acceleration drivers and submitted to the kernel for
4043 * sheduling on the ring. This function schedules the IB
4044 * on the gfx ring for execution by the GPU.
4046 void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
4048 struct radeon_ring *ring = &rdev->ring[ib->ring];
4049 u32 header, control = INDIRECT_BUFFER_VALID;
4051 if (ib->is_const_ib) {
4052 /* set switch buffer packet before const IB */
4053 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
4054 radeon_ring_write(ring, 0);
4056 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
4059 if (ring->rptr_save_reg) {
4060 next_rptr = ring->wptr + 3 + 4;
4061 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
4062 radeon_ring_write(ring, ((ring->rptr_save_reg -
4063 PACKET3_SET_UCONFIG_REG_START) >> 2));
4064 radeon_ring_write(ring, next_rptr);
4065 } else if (rdev->wb.enabled) {
4066 next_rptr = ring->wptr + 5 + 4;
4067 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4068 radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
4069 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
4070 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr));
4071 radeon_ring_write(ring, next_rptr);
4074 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
4077 control |= ib->length_dw |
4078 (ib->vm ? (ib->vm->id << 24) : 0);
4080 radeon_ring_write(ring, header);
4081 radeon_ring_write(ring,
4085 (ib->gpu_addr & 0xFFFFFFFC));
4086 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
4087 radeon_ring_write(ring, control);
4091 * cik_ib_test - basic gfx ring IB test
4093 * @rdev: radeon_device pointer
4094 * @ring: radeon_ring structure holding ring information
4096 * Allocate an IB and execute it on the gfx ring (CIK).
4097 * Provides a basic gfx ring test to verify that IBs are working.
4098 * Returns 0 on success, error on failure.
4100 int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
4102 struct radeon_ib ib;
4108 r = radeon_scratch_get(rdev, &scratch);
4110 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
4113 WREG32(scratch, 0xCAFEDEAD);
4114 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
4116 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
4117 radeon_scratch_free(rdev, scratch);
4120 ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
4121 ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
4122 ib.ptr[2] = 0xDEADBEEF;
4124 r = radeon_ib_schedule(rdev, &ib, NULL, false);
4126 radeon_scratch_free(rdev, scratch);
4127 radeon_ib_free(rdev, &ib);
4128 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
4131 r = radeon_fence_wait(ib.fence, false);
4133 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
4134 radeon_scratch_free(rdev, scratch);
4135 radeon_ib_free(rdev, &ib);
4138 for (i = 0; i < rdev->usec_timeout; i++) {
4139 tmp = RREG32(scratch);
4140 if (tmp == 0xDEADBEEF)
4144 if (i < rdev->usec_timeout) {
4145 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
4147 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
4151 radeon_scratch_free(rdev, scratch);
4152 radeon_ib_free(rdev, &ib);
4158 * On CIK, gfx and compute now have independant command processors.
4161 * Gfx consists of a single ring and can process both gfx jobs and
4162 * compute jobs. The gfx CP consists of three microengines (ME):
4163 * PFP - Pre-Fetch Parser
4165 * CE - Constant Engine
4166 * The PFP and ME make up what is considered the Drawing Engine (DE).
4167 * The CE is an asynchronous engine used for updating buffer desciptors
4168 * used by the DE so that they can be loaded into cache in parallel
4169 * while the DE is processing state update packets.
4172 * The compute CP consists of two microengines (ME):
4173 * MEC1 - Compute MicroEngine 1
4174 * MEC2 - Compute MicroEngine 2
4175 * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
4176 * The queues are exposed to userspace and are programmed directly
4177 * by the compute runtime.
4180 * cik_cp_gfx_enable - enable/disable the gfx CP MEs
4182 * @rdev: radeon_device pointer
4183 * @enable: enable or disable the MEs
4185 * Halts or unhalts the gfx MEs.
4187 static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
4190 WREG32(CP_ME_CNTL, 0);
4192 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
4193 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
4194 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
4195 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4201 * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
4203 * @rdev: radeon_device pointer
4205 * Loads the gfx PFP, ME, and CE ucode.
4206 * Returns 0 for success, -EINVAL if the ucode is not available.
4208 static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
4212 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
4215 cik_cp_gfx_enable(rdev, false);
4218 const struct gfx_firmware_header_v1_0 *pfp_hdr =
4219 (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
4220 const struct gfx_firmware_header_v1_0 *ce_hdr =
4221 (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
4222 const struct gfx_firmware_header_v1_0 *me_hdr =
4223 (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
4224 const __le32 *fw_data;
4227 radeon_ucode_print_gfx_hdr(&pfp_hdr->header);
4228 radeon_ucode_print_gfx_hdr(&ce_hdr->header);
4229 radeon_ucode_print_gfx_hdr(&me_hdr->header);
4232 fw_data = (const __le32 *)
4233 (rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
4234 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
4235 WREG32(CP_PFP_UCODE_ADDR, 0);
4236 for (i = 0; i < fw_size; i++)
4237 WREG32(CP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
4238 WREG32(CP_PFP_UCODE_ADDR, 0);
4241 fw_data = (const __le32 *)
4242 (rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
4243 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
4244 WREG32(CP_CE_UCODE_ADDR, 0);
4245 for (i = 0; i < fw_size; i++)
4246 WREG32(CP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
4247 WREG32(CP_CE_UCODE_ADDR, 0);
4250 fw_data = (const __be32 *)
4251 (rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
4252 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
4253 WREG32(CP_ME_RAM_WADDR, 0);
4254 for (i = 0; i < fw_size; i++)
4255 WREG32(CP_ME_RAM_DATA, le32_to_cpup(fw_data++));
4256 WREG32(CP_ME_RAM_WADDR, 0);
4258 const __be32 *fw_data;
4261 fw_data = (const __be32 *)rdev->pfp_fw->data;
4262 WREG32(CP_PFP_UCODE_ADDR, 0);
4263 for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
4264 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
4265 WREG32(CP_PFP_UCODE_ADDR, 0);
4268 fw_data = (const __be32 *)rdev->ce_fw->data;
4269 WREG32(CP_CE_UCODE_ADDR, 0);
4270 for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
4271 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
4272 WREG32(CP_CE_UCODE_ADDR, 0);
4275 fw_data = (const __be32 *)rdev->me_fw->data;
4276 WREG32(CP_ME_RAM_WADDR, 0);
4277 for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
4278 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
4279 WREG32(CP_ME_RAM_WADDR, 0);
4282 WREG32(CP_PFP_UCODE_ADDR, 0);
4283 WREG32(CP_CE_UCODE_ADDR, 0);
4284 WREG32(CP_ME_RAM_WADDR, 0);
4285 WREG32(CP_ME_RAM_RADDR, 0);
4290 * cik_cp_gfx_start - start the gfx ring
4292 * @rdev: radeon_device pointer
4294 * Enables the ring and loads the clear state context and other
4295 * packets required to init the ring.
4296 * Returns 0 for success, error for failure.
4298 static int cik_cp_gfx_start(struct radeon_device *rdev)
4300 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4304 WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
4305 WREG32(CP_ENDIAN_SWAP, 0);
4306 WREG32(CP_DEVICE_ID, 1);
4308 cik_cp_gfx_enable(rdev, true);
4310 r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
4312 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
4316 /* init the CE partitions. CE only used for gfx on CIK */
4317 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
4318 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
4319 radeon_ring_write(ring, 0xc000);
4320 radeon_ring_write(ring, 0xc000);
4322 /* setup clear context state */
4323 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4324 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
4326 radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
4327 radeon_ring_write(ring, 0x80000000);
4328 radeon_ring_write(ring, 0x80000000);
4330 for (i = 0; i < cik_default_size; i++)
4331 radeon_ring_write(ring, cik_default_state[i]);
4333 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4334 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
4336 /* set clear context state */
4337 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
4338 radeon_ring_write(ring, 0);
4340 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
4341 radeon_ring_write(ring, 0x00000316);
4342 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
4343 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
4345 radeon_ring_unlock_commit(rdev, ring, false);
4351 * cik_cp_gfx_fini - stop the gfx ring
4353 * @rdev: radeon_device pointer
4355 * Stop the gfx ring and tear down the driver ring
4358 static void cik_cp_gfx_fini(struct radeon_device *rdev)
4360 cik_cp_gfx_enable(rdev, false);
4361 radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4365 * cik_cp_gfx_resume - setup the gfx ring buffer registers
4367 * @rdev: radeon_device pointer
4369 * Program the location and size of the gfx ring buffer
4370 * and test it to make sure it's working.
4371 * Returns 0 for success, error for failure.
4373 static int cik_cp_gfx_resume(struct radeon_device *rdev)
4375 struct radeon_ring *ring;
4381 WREG32(CP_SEM_WAIT_TIMER, 0x0);
4382 if (rdev->family != CHIP_HAWAII)
4383 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
4385 /* Set the write pointer delay */
4386 WREG32(CP_RB_WPTR_DELAY, 0);
4388 /* set the RB to use vmid 0 */
4389 WREG32(CP_RB_VMID, 0);
4391 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
4393 /* ring 0 - compute and gfx */
4394 /* Set ring buffer size */
4395 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4396 rb_bufsz = order_base_2(ring->ring_size / 8);
4397 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
4399 tmp |= BUF_SWAP_32BIT;
4401 WREG32(CP_RB0_CNTL, tmp);
4403 /* Initialize the ring buffer's read and write pointers */
4404 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
4406 WREG32(CP_RB0_WPTR, ring->wptr);
4408 /* set the wb address wether it's enabled or not */
4409 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
4410 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
4412 /* scratch register shadowing is no longer supported */
4413 WREG32(SCRATCH_UMSK, 0);
4415 if (!rdev->wb.enabled)
4416 tmp |= RB_NO_UPDATE;
4419 WREG32(CP_RB0_CNTL, tmp);
4421 rb_addr = ring->gpu_addr >> 8;
4422 WREG32(CP_RB0_BASE, rb_addr);
4423 WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
4425 /* start the ring */
4426 cik_cp_gfx_start(rdev);
4427 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
4428 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4430 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4434 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
4435 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
4440 u32 cik_gfx_get_rptr(struct radeon_device *rdev,
4441 struct radeon_ring *ring)
4445 if (rdev->wb.enabled)
4446 rptr = rdev->wb.wb[ring->rptr_offs/4];
4448 rptr = RREG32(CP_RB0_RPTR);
4453 u32 cik_gfx_get_wptr(struct radeon_device *rdev,
4454 struct radeon_ring *ring)
4458 wptr = RREG32(CP_RB0_WPTR);
4463 void cik_gfx_set_wptr(struct radeon_device *rdev,
4464 struct radeon_ring *ring)
4466 WREG32(CP_RB0_WPTR, ring->wptr);
4467 (void)RREG32(CP_RB0_WPTR);
4470 u32 cik_compute_get_rptr(struct radeon_device *rdev,
4471 struct radeon_ring *ring)
4475 if (rdev->wb.enabled) {
4476 rptr = rdev->wb.wb[ring->rptr_offs/4];
4478 mutex_lock(&rdev->srbm_mutex);
4479 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4480 rptr = RREG32(CP_HQD_PQ_RPTR);
4481 cik_srbm_select(rdev, 0, 0, 0, 0);
4482 mutex_unlock(&rdev->srbm_mutex);
4488 u32 cik_compute_get_wptr(struct radeon_device *rdev,
4489 struct radeon_ring *ring)
4493 if (rdev->wb.enabled) {
4494 /* XXX check if swapping is necessary on BE */
4495 wptr = rdev->wb.wb[ring->wptr_offs/4];
4497 mutex_lock(&rdev->srbm_mutex);
4498 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4499 wptr = RREG32(CP_HQD_PQ_WPTR);
4500 cik_srbm_select(rdev, 0, 0, 0, 0);
4501 mutex_unlock(&rdev->srbm_mutex);
4507 void cik_compute_set_wptr(struct radeon_device *rdev,
4508 struct radeon_ring *ring)
4510 /* XXX check if swapping is necessary on BE */
4511 rdev->wb.wb[ring->wptr_offs/4] = ring->wptr;
4512 WDOORBELL32(ring->doorbell_index, ring->wptr);
4516 * cik_cp_compute_enable - enable/disable the compute CP MEs
4518 * @rdev: radeon_device pointer
4519 * @enable: enable or disable the MEs
4521 * Halts or unhalts the compute MEs.
4523 static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
4526 WREG32(CP_MEC_CNTL, 0);
4528 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
4529 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
4530 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
4536 * cik_cp_compute_load_microcode - load the compute CP ME ucode
4538 * @rdev: radeon_device pointer
4540 * Loads the compute MEC1&2 ucode.
4541 * Returns 0 for success, -EINVAL if the ucode is not available.
4543 static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
4550 cik_cp_compute_enable(rdev, false);
4553 const struct gfx_firmware_header_v1_0 *mec_hdr =
4554 (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;
4555 const __le32 *fw_data;
4558 radeon_ucode_print_gfx_hdr(&mec_hdr->header);
4561 fw_data = (const __le32 *)
4562 (rdev->mec_fw->data + le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
4563 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes) / 4;
4564 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4565 for (i = 0; i < fw_size; i++)
4566 WREG32(CP_MEC_ME1_UCODE_DATA, le32_to_cpup(fw_data++));
4567 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4570 if (rdev->family == CHIP_KAVERI) {
4571 const struct gfx_firmware_header_v1_0 *mec2_hdr =
4572 (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;
4574 fw_data = (const __le32 *)
4575 (rdev->mec2_fw->data +
4576 le32_to_cpu(mec2_hdr->header.ucode_array_offset_bytes));
4577 fw_size = le32_to_cpu(mec2_hdr->header.ucode_size_bytes) / 4;
4578 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4579 for (i = 0; i < fw_size; i++)
4580 WREG32(CP_MEC_ME2_UCODE_DATA, le32_to_cpup(fw_data++));
4581 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4584 const __be32 *fw_data;
4587 fw_data = (const __be32 *)rdev->mec_fw->data;
4588 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4589 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4590 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
4591 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4593 if (rdev->family == CHIP_KAVERI) {
4595 fw_data = (const __be32 *)rdev->mec_fw->data;
4596 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4597 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4598 WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
4599 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4607 * cik_cp_compute_start - start the compute queues
4609 * @rdev: radeon_device pointer
4611 * Enable the compute queues.
4612 * Returns 0 for success, error for failure.
4614 static int cik_cp_compute_start(struct radeon_device *rdev)
4616 cik_cp_compute_enable(rdev, true);
4622 * cik_cp_compute_fini - stop the compute queues
4624 * @rdev: radeon_device pointer
4626 * Stop the compute queues and tear down the driver queue
4629 static void cik_cp_compute_fini(struct radeon_device *rdev)
4633 cik_cp_compute_enable(rdev, false);
4635 for (i = 0; i < 2; i++) {
4637 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4639 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4641 if (rdev->ring[idx].mqd_obj) {
4642 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4643 if (unlikely(r != 0))
4644 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
4646 radeon_bo_unpin(rdev->ring[idx].mqd_obj);
4647 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4649 radeon_bo_unref(&rdev->ring[idx].mqd_obj);
4650 rdev->ring[idx].mqd_obj = NULL;
4655 static void cik_mec_fini(struct radeon_device *rdev)
4659 if (rdev->mec.hpd_eop_obj) {
4660 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4661 if (unlikely(r != 0))
4662 dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
4663 radeon_bo_unpin(rdev->mec.hpd_eop_obj);
4664 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4666 radeon_bo_unref(&rdev->mec.hpd_eop_obj);
4667 rdev->mec.hpd_eop_obj = NULL;
4671 #define MEC_HPD_SIZE 2048
4673 static int cik_mec_init(struct radeon_device *rdev)
4679 * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
4680 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
4682 if (rdev->family == CHIP_KAVERI)
4683 rdev->mec.num_mec = 2;
4685 rdev->mec.num_mec = 1;
4686 rdev->mec.num_pipe = 4;
4687 rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
4689 if (rdev->mec.hpd_eop_obj == NULL) {
4690 r = radeon_bo_create(rdev,
4691 rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
4693 RADEON_GEM_DOMAIN_GTT, 0, NULL,
4694 &rdev->mec.hpd_eop_obj);
4696 dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
4701 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4702 if (unlikely(r != 0)) {
4706 r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
4707 &rdev->mec.hpd_eop_gpu_addr);
4709 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
4713 r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
4715 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
4720 /* clear memory. Not sure if this is required or not */
4721 memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
4723 radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
4724 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4729 struct hqd_registers
4731 u32 cp_mqd_base_addr;
4732 u32 cp_mqd_base_addr_hi;
4735 u32 cp_hqd_persistent_state;
4736 u32 cp_hqd_pipe_priority;
4737 u32 cp_hqd_queue_priority;
4740 u32 cp_hqd_pq_base_hi;
4742 u32 cp_hqd_pq_rptr_report_addr;
4743 u32 cp_hqd_pq_rptr_report_addr_hi;
4744 u32 cp_hqd_pq_wptr_poll_addr;
4745 u32 cp_hqd_pq_wptr_poll_addr_hi;
4746 u32 cp_hqd_pq_doorbell_control;
4748 u32 cp_hqd_pq_control;
4749 u32 cp_hqd_ib_base_addr;
4750 u32 cp_hqd_ib_base_addr_hi;
4752 u32 cp_hqd_ib_control;
4753 u32 cp_hqd_iq_timer;
4755 u32 cp_hqd_dequeue_request;
4756 u32 cp_hqd_dma_offload;
4757 u32 cp_hqd_sema_cmd;
4758 u32 cp_hqd_msg_type;
4759 u32 cp_hqd_atomic0_preop_lo;
4760 u32 cp_hqd_atomic0_preop_hi;
4761 u32 cp_hqd_atomic1_preop_lo;
4762 u32 cp_hqd_atomic1_preop_hi;
4763 u32 cp_hqd_hq_scheduler0;
4764 u32 cp_hqd_hq_scheduler1;
4771 u32 dispatch_initiator;
4775 u32 pipeline_stat_enable;
4776 u32 perf_counter_enable;
4782 u32 resource_limits;
4783 u32 static_thread_mgmt01[2];
4785 u32 static_thread_mgmt23[2];
4787 u32 thread_trace_enable;
4790 u32 vgtcs_invoke_count[2];
4791 struct hqd_registers queue_state;
4793 u32 interrupt_queue[64];
4797 * cik_cp_compute_resume - setup the compute queue registers
4799 * @rdev: radeon_device pointer
4801 * Program the compute queues and test them to make sure they
4803 * Returns 0 for success, error for failure.
4805 static int cik_cp_compute_resume(struct radeon_device *rdev)
4809 bool use_doorbell = true;
4815 struct bonaire_mqd *mqd;
4817 r = cik_cp_compute_start(rdev);
4821 /* fix up chicken bits */
4822 tmp = RREG32(CP_CPF_DEBUG);
4824 WREG32(CP_CPF_DEBUG, tmp);
4826 /* init the pipes */
4827 mutex_lock(&rdev->srbm_mutex);
4828 for (i = 0; i < (rdev->mec.num_pipe * rdev->mec.num_mec); i++) {
4829 int me = (i < 4) ? 1 : 2;
4830 int pipe = (i < 4) ? i : (i - 4);
4832 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2);
4834 cik_srbm_select(rdev, me, pipe, 0, 0);
4836 /* write the EOP addr */
4837 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
4838 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
4840 /* set the VMID assigned */
4841 WREG32(CP_HPD_EOP_VMID, 0);
4843 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4844 tmp = RREG32(CP_HPD_EOP_CONTROL);
4845 tmp &= ~EOP_SIZE_MASK;
4846 tmp |= order_base_2(MEC_HPD_SIZE / 8);
4847 WREG32(CP_HPD_EOP_CONTROL, tmp);
4849 cik_srbm_select(rdev, 0, 0, 0, 0);
4850 mutex_unlock(&rdev->srbm_mutex);
4852 /* init the queues. Just two for now. */
4853 for (i = 0; i < 2; i++) {
4855 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4857 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4859 if (rdev->ring[idx].mqd_obj == NULL) {
4860 r = radeon_bo_create(rdev,
4861 sizeof(struct bonaire_mqd),
4863 RADEON_GEM_DOMAIN_GTT, 0, NULL,
4864 &rdev->ring[idx].mqd_obj);
4866 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
4871 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4872 if (unlikely(r != 0)) {
4873 cik_cp_compute_fini(rdev);
4876 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
4879 dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
4880 cik_cp_compute_fini(rdev);
4883 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
4885 dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
4886 cik_cp_compute_fini(rdev);
4890 /* init the mqd struct */
4891 memset(buf, 0, sizeof(struct bonaire_mqd));
4893 mqd = (struct bonaire_mqd *)buf;
4894 mqd->header = 0xC0310800;
4895 mqd->static_thread_mgmt01[0] = 0xffffffff;
4896 mqd->static_thread_mgmt01[1] = 0xffffffff;
4897 mqd->static_thread_mgmt23[0] = 0xffffffff;
4898 mqd->static_thread_mgmt23[1] = 0xffffffff;
4900 mutex_lock(&rdev->srbm_mutex);
4901 cik_srbm_select(rdev, rdev->ring[idx].me,
4902 rdev->ring[idx].pipe,
4903 rdev->ring[idx].queue, 0);
4905 /* disable wptr polling */
4906 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4907 tmp &= ~WPTR_POLL_EN;
4908 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4910 /* enable doorbell? */
4911 mqd->queue_state.cp_hqd_pq_doorbell_control =
4912 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4914 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4916 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
4917 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4918 mqd->queue_state.cp_hqd_pq_doorbell_control);
4920 /* disable the queue if it's active */
4921 mqd->queue_state.cp_hqd_dequeue_request = 0;
4922 mqd->queue_state.cp_hqd_pq_rptr = 0;
4923 mqd->queue_state.cp_hqd_pq_wptr= 0;
4924 if (RREG32(CP_HQD_ACTIVE) & 1) {
4925 WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
4926 for (i = 0; i < rdev->usec_timeout; i++) {
4927 if (!(RREG32(CP_HQD_ACTIVE) & 1))
4931 WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
4932 WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
4933 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4936 /* set the pointer to the MQD */
4937 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
4938 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
4939 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
4940 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
4941 /* set MQD vmid to 0 */
4942 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
4943 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
4944 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
4946 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4947 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
4948 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
4949 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4950 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
4951 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
4953 /* set up the HQD, this is similar to CP_RB0_CNTL */
4954 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
4955 mqd->queue_state.cp_hqd_pq_control &=
4956 ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
4958 mqd->queue_state.cp_hqd_pq_control |=
4959 order_base_2(rdev->ring[idx].ring_size / 8);
4960 mqd->queue_state.cp_hqd_pq_control |=
4961 (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8);
4963 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
4965 mqd->queue_state.cp_hqd_pq_control &=
4966 ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
4967 mqd->queue_state.cp_hqd_pq_control |=
4968 PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
4969 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
4971 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
4973 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
4975 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
4976 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
4977 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4978 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
4979 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
4980 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
4982 /* set the wb address wether it's enabled or not */
4984 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
4986 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
4987 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
4988 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
4989 upper_32_bits(wb_gpu_addr) & 0xffff;
4990 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
4991 mqd->queue_state.cp_hqd_pq_rptr_report_addr);
4992 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
4993 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
4995 /* enable the doorbell if requested */
4997 mqd->queue_state.cp_hqd_pq_doorbell_control =
4998 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4999 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
5000 mqd->queue_state.cp_hqd_pq_doorbell_control |=
5001 DOORBELL_OFFSET(rdev->ring[idx].doorbell_index);
5002 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
5003 mqd->queue_state.cp_hqd_pq_doorbell_control &=
5004 ~(DOORBELL_SOURCE | DOORBELL_HIT);
5007 mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
5009 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
5010 mqd->queue_state.cp_hqd_pq_doorbell_control);
5012 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
5013 rdev->ring[idx].wptr = 0;
5014 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
5015 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
5016 mqd->queue_state.cp_hqd_pq_rptr = RREG32(CP_HQD_PQ_RPTR);
5018 /* set the vmid for the queue */
5019 mqd->queue_state.cp_hqd_vmid = 0;
5020 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
5022 /* activate the queue */
5023 mqd->queue_state.cp_hqd_active = 1;
5024 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
5026 cik_srbm_select(rdev, 0, 0, 0, 0);
5027 mutex_unlock(&rdev->srbm_mutex);
5029 radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
5030 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
5032 rdev->ring[idx].ready = true;
5033 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
5035 rdev->ring[idx].ready = false;
5041 static void cik_cp_enable(struct radeon_device *rdev, bool enable)
5043 cik_cp_gfx_enable(rdev, enable);
5044 cik_cp_compute_enable(rdev, enable);
5047 static int cik_cp_load_microcode(struct radeon_device *rdev)
5051 r = cik_cp_gfx_load_microcode(rdev);
5054 r = cik_cp_compute_load_microcode(rdev);
5061 static void cik_cp_fini(struct radeon_device *rdev)
5063 cik_cp_gfx_fini(rdev);
5064 cik_cp_compute_fini(rdev);
5067 static int cik_cp_resume(struct radeon_device *rdev)
5071 cik_enable_gui_idle_interrupt(rdev, false);
5073 r = cik_cp_load_microcode(rdev);
5077 r = cik_cp_gfx_resume(rdev);
5080 r = cik_cp_compute_resume(rdev);
5084 cik_enable_gui_idle_interrupt(rdev, true);
5089 static void cik_print_gpu_status_regs(struct radeon_device *rdev)
5091 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
5092 RREG32(GRBM_STATUS));
5093 dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n",
5094 RREG32(GRBM_STATUS2));
5095 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
5096 RREG32(GRBM_STATUS_SE0));
5097 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
5098 RREG32(GRBM_STATUS_SE1));
5099 dev_info(rdev->dev, " GRBM_STATUS_SE2=0x%08X\n",
5100 RREG32(GRBM_STATUS_SE2));
5101 dev_info(rdev->dev, " GRBM_STATUS_SE3=0x%08X\n",
5102 RREG32(GRBM_STATUS_SE3));
5103 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
5104 RREG32(SRBM_STATUS));
5105 dev_info(rdev->dev, " SRBM_STATUS2=0x%08X\n",
5106 RREG32(SRBM_STATUS2));
5107 dev_info(rdev->dev, " SDMA0_STATUS_REG = 0x%08X\n",
5108 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
5109 dev_info(rdev->dev, " SDMA1_STATUS_REG = 0x%08X\n",
5110 RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
5111 dev_info(rdev->dev, " CP_STAT = 0x%08x\n", RREG32(CP_STAT));
5112 dev_info(rdev->dev, " CP_STALLED_STAT1 = 0x%08x\n",
5113 RREG32(CP_STALLED_STAT1));
5114 dev_info(rdev->dev, " CP_STALLED_STAT2 = 0x%08x\n",
5115 RREG32(CP_STALLED_STAT2));
5116 dev_info(rdev->dev, " CP_STALLED_STAT3 = 0x%08x\n",
5117 RREG32(CP_STALLED_STAT3));
5118 dev_info(rdev->dev, " CP_CPF_BUSY_STAT = 0x%08x\n",
5119 RREG32(CP_CPF_BUSY_STAT));
5120 dev_info(rdev->dev, " CP_CPF_STALLED_STAT1 = 0x%08x\n",
5121 RREG32(CP_CPF_STALLED_STAT1));
5122 dev_info(rdev->dev, " CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
5123 dev_info(rdev->dev, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
5124 dev_info(rdev->dev, " CP_CPC_STALLED_STAT1 = 0x%08x\n",
5125 RREG32(CP_CPC_STALLED_STAT1));
5126 dev_info(rdev->dev, " CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
5130 * cik_gpu_check_soft_reset - check which blocks are busy
5132 * @rdev: radeon_device pointer
5134 * Check which blocks are busy and return the relevant reset
5135 * mask to be used by cik_gpu_soft_reset().
5136 * Returns a mask of the blocks to be reset.
5138 u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
5144 tmp = RREG32(GRBM_STATUS);
5145 if (tmp & (PA_BUSY | SC_BUSY |
5146 BCI_BUSY | SX_BUSY |
5147 TA_BUSY | VGT_BUSY |
5149 GDS_BUSY | SPI_BUSY |
5150 IA_BUSY | IA_BUSY_NO_DMA))
5151 reset_mask |= RADEON_RESET_GFX;
5153 if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
5154 reset_mask |= RADEON_RESET_CP;
5157 tmp = RREG32(GRBM_STATUS2);
5159 reset_mask |= RADEON_RESET_RLC;
5161 /* SDMA0_STATUS_REG */
5162 tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
5163 if (!(tmp & SDMA_IDLE))
5164 reset_mask |= RADEON_RESET_DMA;
5166 /* SDMA1_STATUS_REG */
5167 tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
5168 if (!(tmp & SDMA_IDLE))
5169 reset_mask |= RADEON_RESET_DMA1;
5172 tmp = RREG32(SRBM_STATUS2);
5173 if (tmp & SDMA_BUSY)
5174 reset_mask |= RADEON_RESET_DMA;
5176 if (tmp & SDMA1_BUSY)
5177 reset_mask |= RADEON_RESET_DMA1;
5180 tmp = RREG32(SRBM_STATUS);
5183 reset_mask |= RADEON_RESET_IH;
5186 reset_mask |= RADEON_RESET_SEM;
5188 if (tmp & GRBM_RQ_PENDING)
5189 reset_mask |= RADEON_RESET_GRBM;
5192 reset_mask |= RADEON_RESET_VMC;
5194 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
5195 MCC_BUSY | MCD_BUSY))
5196 reset_mask |= RADEON_RESET_MC;
5198 if (evergreen_is_display_hung(rdev))
5199 reset_mask |= RADEON_RESET_DISPLAY;
5201 /* Skip MC reset as it's mostly likely not hung, just busy */
5202 if (reset_mask & RADEON_RESET_MC) {
5203 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
5204 reset_mask &= ~RADEON_RESET_MC;
5211 * cik_gpu_soft_reset - soft reset GPU
5213 * @rdev: radeon_device pointer
5214 * @reset_mask: mask of which blocks to reset
5216 * Soft reset the blocks specified in @reset_mask.
5218 static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
5220 struct evergreen_mc_save save;
5221 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
5224 if (reset_mask == 0)
5227 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
5229 cik_print_gpu_status_regs(rdev);
5230 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
5231 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
5232 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
5233 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
5242 /* Disable GFX parsing/prefetching */
5243 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
5245 /* Disable MEC parsing/prefetching */
5246 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
5248 if (reset_mask & RADEON_RESET_DMA) {
5250 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5252 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5254 if (reset_mask & RADEON_RESET_DMA1) {
5256 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5258 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5261 evergreen_mc_stop(rdev, &save);
5262 if (evergreen_mc_wait_for_idle(rdev)) {
5263 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5266 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
5267 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
5269 if (reset_mask & RADEON_RESET_CP) {
5270 grbm_soft_reset |= SOFT_RESET_CP;
5272 srbm_soft_reset |= SOFT_RESET_GRBM;
5275 if (reset_mask & RADEON_RESET_DMA)
5276 srbm_soft_reset |= SOFT_RESET_SDMA;
5278 if (reset_mask & RADEON_RESET_DMA1)
5279 srbm_soft_reset |= SOFT_RESET_SDMA1;
5281 if (reset_mask & RADEON_RESET_DISPLAY)
5282 srbm_soft_reset |= SOFT_RESET_DC;
5284 if (reset_mask & RADEON_RESET_RLC)
5285 grbm_soft_reset |= SOFT_RESET_RLC;
5287 if (reset_mask & RADEON_RESET_SEM)
5288 srbm_soft_reset |= SOFT_RESET_SEM;
5290 if (reset_mask & RADEON_RESET_IH)
5291 srbm_soft_reset |= SOFT_RESET_IH;
5293 if (reset_mask & RADEON_RESET_GRBM)
5294 srbm_soft_reset |= SOFT_RESET_GRBM;
5296 if (reset_mask & RADEON_RESET_VMC)
5297 srbm_soft_reset |= SOFT_RESET_VMC;
5299 if (!(rdev->flags & RADEON_IS_IGP)) {
5300 if (reset_mask & RADEON_RESET_MC)
5301 srbm_soft_reset |= SOFT_RESET_MC;
5304 if (grbm_soft_reset) {
5305 tmp = RREG32(GRBM_SOFT_RESET);
5306 tmp |= grbm_soft_reset;
5307 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
5308 WREG32(GRBM_SOFT_RESET, tmp);
5309 tmp = RREG32(GRBM_SOFT_RESET);
5313 tmp &= ~grbm_soft_reset;
5314 WREG32(GRBM_SOFT_RESET, tmp);
5315 tmp = RREG32(GRBM_SOFT_RESET);
5318 if (srbm_soft_reset) {
5319 tmp = RREG32(SRBM_SOFT_RESET);
5320 tmp |= srbm_soft_reset;
5321 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
5322 WREG32(SRBM_SOFT_RESET, tmp);
5323 tmp = RREG32(SRBM_SOFT_RESET);
5327 tmp &= ~srbm_soft_reset;
5328 WREG32(SRBM_SOFT_RESET, tmp);
5329 tmp = RREG32(SRBM_SOFT_RESET);
5332 /* Wait a little for things to settle down */
5335 evergreen_mc_resume(rdev, &save);
5338 cik_print_gpu_status_regs(rdev);
5341 struct kv_reset_save_regs {
5342 u32 gmcon_reng_execute;
5347 static void kv_save_regs_for_reset(struct radeon_device *rdev,
5348 struct kv_reset_save_regs *save)
5350 save->gmcon_reng_execute = RREG32(GMCON_RENG_EXECUTE);
5351 save->gmcon_misc = RREG32(GMCON_MISC);
5352 save->gmcon_misc3 = RREG32(GMCON_MISC3);
5354 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute & ~RENG_EXECUTE_ON_PWR_UP);
5355 WREG32(GMCON_MISC, save->gmcon_misc & ~(RENG_EXECUTE_ON_REG_UPDATE |
5356 STCTRL_STUTTER_EN));
5359 static void kv_restore_regs_for_reset(struct radeon_device *rdev,
5360 struct kv_reset_save_regs *save)
5364 WREG32(GMCON_PGFSM_WRITE, 0);
5365 WREG32(GMCON_PGFSM_CONFIG, 0x200010ff);
5367 for (i = 0; i < 5; i++)
5368 WREG32(GMCON_PGFSM_WRITE, 0);
5370 WREG32(GMCON_PGFSM_WRITE, 0);
5371 WREG32(GMCON_PGFSM_CONFIG, 0x300010ff);
5373 for (i = 0; i < 5; i++)
5374 WREG32(GMCON_PGFSM_WRITE, 0);
5376 WREG32(GMCON_PGFSM_WRITE, 0x210000);
5377 WREG32(GMCON_PGFSM_CONFIG, 0xa00010ff);
5379 for (i = 0; i < 5; i++)
5380 WREG32(GMCON_PGFSM_WRITE, 0);
5382 WREG32(GMCON_PGFSM_WRITE, 0x21003);
5383 WREG32(GMCON_PGFSM_CONFIG, 0xb00010ff);
5385 for (i = 0; i < 5; i++)
5386 WREG32(GMCON_PGFSM_WRITE, 0);
5388 WREG32(GMCON_PGFSM_WRITE, 0x2b00);
5389 WREG32(GMCON_PGFSM_CONFIG, 0xc00010ff);
5391 for (i = 0; i < 5; i++)
5392 WREG32(GMCON_PGFSM_WRITE, 0);
5394 WREG32(GMCON_PGFSM_WRITE, 0);
5395 WREG32(GMCON_PGFSM_CONFIG, 0xd00010ff);
5397 for (i = 0; i < 5; i++)
5398 WREG32(GMCON_PGFSM_WRITE, 0);
5400 WREG32(GMCON_PGFSM_WRITE, 0x420000);
5401 WREG32(GMCON_PGFSM_CONFIG, 0x100010ff);
5403 for (i = 0; i < 5; i++)
5404 WREG32(GMCON_PGFSM_WRITE, 0);
5406 WREG32(GMCON_PGFSM_WRITE, 0x120202);
5407 WREG32(GMCON_PGFSM_CONFIG, 0x500010ff);
5409 for (i = 0; i < 5; i++)
5410 WREG32(GMCON_PGFSM_WRITE, 0);
5412 WREG32(GMCON_PGFSM_WRITE, 0x3e3e36);
5413 WREG32(GMCON_PGFSM_CONFIG, 0x600010ff);
5415 for (i = 0; i < 5; i++)
5416 WREG32(GMCON_PGFSM_WRITE, 0);
5418 WREG32(GMCON_PGFSM_WRITE, 0x373f3e);
5419 WREG32(GMCON_PGFSM_CONFIG, 0x700010ff);
5421 for (i = 0; i < 5; i++)
5422 WREG32(GMCON_PGFSM_WRITE, 0);
5424 WREG32(GMCON_PGFSM_WRITE, 0x3e1332);
5425 WREG32(GMCON_PGFSM_CONFIG, 0xe00010ff);
5427 WREG32(GMCON_MISC3, save->gmcon_misc3);
5428 WREG32(GMCON_MISC, save->gmcon_misc);
5429 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute);
5432 static void cik_gpu_pci_config_reset(struct radeon_device *rdev)
5434 struct evergreen_mc_save save;
5435 struct kv_reset_save_regs kv_save = { 0 };
5438 dev_info(rdev->dev, "GPU pci config reset\n");
5446 /* Disable GFX parsing/prefetching */
5447 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
5449 /* Disable MEC parsing/prefetching */
5450 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
5453 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5455 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5457 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5459 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5460 /* XXX other engines? */
5462 /* halt the rlc, disable cp internal ints */
5467 /* disable mem access */
5468 evergreen_mc_stop(rdev, &save);
5469 if (evergreen_mc_wait_for_idle(rdev)) {
5470 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
5473 if (rdev->flags & RADEON_IS_IGP)
5474 kv_save_regs_for_reset(rdev, &kv_save);
5477 pci_clear_master(rdev->pdev);
5479 radeon_pci_config_reset(rdev);
5483 /* wait for asic to come out of reset */
5484 for (i = 0; i < rdev->usec_timeout; i++) {
5485 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
5490 /* does asic init need to be run first??? */
5491 if (rdev->flags & RADEON_IS_IGP)
5492 kv_restore_regs_for_reset(rdev, &kv_save);
5496 * cik_asic_reset - soft reset GPU
5498 * @rdev: radeon_device pointer
5500 * Look up which blocks are hung and attempt
5502 * Returns 0 for success.
5504 int cik_asic_reset(struct radeon_device *rdev)
5508 reset_mask = cik_gpu_check_soft_reset(rdev);
5511 r600_set_bios_scratch_engine_hung(rdev, true);
5513 /* try soft reset */
5514 cik_gpu_soft_reset(rdev, reset_mask);
5516 reset_mask = cik_gpu_check_soft_reset(rdev);
5518 /* try pci config reset */
5519 if (reset_mask && radeon_hard_reset)
5520 cik_gpu_pci_config_reset(rdev);
5522 reset_mask = cik_gpu_check_soft_reset(rdev);
5525 r600_set_bios_scratch_engine_hung(rdev, false);
5531 * cik_gfx_is_lockup - check if the 3D engine is locked up
5533 * @rdev: radeon_device pointer
5534 * @ring: radeon_ring structure holding ring information
5536 * Check if the 3D engine is locked up (CIK).
5537 * Returns true if the engine is locked, false if not.
5539 bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
5541 u32 reset_mask = cik_gpu_check_soft_reset(rdev);
5543 if (!(reset_mask & (RADEON_RESET_GFX |
5544 RADEON_RESET_COMPUTE |
5545 RADEON_RESET_CP))) {
5546 radeon_ring_lockup_update(rdev, ring);
5549 return radeon_ring_test_lockup(rdev, ring);
5554 * cik_mc_program - program the GPU memory controller
5556 * @rdev: radeon_device pointer
5558 * Set the location of vram, gart, and AGP in the GPU's
5559 * physical address space (CIK).
5561 static void cik_mc_program(struct radeon_device *rdev)
5563 struct evergreen_mc_save save;
5567 /* Initialize HDP */
5568 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
5569 WREG32((0x2c14 + j), 0x00000000);
5570 WREG32((0x2c18 + j), 0x00000000);
5571 WREG32((0x2c1c + j), 0x00000000);
5572 WREG32((0x2c20 + j), 0x00000000);
5573 WREG32((0x2c24 + j), 0x00000000);
5575 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
5577 evergreen_mc_stop(rdev, &save);
5578 if (radeon_mc_wait_for_idle(rdev)) {
5579 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5581 /* Lockout access through VGA aperture*/
5582 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
5583 /* Update configuration */
5584 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
5585 rdev->mc.vram_start >> 12);
5586 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
5587 rdev->mc.vram_end >> 12);
5588 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
5589 rdev->vram_scratch.gpu_addr >> 12);
5590 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
5591 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
5592 WREG32(MC_VM_FB_LOCATION, tmp);
5593 /* XXX double check these! */
5594 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
5595 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
5596 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
5597 WREG32(MC_VM_AGP_BASE, 0);
5598 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
5599 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
5600 if (radeon_mc_wait_for_idle(rdev)) {
5601 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5603 evergreen_mc_resume(rdev, &save);
5604 /* we need to own VRAM, so turn off the VGA renderer here
5605 * to stop it overwriting our objects */
5606 rv515_vga_render_disable(rdev);
5610 * cik_mc_init - initialize the memory controller driver params
5612 * @rdev: radeon_device pointer
5614 * Look up the amount of vram, vram width, and decide how to place
5615 * vram and gart within the GPU's physical address space (CIK).
5616 * Returns 0 for success.
5618 static int cik_mc_init(struct radeon_device *rdev)
5621 int chansize, numchan;
5623 /* Get VRAM informations */
5624 rdev->mc.vram_is_ddr = true;
5625 tmp = RREG32(MC_ARB_RAMCFG);
5626 if (tmp & CHANSIZE_MASK) {
5631 tmp = RREG32(MC_SHARED_CHMAP);
5632 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
5662 rdev->mc.vram_width = numchan * chansize;
5663 /* Could aper size report 0 ? */
5664 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
5665 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
5666 /* size in MB on si */
5667 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5668 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5669 rdev->mc.visible_vram_size = rdev->mc.aper_size;
5670 si_vram_gtt_location(rdev, &rdev->mc);
5671 radeon_update_bandwidth_info(rdev);
5678 * VMID 0 is the physical GPU addresses as used by the kernel.
5679 * VMIDs 1-15 are used for userspace clients and are handled
5680 * by the radeon vm/hsa code.
5683 * cik_pcie_gart_tlb_flush - gart tlb flush callback
5685 * @rdev: radeon_device pointer
5687 * Flush the TLB for the VMID 0 page table (CIK).
5689 void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
5691 /* flush hdp cache */
5692 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
5694 /* bits 0-15 are the VM contexts0-15 */
5695 WREG32(VM_INVALIDATE_REQUEST, 0x1);
5699 * cik_pcie_gart_enable - gart enable
5701 * @rdev: radeon_device pointer
5703 * This sets up the TLBs, programs the page tables for VMID0,
5704 * sets up the hw for VMIDs 1-15 which are allocated on
5705 * demand, and sets up the global locations for the LDS, GDS,
5706 * and GPUVM for FSA64 clients (CIK).
5707 * Returns 0 for success, errors for failure.
5709 static int cik_pcie_gart_enable(struct radeon_device *rdev)
5713 if (rdev->gart.robj == NULL) {
5714 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
5717 r = radeon_gart_table_vram_pin(rdev);
5720 /* Setup TLB control */
5721 WREG32(MC_VM_MX_L1_TLB_CNTL,
5724 ENABLE_L1_FRAGMENT_PROCESSING |
5725 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5726 ENABLE_ADVANCED_DRIVER_MODEL |
5727 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5728 /* Setup L2 cache */
5729 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
5730 ENABLE_L2_FRAGMENT_PROCESSING |
5731 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5732 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5733 EFFECTIVE_L2_QUEUE_SIZE(7) |
5734 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5735 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
5736 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5738 L2_CACHE_BIGK_FRAGMENT_SIZE(4));
5739 /* setup context0 */
5740 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
5741 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
5742 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
5743 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
5744 (u32)(rdev->dummy_page.addr >> 12));
5745 WREG32(VM_CONTEXT0_CNTL2, 0);
5746 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
5747 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
5753 /* empty context1-15 */
5754 /* FIXME start with 4G, once using 2 level pt switch to full
5757 /* set vm size, must be a multiple of 4 */
5758 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
5759 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
5760 for (i = 1; i < 16; i++) {
5762 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
5763 rdev->gart.table_addr >> 12);
5765 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
5766 rdev->gart.table_addr >> 12);
5769 /* enable context1-15 */
5770 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
5771 (u32)(rdev->dummy_page.addr >> 12));
5772 WREG32(VM_CONTEXT1_CNTL2, 4);
5773 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
5774 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
5775 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5776 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5777 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5778 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5779 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
5780 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
5781 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
5782 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
5783 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
5784 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
5785 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5786 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
5788 if (rdev->family == CHIP_KAVERI) {
5789 u32 tmp = RREG32(CHUB_CONTROL);
5791 WREG32(CHUB_CONTROL, tmp);
5794 /* XXX SH_MEM regs */
5795 /* where to put LDS, scratch, GPUVM in FSA64 space */
5796 mutex_lock(&rdev->srbm_mutex);
5797 for (i = 0; i < 16; i++) {
5798 cik_srbm_select(rdev, 0, 0, 0, i);
5799 /* CP and shaders */
5800 WREG32(SH_MEM_CONFIG, 0);
5801 WREG32(SH_MEM_APE1_BASE, 1);
5802 WREG32(SH_MEM_APE1_LIMIT, 0);
5803 WREG32(SH_MEM_BASES, 0);
5805 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
5806 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
5807 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
5808 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
5809 /* XXX SDMA RLC - todo */
5811 cik_srbm_select(rdev, 0, 0, 0, 0);
5812 mutex_unlock(&rdev->srbm_mutex);
5814 cik_pcie_gart_tlb_flush(rdev);
5815 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
5816 (unsigned)(rdev->mc.gtt_size >> 20),
5817 (unsigned long long)rdev->gart.table_addr);
5818 rdev->gart.ready = true;
5823 * cik_pcie_gart_disable - gart disable
5825 * @rdev: radeon_device pointer
5827 * This disables all VM page table (CIK).
5829 static void cik_pcie_gart_disable(struct radeon_device *rdev)
5831 /* Disable all tables */
5832 WREG32(VM_CONTEXT0_CNTL, 0);
5833 WREG32(VM_CONTEXT1_CNTL, 0);
5834 /* Setup TLB control */
5835 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5836 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5837 /* Setup L2 cache */
5839 ENABLE_L2_FRAGMENT_PROCESSING |
5840 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5841 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5842 EFFECTIVE_L2_QUEUE_SIZE(7) |
5843 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5844 WREG32(VM_L2_CNTL2, 0);
5845 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5846 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5847 radeon_gart_table_vram_unpin(rdev);
5851 * cik_pcie_gart_fini - vm fini callback
5853 * @rdev: radeon_device pointer
5855 * Tears down the driver GART/VM setup (CIK).
5857 static void cik_pcie_gart_fini(struct radeon_device *rdev)
5859 cik_pcie_gart_disable(rdev);
5860 radeon_gart_table_vram_free(rdev);
5861 radeon_gart_fini(rdev);
5866 * cik_ib_parse - vm ib_parse callback
5868 * @rdev: radeon_device pointer
5869 * @ib: indirect buffer pointer
5871 * CIK uses hw IB checking so this is a nop (CIK).
5873 int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
5880 * VMID 0 is the physical GPU addresses as used by the kernel.
5881 * VMIDs 1-15 are used for userspace clients and are handled
5882 * by the radeon vm/hsa code.
5885 * cik_vm_init - cik vm init callback
5887 * @rdev: radeon_device pointer
5889 * Inits cik specific vm parameters (number of VMs, base of vram for
5890 * VMIDs 1-15) (CIK).
5891 * Returns 0 for success.
5893 int cik_vm_init(struct radeon_device *rdev)
5896 rdev->vm_manager.nvm = 16;
5897 /* base offset of vram pages */
5898 if (rdev->flags & RADEON_IS_IGP) {
5899 u64 tmp = RREG32(MC_VM_FB_OFFSET);
5901 rdev->vm_manager.vram_base_offset = tmp;
5903 rdev->vm_manager.vram_base_offset = 0;
5909 * cik_vm_fini - cik vm fini callback
5911 * @rdev: radeon_device pointer
5913 * Tear down any asic specific VM setup (CIK).
5915 void cik_vm_fini(struct radeon_device *rdev)
5920 * cik_vm_decode_fault - print human readable fault info
5922 * @rdev: radeon_device pointer
5923 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
5924 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
5926 * Print human readable fault information (CIK).
5928 static void cik_vm_decode_fault(struct radeon_device *rdev,
5929 u32 status, u32 addr, u32 mc_client)
5932 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
5933 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
5934 char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
5935 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
5937 if (rdev->family == CHIP_HAWAII)
5938 mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5940 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5942 printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
5943 protections, vmid, addr,
5944 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
5945 block, mc_client, mc_id);
5949 * cik_vm_flush - cik vm flush using the CP
5951 * @rdev: radeon_device pointer
5953 * Update the page table base and flush the VM TLB
5954 * using the CP (CIK).
5956 void cik_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
5958 struct radeon_ring *ring = &rdev->ring[ridx];
5959 int usepfp = (ridx == RADEON_RING_TYPE_GFX_INDEX);
5964 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5965 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
5966 WRITE_DATA_DST_SEL(0)));
5968 radeon_ring_write(ring,
5969 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
5971 radeon_ring_write(ring,
5972 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
5974 radeon_ring_write(ring, 0);
5975 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
5977 /* update SH_MEM_* regs */
5978 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5979 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
5980 WRITE_DATA_DST_SEL(0)));
5981 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5982 radeon_ring_write(ring, 0);
5983 radeon_ring_write(ring, VMID(vm->id));
5985 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
5986 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
5987 WRITE_DATA_DST_SEL(0)));
5988 radeon_ring_write(ring, SH_MEM_BASES >> 2);
5989 radeon_ring_write(ring, 0);
5991 radeon_ring_write(ring, 0); /* SH_MEM_BASES */
5992 radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */
5993 radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
5994 radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
5996 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5997 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
5998 WRITE_DATA_DST_SEL(0)));
5999 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
6000 radeon_ring_write(ring, 0);
6001 radeon_ring_write(ring, VMID(0));
6004 cik_hdp_flush_cp_ring_emit(rdev, ridx);
6006 /* bits 0-15 are the VM contexts0-15 */
6007 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
6008 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
6009 WRITE_DATA_DST_SEL(0)));
6010 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
6011 radeon_ring_write(ring, 0);
6012 radeon_ring_write(ring, 1 << vm->id);
6014 /* compute doesn't have PFP */
6016 /* sync PFP to ME, otherwise we might get invalid PFP reads */
6017 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
6018 radeon_ring_write(ring, 0x0);
6024 * The RLC is a multi-purpose microengine that handles a
6025 * variety of functions, the most important of which is
6026 * the interrupt controller.
6028 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
6031 u32 tmp = RREG32(CP_INT_CNTL_RING0);
6034 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6036 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6037 WREG32(CP_INT_CNTL_RING0, tmp);
6040 static void cik_enable_lbpw(struct radeon_device *rdev, bool enable)
6044 tmp = RREG32(RLC_LB_CNTL);
6046 tmp |= LOAD_BALANCE_ENABLE;
6048 tmp &= ~LOAD_BALANCE_ENABLE;
6049 WREG32(RLC_LB_CNTL, tmp);
6052 static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)
6057 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6058 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6059 cik_select_se_sh(rdev, i, j);
6060 for (k = 0; k < rdev->usec_timeout; k++) {
6061 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
6067 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6069 mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
6070 for (k = 0; k < rdev->usec_timeout; k++) {
6071 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
6077 static void cik_update_rlc(struct radeon_device *rdev, u32 rlc)
6081 tmp = RREG32(RLC_CNTL);
6083 WREG32(RLC_CNTL, rlc);
6086 static u32 cik_halt_rlc(struct radeon_device *rdev)
6090 orig = data = RREG32(RLC_CNTL);
6092 if (data & RLC_ENABLE) {
6095 data &= ~RLC_ENABLE;
6096 WREG32(RLC_CNTL, data);
6098 for (i = 0; i < rdev->usec_timeout; i++) {
6099 if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0)
6104 cik_wait_for_rlc_serdes(rdev);
6110 void cik_enter_rlc_safe_mode(struct radeon_device *rdev)
6114 tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE);
6115 WREG32(RLC_GPR_REG2, tmp);
6117 mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS;
6118 for (i = 0; i < rdev->usec_timeout; i++) {
6119 if ((RREG32(RLC_GPM_STAT) & mask) == mask)
6124 for (i = 0; i < rdev->usec_timeout; i++) {
6125 if ((RREG32(RLC_GPR_REG2) & REQ) == 0)
6131 void cik_exit_rlc_safe_mode(struct radeon_device *rdev)
6135 tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE);
6136 WREG32(RLC_GPR_REG2, tmp);
6140 * cik_rlc_stop - stop the RLC ME
6142 * @rdev: radeon_device pointer
6144 * Halt the RLC ME (MicroEngine) (CIK).
6146 static void cik_rlc_stop(struct radeon_device *rdev)
6148 WREG32(RLC_CNTL, 0);
6150 cik_enable_gui_idle_interrupt(rdev, false);
6152 cik_wait_for_rlc_serdes(rdev);
6156 * cik_rlc_start - start the RLC ME
6158 * @rdev: radeon_device pointer
6160 * Unhalt the RLC ME (MicroEngine) (CIK).
6162 static void cik_rlc_start(struct radeon_device *rdev)
6164 WREG32(RLC_CNTL, RLC_ENABLE);
6166 cik_enable_gui_idle_interrupt(rdev, true);
6172 * cik_rlc_resume - setup the RLC hw
6174 * @rdev: radeon_device pointer
6176 * Initialize the RLC registers, load the ucode,
6177 * and start the RLC (CIK).
6178 * Returns 0 for success, -EINVAL if the ucode is not available.
6180 static int cik_rlc_resume(struct radeon_device *rdev)
6190 tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
6191 WREG32(RLC_CGCG_CGLS_CTRL, tmp);
6199 WREG32(RLC_LB_CNTR_INIT, 0);
6200 WREG32(RLC_LB_CNTR_MAX, 0x00008000);
6202 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6203 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
6204 WREG32(RLC_LB_PARAMS, 0x00600408);
6205 WREG32(RLC_LB_CNTL, 0x80000004);
6207 WREG32(RLC_MC_CNTL, 0);
6208 WREG32(RLC_UCODE_CNTL, 0);
6211 const struct rlc_firmware_header_v1_0 *hdr =
6212 (const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data;
6213 const __le32 *fw_data = (const __le32 *)
6214 (rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6216 radeon_ucode_print_rlc_hdr(&hdr->header);
6218 size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
6219 WREG32(RLC_GPM_UCODE_ADDR, 0);
6220 for (i = 0; i < size; i++)
6221 WREG32(RLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++));
6222 WREG32(RLC_GPM_UCODE_ADDR, 0);
6224 const __be32 *fw_data;
6226 switch (rdev->family) {
6230 size = BONAIRE_RLC_UCODE_SIZE;
6233 size = KV_RLC_UCODE_SIZE;
6236 size = KB_RLC_UCODE_SIZE;
6239 size = ML_RLC_UCODE_SIZE;
6243 fw_data = (const __be32 *)rdev->rlc_fw->data;
6244 WREG32(RLC_GPM_UCODE_ADDR, 0);
6245 for (i = 0; i < size; i++)
6246 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
6247 WREG32(RLC_GPM_UCODE_ADDR, 0);
6250 /* XXX - find out what chips support lbpw */
6251 cik_enable_lbpw(rdev, false);
6253 if (rdev->family == CHIP_BONAIRE)
6254 WREG32(RLC_DRIVER_DMA_STATUS, 0);
6256 cik_rlc_start(rdev);
6261 static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)
6263 u32 data, orig, tmp, tmp2;
6265 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
6267 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
6268 cik_enable_gui_idle_interrupt(rdev, true);
6270 tmp = cik_halt_rlc(rdev);
6272 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6273 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6274 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6275 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
6276 WREG32(RLC_SERDES_WR_CTRL, tmp2);
6278 cik_update_rlc(rdev, tmp);
6280 data |= CGCG_EN | CGLS_EN;
6282 cik_enable_gui_idle_interrupt(rdev, false);
6284 RREG32(CB_CGTT_SCLK_CTRL);
6285 RREG32(CB_CGTT_SCLK_CTRL);
6286 RREG32(CB_CGTT_SCLK_CTRL);
6287 RREG32(CB_CGTT_SCLK_CTRL);
6289 data &= ~(CGCG_EN | CGLS_EN);
6293 WREG32(RLC_CGCG_CGLS_CTRL, data);
6297 static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)
6299 u32 data, orig, tmp = 0;
6301 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
6302 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {
6303 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
6304 orig = data = RREG32(CP_MEM_SLP_CNTL);
6305 data |= CP_MEM_LS_EN;
6307 WREG32(CP_MEM_SLP_CNTL, data);
6311 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
6314 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
6316 tmp = cik_halt_rlc(rdev);
6318 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6319 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6320 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6321 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
6322 WREG32(RLC_SERDES_WR_CTRL, data);
6324 cik_update_rlc(rdev, tmp);
6326 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {
6327 orig = data = RREG32(CGTS_SM_CTRL_REG);
6328 data &= ~SM_MODE_MASK;
6329 data |= SM_MODE(0x2);
6330 data |= SM_MODE_ENABLE;
6331 data &= ~CGTS_OVERRIDE;
6332 if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&
6333 (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))
6334 data &= ~CGTS_LS_OVERRIDE;
6335 data &= ~ON_MONITOR_ADD_MASK;
6336 data |= ON_MONITOR_ADD_EN;
6337 data |= ON_MONITOR_ADD(0x96);
6339 WREG32(CGTS_SM_CTRL_REG, data);
6342 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
6345 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
6347 data = RREG32(RLC_MEM_SLP_CNTL);
6348 if (data & RLC_MEM_LS_EN) {
6349 data &= ~RLC_MEM_LS_EN;
6350 WREG32(RLC_MEM_SLP_CNTL, data);
6353 data = RREG32(CP_MEM_SLP_CNTL);
6354 if (data & CP_MEM_LS_EN) {
6355 data &= ~CP_MEM_LS_EN;
6356 WREG32(CP_MEM_SLP_CNTL, data);
6359 orig = data = RREG32(CGTS_SM_CTRL_REG);
6360 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE;
6362 WREG32(CGTS_SM_CTRL_REG, data);
6364 tmp = cik_halt_rlc(rdev);
6366 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6367 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6368 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6369 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
6370 WREG32(RLC_SERDES_WR_CTRL, data);
6372 cik_update_rlc(rdev, tmp);
6376 static const u32 mc_cg_registers[] =
6389 static void cik_enable_mc_ls(struct radeon_device *rdev,
6395 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6396 orig = data = RREG32(mc_cg_registers[i]);
6397 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
6398 data |= MC_LS_ENABLE;
6400 data &= ~MC_LS_ENABLE;
6402 WREG32(mc_cg_registers[i], data);
6406 static void cik_enable_mc_mgcg(struct radeon_device *rdev,
6412 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6413 orig = data = RREG32(mc_cg_registers[i]);
6414 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
6415 data |= MC_CG_ENABLE;
6417 data &= ~MC_CG_ENABLE;
6419 WREG32(mc_cg_registers[i], data);
6423 static void cik_enable_sdma_mgcg(struct radeon_device *rdev,
6428 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
6429 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);
6430 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);
6432 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET);
6435 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data);
6437 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET);
6440 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data);
6444 static void cik_enable_sdma_mgls(struct radeon_device *rdev,
6449 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {
6450 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6453 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6455 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6458 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6460 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6463 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6465 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6468 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6472 static void cik_enable_uvd_mgcg(struct radeon_device *rdev,
6477 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
6478 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6480 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6482 orig = data = RREG32(UVD_CGC_CTRL);
6485 WREG32(UVD_CGC_CTRL, data);
6487 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6489 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6491 orig = data = RREG32(UVD_CGC_CTRL);
6494 WREG32(UVD_CGC_CTRL, data);
6498 static void cik_enable_bif_mgls(struct radeon_device *rdev,
6503 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
6505 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
6506 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
6507 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
6509 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
6510 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
6513 WREG32_PCIE_PORT(PCIE_CNTL2, data);
6516 static void cik_enable_hdp_mgcg(struct radeon_device *rdev,
6521 orig = data = RREG32(HDP_HOST_PATH_CNTL);
6523 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
6524 data &= ~CLOCK_GATING_DIS;
6526 data |= CLOCK_GATING_DIS;
6529 WREG32(HDP_HOST_PATH_CNTL, data);
6532 static void cik_enable_hdp_ls(struct radeon_device *rdev,
6537 orig = data = RREG32(HDP_MEM_POWER_LS);
6539 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
6540 data |= HDP_LS_ENABLE;
6542 data &= ~HDP_LS_ENABLE;
6545 WREG32(HDP_MEM_POWER_LS, data);
6548 void cik_update_cg(struct radeon_device *rdev,
6549 u32 block, bool enable)
6552 if (block & RADEON_CG_BLOCK_GFX) {
6553 cik_enable_gui_idle_interrupt(rdev, false);
6554 /* order matters! */
6556 cik_enable_mgcg(rdev, true);
6557 cik_enable_cgcg(rdev, true);
6559 cik_enable_cgcg(rdev, false);
6560 cik_enable_mgcg(rdev, false);
6562 cik_enable_gui_idle_interrupt(rdev, true);
6565 if (block & RADEON_CG_BLOCK_MC) {
6566 if (!(rdev->flags & RADEON_IS_IGP)) {
6567 cik_enable_mc_mgcg(rdev, enable);
6568 cik_enable_mc_ls(rdev, enable);
6572 if (block & RADEON_CG_BLOCK_SDMA) {
6573 cik_enable_sdma_mgcg(rdev, enable);
6574 cik_enable_sdma_mgls(rdev, enable);
6577 if (block & RADEON_CG_BLOCK_BIF) {
6578 cik_enable_bif_mgls(rdev, enable);
6581 if (block & RADEON_CG_BLOCK_UVD) {
6583 cik_enable_uvd_mgcg(rdev, enable);
6586 if (block & RADEON_CG_BLOCK_HDP) {
6587 cik_enable_hdp_mgcg(rdev, enable);
6588 cik_enable_hdp_ls(rdev, enable);
6591 if (block & RADEON_CG_BLOCK_VCE) {
6592 vce_v2_0_enable_mgcg(rdev, enable);
6596 static void cik_init_cg(struct radeon_device *rdev)
6599 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true);
6602 si_init_uvd_internal_cg(rdev);
6604 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6605 RADEON_CG_BLOCK_SDMA |
6606 RADEON_CG_BLOCK_BIF |
6607 RADEON_CG_BLOCK_UVD |
6608 RADEON_CG_BLOCK_HDP), true);
6611 static void cik_fini_cg(struct radeon_device *rdev)
6613 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6614 RADEON_CG_BLOCK_SDMA |
6615 RADEON_CG_BLOCK_BIF |
6616 RADEON_CG_BLOCK_UVD |
6617 RADEON_CG_BLOCK_HDP), false);
6619 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false);
6622 static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev,
6627 orig = data = RREG32(RLC_PG_CNTL);
6628 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
6629 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6631 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6633 WREG32(RLC_PG_CNTL, data);
6636 static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev,
6641 orig = data = RREG32(RLC_PG_CNTL);
6642 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
6643 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6645 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6647 WREG32(RLC_PG_CNTL, data);
6650 static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable)
6654 orig = data = RREG32(RLC_PG_CNTL);
6655 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))
6656 data &= ~DISABLE_CP_PG;
6658 data |= DISABLE_CP_PG;
6660 WREG32(RLC_PG_CNTL, data);
6663 static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable)
6667 orig = data = RREG32(RLC_PG_CNTL);
6668 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))
6669 data &= ~DISABLE_GDS_PG;
6671 data |= DISABLE_GDS_PG;
6673 WREG32(RLC_PG_CNTL, data);
6676 #define CP_ME_TABLE_SIZE 96
6677 #define CP_ME_TABLE_OFFSET 2048
6678 #define CP_MEC_TABLE_OFFSET 4096
6680 void cik_init_cp_pg_table(struct radeon_device *rdev)
6682 volatile u32 *dst_ptr;
6683 int me, i, max_me = 4;
6685 u32 table_offset, table_size;
6687 if (rdev->family == CHIP_KAVERI)
6690 if (rdev->rlc.cp_table_ptr == NULL)
6693 /* write the cp table buffer */
6694 dst_ptr = rdev->rlc.cp_table_ptr;
6695 for (me = 0; me < max_me; me++) {
6697 const __le32 *fw_data;
6698 const struct gfx_firmware_header_v1_0 *hdr;
6701 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
6702 fw_data = (const __le32 *)
6703 (rdev->ce_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6704 table_offset = le32_to_cpu(hdr->jt_offset);
6705 table_size = le32_to_cpu(hdr->jt_size);
6706 } else if (me == 1) {
6707 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
6708 fw_data = (const __le32 *)
6709 (rdev->pfp_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6710 table_offset = le32_to_cpu(hdr->jt_offset);
6711 table_size = le32_to_cpu(hdr->jt_size);
6712 } else if (me == 2) {
6713 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
6714 fw_data = (const __le32 *)
6715 (rdev->me_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6716 table_offset = le32_to_cpu(hdr->jt_offset);
6717 table_size = le32_to_cpu(hdr->jt_size);
6718 } else if (me == 3) {
6719 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;
6720 fw_data = (const __le32 *)
6721 (rdev->mec_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6722 table_offset = le32_to_cpu(hdr->jt_offset);
6723 table_size = le32_to_cpu(hdr->jt_size);
6725 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;
6726 fw_data = (const __le32 *)
6727 (rdev->mec2_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6728 table_offset = le32_to_cpu(hdr->jt_offset);
6729 table_size = le32_to_cpu(hdr->jt_size);
6732 for (i = 0; i < table_size; i ++) {
6733 dst_ptr[bo_offset + i] =
6734 cpu_to_le32(le32_to_cpu(fw_data[table_offset + i]));
6736 bo_offset += table_size;
6738 const __be32 *fw_data;
6739 table_size = CP_ME_TABLE_SIZE;
6742 fw_data = (const __be32 *)rdev->ce_fw->data;
6743 table_offset = CP_ME_TABLE_OFFSET;
6744 } else if (me == 1) {
6745 fw_data = (const __be32 *)rdev->pfp_fw->data;
6746 table_offset = CP_ME_TABLE_OFFSET;
6747 } else if (me == 2) {
6748 fw_data = (const __be32 *)rdev->me_fw->data;
6749 table_offset = CP_ME_TABLE_OFFSET;
6751 fw_data = (const __be32 *)rdev->mec_fw->data;
6752 table_offset = CP_MEC_TABLE_OFFSET;
6755 for (i = 0; i < table_size; i ++) {
6756 dst_ptr[bo_offset + i] =
6757 cpu_to_le32(be32_to_cpu(fw_data[table_offset + i]));
6759 bo_offset += table_size;
6764 static void cik_enable_gfx_cgpg(struct radeon_device *rdev,
6769 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
6770 orig = data = RREG32(RLC_PG_CNTL);
6771 data |= GFX_PG_ENABLE;
6773 WREG32(RLC_PG_CNTL, data);
6775 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6778 WREG32(RLC_AUTO_PG_CTRL, data);
6780 orig = data = RREG32(RLC_PG_CNTL);
6781 data &= ~GFX_PG_ENABLE;
6783 WREG32(RLC_PG_CNTL, data);
6785 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6786 data &= ~AUTO_PG_EN;
6788 WREG32(RLC_AUTO_PG_CTRL, data);
6790 data = RREG32(DB_RENDER_CONTROL);
6794 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
6796 u32 mask = 0, tmp, tmp1;
6799 cik_select_se_sh(rdev, se, sh);
6800 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
6801 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
6802 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6809 for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {
6814 return (~tmp) & mask;
6817 static void cik_init_ao_cu_mask(struct radeon_device *rdev)
6819 u32 i, j, k, active_cu_number = 0;
6820 u32 mask, counter, cu_bitmap;
6823 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6824 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6828 for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {
6829 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) {
6837 active_cu_number += counter;
6838 tmp |= (cu_bitmap << (i * 16 + j * 8));
6842 WREG32(RLC_PG_AO_CU_MASK, tmp);
6844 tmp = RREG32(RLC_MAX_PG_CU);
6845 tmp &= ~MAX_PU_CU_MASK;
6846 tmp |= MAX_PU_CU(active_cu_number);
6847 WREG32(RLC_MAX_PG_CU, tmp);
6850 static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev,
6855 orig = data = RREG32(RLC_PG_CNTL);
6856 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))
6857 data |= STATIC_PER_CU_PG_ENABLE;
6859 data &= ~STATIC_PER_CU_PG_ENABLE;
6861 WREG32(RLC_PG_CNTL, data);
6864 static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev,
6869 orig = data = RREG32(RLC_PG_CNTL);
6870 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))
6871 data |= DYN_PER_CU_PG_ENABLE;
6873 data &= ~DYN_PER_CU_PG_ENABLE;
6875 WREG32(RLC_PG_CNTL, data);
6878 #define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
6879 #define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET 0x3D
6881 static void cik_init_gfx_cgpg(struct radeon_device *rdev)
6886 if (rdev->rlc.cs_data) {
6887 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6888 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));
6889 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));
6890 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);
6892 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6893 for (i = 0; i < 3; i++)
6894 WREG32(RLC_GPM_SCRATCH_DATA, 0);
6896 if (rdev->rlc.reg_list) {
6897 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET);
6898 for (i = 0; i < rdev->rlc.reg_list_size; i++)
6899 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);
6902 orig = data = RREG32(RLC_PG_CNTL);
6905 WREG32(RLC_PG_CNTL, data);
6907 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
6908 WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);
6910 data = RREG32(CP_RB_WPTR_POLL_CNTL);
6911 data &= ~IDLE_POLL_COUNT_MASK;
6912 data |= IDLE_POLL_COUNT(0x60);
6913 WREG32(CP_RB_WPTR_POLL_CNTL, data);
6916 WREG32(RLC_PG_DELAY, data);
6918 data = RREG32(RLC_PG_DELAY_2);
6921 WREG32(RLC_PG_DELAY_2, data);
6923 data = RREG32(RLC_AUTO_PG_CTRL);
6924 data &= ~GRBM_REG_SGIT_MASK;
6925 data |= GRBM_REG_SGIT(0x700);
6926 WREG32(RLC_AUTO_PG_CTRL, data);
6930 static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable)
6932 cik_enable_gfx_cgpg(rdev, enable);
6933 cik_enable_gfx_static_mgpg(rdev, enable);
6934 cik_enable_gfx_dynamic_mgpg(rdev, enable);
6937 u32 cik_get_csb_size(struct radeon_device *rdev)
6940 const struct cs_section_def *sect = NULL;
6941 const struct cs_extent_def *ext = NULL;
6943 if (rdev->rlc.cs_data == NULL)
6946 /* begin clear state */
6948 /* context control state */
6951 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6952 for (ext = sect->section; ext->extent != NULL; ++ext) {
6953 if (sect->id == SECT_CONTEXT)
6954 count += 2 + ext->reg_count;
6959 /* pa_sc_raster_config/pa_sc_raster_config1 */
6961 /* end clear state */
6969 void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
6972 const struct cs_section_def *sect = NULL;
6973 const struct cs_extent_def *ext = NULL;
6975 if (rdev->rlc.cs_data == NULL)
6980 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6981 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
6983 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
6984 buffer[count++] = cpu_to_le32(0x80000000);
6985 buffer[count++] = cpu_to_le32(0x80000000);
6987 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6988 for (ext = sect->section; ext->extent != NULL; ++ext) {
6989 if (sect->id == SECT_CONTEXT) {
6991 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
6992 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
6993 for (i = 0; i < ext->reg_count; i++)
6994 buffer[count++] = cpu_to_le32(ext->extent[i]);
7001 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
7002 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
7003 switch (rdev->family) {
7005 buffer[count++] = cpu_to_le32(0x16000012);
7006 buffer[count++] = cpu_to_le32(0x00000000);
7009 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
7010 buffer[count++] = cpu_to_le32(0x00000000);
7014 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
7015 buffer[count++] = cpu_to_le32(0x00000000);
7018 buffer[count++] = cpu_to_le32(0x3a00161a);
7019 buffer[count++] = cpu_to_le32(0x0000002e);
7022 buffer[count++] = cpu_to_le32(0x00000000);
7023 buffer[count++] = cpu_to_le32(0x00000000);
7027 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
7028 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
7030 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
7031 buffer[count++] = cpu_to_le32(0);
7034 static void cik_init_pg(struct radeon_device *rdev)
7036 if (rdev->pg_flags) {
7037 cik_enable_sck_slowdown_on_pu(rdev, true);
7038 cik_enable_sck_slowdown_on_pd(rdev, true);
7039 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
7040 cik_init_gfx_cgpg(rdev);
7041 cik_enable_cp_pg(rdev, true);
7042 cik_enable_gds_pg(rdev, true);
7044 cik_init_ao_cu_mask(rdev);
7045 cik_update_gfx_pg(rdev, true);
7049 static void cik_fini_pg(struct radeon_device *rdev)
7051 if (rdev->pg_flags) {
7052 cik_update_gfx_pg(rdev, false);
7053 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
7054 cik_enable_cp_pg(rdev, false);
7055 cik_enable_gds_pg(rdev, false);
7062 * Starting with r6xx, interrupts are handled via a ring buffer.
7063 * Ring buffers are areas of GPU accessible memory that the GPU
7064 * writes interrupt vectors into and the host reads vectors out of.
7065 * There is a rptr (read pointer) that determines where the
7066 * host is currently reading, and a wptr (write pointer)
7067 * which determines where the GPU has written. When the
7068 * pointers are equal, the ring is idle. When the GPU
7069 * writes vectors to the ring buffer, it increments the
7070 * wptr. When there is an interrupt, the host then starts
7071 * fetching commands and processing them until the pointers are
7072 * equal again at which point it updates the rptr.
7076 * cik_enable_interrupts - Enable the interrupt ring buffer
7078 * @rdev: radeon_device pointer
7080 * Enable the interrupt ring buffer (CIK).
7082 static void cik_enable_interrupts(struct radeon_device *rdev)
7084 u32 ih_cntl = RREG32(IH_CNTL);
7085 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
7087 ih_cntl |= ENABLE_INTR;
7088 ih_rb_cntl |= IH_RB_ENABLE;
7089 WREG32(IH_CNTL, ih_cntl);
7090 WREG32(IH_RB_CNTL, ih_rb_cntl);
7091 rdev->ih.enabled = true;
7095 * cik_disable_interrupts - Disable the interrupt ring buffer
7097 * @rdev: radeon_device pointer
7099 * Disable the interrupt ring buffer (CIK).
7101 static void cik_disable_interrupts(struct radeon_device *rdev)
7103 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
7104 u32 ih_cntl = RREG32(IH_CNTL);
7106 ih_rb_cntl &= ~IH_RB_ENABLE;
7107 ih_cntl &= ~ENABLE_INTR;
7108 WREG32(IH_RB_CNTL, ih_rb_cntl);
7109 WREG32(IH_CNTL, ih_cntl);
7110 /* set rptr, wptr to 0 */
7111 WREG32(IH_RB_RPTR, 0);
7112 WREG32(IH_RB_WPTR, 0);
7113 rdev->ih.enabled = false;
7118 * cik_disable_interrupt_state - Disable all interrupt sources
7120 * @rdev: radeon_device pointer
7122 * Clear all interrupt enable bits used by the driver (CIK).
7124 static void cik_disable_interrupt_state(struct radeon_device *rdev)
7129 tmp = RREG32(CP_INT_CNTL_RING0) &
7130 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
7131 WREG32(CP_INT_CNTL_RING0, tmp);
7133 tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
7134 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
7135 tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
7136 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
7137 /* compute queues */
7138 WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
7139 WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
7140 WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
7141 WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
7142 WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
7143 WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
7144 WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
7145 WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
7147 WREG32(GRBM_INT_CNTL, 0);
7148 /* vline/vblank, etc. */
7149 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
7150 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
7151 if (rdev->num_crtc >= 4) {
7152 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
7153 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
7155 if (rdev->num_crtc >= 6) {
7156 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
7157 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
7160 if (rdev->num_crtc >= 2) {
7161 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
7162 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
7164 if (rdev->num_crtc >= 4) {
7165 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
7166 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
7168 if (rdev->num_crtc >= 6) {
7169 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
7170 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
7174 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
7176 /* digital hotplug */
7177 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7178 WREG32(DC_HPD1_INT_CONTROL, tmp);
7179 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7180 WREG32(DC_HPD2_INT_CONTROL, tmp);
7181 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7182 WREG32(DC_HPD3_INT_CONTROL, tmp);
7183 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7184 WREG32(DC_HPD4_INT_CONTROL, tmp);
7185 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7186 WREG32(DC_HPD5_INT_CONTROL, tmp);
7187 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7188 WREG32(DC_HPD6_INT_CONTROL, tmp);
7193 * cik_irq_init - init and enable the interrupt ring
7195 * @rdev: radeon_device pointer
7197 * Allocate a ring buffer for the interrupt controller,
7198 * enable the RLC, disable interrupts, enable the IH
7199 * ring buffer and enable it (CIK).
7200 * Called at device load and reume.
7201 * Returns 0 for success, errors for failure.
7203 static int cik_irq_init(struct radeon_device *rdev)
7207 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
7210 ret = r600_ih_ring_alloc(rdev);
7215 cik_disable_interrupts(rdev);
7218 ret = cik_rlc_resume(rdev);
7220 r600_ih_ring_fini(rdev);
7224 /* setup interrupt control */
7225 /* XXX this should actually be a bus address, not an MC address. same on older asics */
7226 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
7227 interrupt_cntl = RREG32(INTERRUPT_CNTL);
7228 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
7229 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
7231 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
7232 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
7233 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
7234 WREG32(INTERRUPT_CNTL, interrupt_cntl);
7236 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
7237 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
7239 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
7240 IH_WPTR_OVERFLOW_CLEAR |
7243 if (rdev->wb.enabled)
7244 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
7246 /* set the writeback address whether it's enabled or not */
7247 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
7248 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
7250 WREG32(IH_RB_CNTL, ih_rb_cntl);
7252 /* set rptr, wptr to 0 */
7253 WREG32(IH_RB_RPTR, 0);
7254 WREG32(IH_RB_WPTR, 0);
7256 /* Default settings for IH_CNTL (disabled at first) */
7257 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
7258 /* RPTR_REARM only works if msi's are enabled */
7259 if (rdev->msi_enabled)
7260 ih_cntl |= RPTR_REARM;
7261 WREG32(IH_CNTL, ih_cntl);
7263 /* force the active interrupt state to all disabled */
7264 cik_disable_interrupt_state(rdev);
7266 pci_set_master(rdev->pdev);
7269 cik_enable_interrupts(rdev);
7275 * cik_irq_set - enable/disable interrupt sources
7277 * @rdev: radeon_device pointer
7279 * Enable interrupt sources on the GPU (vblanks, hpd,
7281 * Returns 0 for success, errors for failure.
7283 int cik_irq_set(struct radeon_device *rdev)
7286 u32 cp_m1p0, cp_m1p1, cp_m1p2, cp_m1p3;
7287 u32 cp_m2p0, cp_m2p1, cp_m2p2, cp_m2p3;
7288 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
7289 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
7290 u32 grbm_int_cntl = 0;
7291 u32 dma_cntl, dma_cntl1;
7294 if (!rdev->irq.installed) {
7295 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
7298 /* don't enable anything if the ih is disabled */
7299 if (!rdev->ih.enabled) {
7300 cik_disable_interrupts(rdev);
7301 /* force the active interrupt state to all disabled */
7302 cik_disable_interrupt_state(rdev);
7306 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
7307 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
7308 cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
7310 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
7311 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
7312 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
7313 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
7314 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
7315 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
7317 dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
7318 dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
7320 cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7321 cp_m1p1 = RREG32(CP_ME1_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7322 cp_m1p2 = RREG32(CP_ME1_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7323 cp_m1p3 = RREG32(CP_ME1_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7324 cp_m2p0 = RREG32(CP_ME2_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7325 cp_m2p1 = RREG32(CP_ME2_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7326 cp_m2p2 = RREG32(CP_ME2_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7327 cp_m2p3 = RREG32(CP_ME2_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7329 if (rdev->flags & RADEON_IS_IGP)
7330 thermal_int = RREG32_SMC(CG_THERMAL_INT_CTRL) &
7331 ~(THERM_INTH_MASK | THERM_INTL_MASK);
7333 thermal_int = RREG32_SMC(CG_THERMAL_INT) &
7334 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
7336 /* enable CP interrupts on all rings */
7337 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
7338 DRM_DEBUG("cik_irq_set: sw int gfx\n");
7339 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
7341 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
7342 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7343 DRM_DEBUG("si_irq_set: sw int cp1\n");
7344 if (ring->me == 1) {
7345 switch (ring->pipe) {
7347 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7350 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
7353 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7356 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7359 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
7362 } else if (ring->me == 2) {
7363 switch (ring->pipe) {
7365 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
7368 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
7371 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7374 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7377 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
7381 DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
7384 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
7385 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7386 DRM_DEBUG("si_irq_set: sw int cp2\n");
7387 if (ring->me == 1) {
7388 switch (ring->pipe) {
7390 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7393 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
7396 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7399 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7402 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
7405 } else if (ring->me == 2) {
7406 switch (ring->pipe) {
7408 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
7411 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
7414 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7417 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7420 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
7424 DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
7428 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
7429 DRM_DEBUG("cik_irq_set: sw int dma\n");
7430 dma_cntl |= TRAP_ENABLE;
7433 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
7434 DRM_DEBUG("cik_irq_set: sw int dma1\n");
7435 dma_cntl1 |= TRAP_ENABLE;
7438 if (rdev->irq.crtc_vblank_int[0] ||
7439 atomic_read(&rdev->irq.pflip[0])) {
7440 DRM_DEBUG("cik_irq_set: vblank 0\n");
7441 crtc1 |= VBLANK_INTERRUPT_MASK;
7443 if (rdev->irq.crtc_vblank_int[1] ||
7444 atomic_read(&rdev->irq.pflip[1])) {
7445 DRM_DEBUG("cik_irq_set: vblank 1\n");
7446 crtc2 |= VBLANK_INTERRUPT_MASK;
7448 if (rdev->irq.crtc_vblank_int[2] ||
7449 atomic_read(&rdev->irq.pflip[2])) {
7450 DRM_DEBUG("cik_irq_set: vblank 2\n");
7451 crtc3 |= VBLANK_INTERRUPT_MASK;
7453 if (rdev->irq.crtc_vblank_int[3] ||
7454 atomic_read(&rdev->irq.pflip[3])) {
7455 DRM_DEBUG("cik_irq_set: vblank 3\n");
7456 crtc4 |= VBLANK_INTERRUPT_MASK;
7458 if (rdev->irq.crtc_vblank_int[4] ||
7459 atomic_read(&rdev->irq.pflip[4])) {
7460 DRM_DEBUG("cik_irq_set: vblank 4\n");
7461 crtc5 |= VBLANK_INTERRUPT_MASK;
7463 if (rdev->irq.crtc_vblank_int[5] ||
7464 atomic_read(&rdev->irq.pflip[5])) {
7465 DRM_DEBUG("cik_irq_set: vblank 5\n");
7466 crtc6 |= VBLANK_INTERRUPT_MASK;
7468 if (rdev->irq.hpd[0]) {
7469 DRM_DEBUG("cik_irq_set: hpd 1\n");
7470 hpd1 |= DC_HPDx_INT_EN;
7472 if (rdev->irq.hpd[1]) {
7473 DRM_DEBUG("cik_irq_set: hpd 2\n");
7474 hpd2 |= DC_HPDx_INT_EN;
7476 if (rdev->irq.hpd[2]) {
7477 DRM_DEBUG("cik_irq_set: hpd 3\n");
7478 hpd3 |= DC_HPDx_INT_EN;
7480 if (rdev->irq.hpd[3]) {
7481 DRM_DEBUG("cik_irq_set: hpd 4\n");
7482 hpd4 |= DC_HPDx_INT_EN;
7484 if (rdev->irq.hpd[4]) {
7485 DRM_DEBUG("cik_irq_set: hpd 5\n");
7486 hpd5 |= DC_HPDx_INT_EN;
7488 if (rdev->irq.hpd[5]) {
7489 DRM_DEBUG("cik_irq_set: hpd 6\n");
7490 hpd6 |= DC_HPDx_INT_EN;
7493 if (rdev->irq.dpm_thermal) {
7494 DRM_DEBUG("dpm thermal\n");
7495 if (rdev->flags & RADEON_IS_IGP)
7496 thermal_int |= THERM_INTH_MASK | THERM_INTL_MASK;
7498 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
7501 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
7503 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
7504 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
7506 WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
7507 WREG32(CP_ME1_PIPE1_INT_CNTL, cp_m1p1);
7508 WREG32(CP_ME1_PIPE2_INT_CNTL, cp_m1p2);
7509 WREG32(CP_ME1_PIPE3_INT_CNTL, cp_m1p3);
7510 WREG32(CP_ME2_PIPE0_INT_CNTL, cp_m2p0);
7511 WREG32(CP_ME2_PIPE1_INT_CNTL, cp_m2p1);
7512 WREG32(CP_ME2_PIPE2_INT_CNTL, cp_m2p2);
7513 WREG32(CP_ME2_PIPE3_INT_CNTL, cp_m2p3);
7515 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
7517 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
7518 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
7519 if (rdev->num_crtc >= 4) {
7520 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
7521 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
7523 if (rdev->num_crtc >= 6) {
7524 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
7525 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
7528 if (rdev->num_crtc >= 2) {
7529 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
7530 GRPH_PFLIP_INT_MASK);
7531 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
7532 GRPH_PFLIP_INT_MASK);
7534 if (rdev->num_crtc >= 4) {
7535 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
7536 GRPH_PFLIP_INT_MASK);
7537 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
7538 GRPH_PFLIP_INT_MASK);
7540 if (rdev->num_crtc >= 6) {
7541 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
7542 GRPH_PFLIP_INT_MASK);
7543 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
7544 GRPH_PFLIP_INT_MASK);
7547 WREG32(DC_HPD1_INT_CONTROL, hpd1);
7548 WREG32(DC_HPD2_INT_CONTROL, hpd2);
7549 WREG32(DC_HPD3_INT_CONTROL, hpd3);
7550 WREG32(DC_HPD4_INT_CONTROL, hpd4);
7551 WREG32(DC_HPD5_INT_CONTROL, hpd5);
7552 WREG32(DC_HPD6_INT_CONTROL, hpd6);
7554 if (rdev->flags & RADEON_IS_IGP)
7555 WREG32_SMC(CG_THERMAL_INT_CTRL, thermal_int);
7557 WREG32_SMC(CG_THERMAL_INT, thermal_int);
7563 * cik_irq_ack - ack interrupt sources
7565 * @rdev: radeon_device pointer
7567 * Ack interrupt sources on the GPU (vblanks, hpd,
7568 * etc.) (CIK). Certain interrupts sources are sw
7569 * generated and do not require an explicit ack.
7571 static inline void cik_irq_ack(struct radeon_device *rdev)
7575 rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
7576 rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
7577 rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
7578 rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
7579 rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
7580 rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
7581 rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
7583 rdev->irq.stat_regs.cik.d1grph_int = RREG32(GRPH_INT_STATUS +
7584 EVERGREEN_CRTC0_REGISTER_OFFSET);
7585 rdev->irq.stat_regs.cik.d2grph_int = RREG32(GRPH_INT_STATUS +
7586 EVERGREEN_CRTC1_REGISTER_OFFSET);
7587 if (rdev->num_crtc >= 4) {
7588 rdev->irq.stat_regs.cik.d3grph_int = RREG32(GRPH_INT_STATUS +
7589 EVERGREEN_CRTC2_REGISTER_OFFSET);
7590 rdev->irq.stat_regs.cik.d4grph_int = RREG32(GRPH_INT_STATUS +
7591 EVERGREEN_CRTC3_REGISTER_OFFSET);
7593 if (rdev->num_crtc >= 6) {
7594 rdev->irq.stat_regs.cik.d5grph_int = RREG32(GRPH_INT_STATUS +
7595 EVERGREEN_CRTC4_REGISTER_OFFSET);
7596 rdev->irq.stat_regs.cik.d6grph_int = RREG32(GRPH_INT_STATUS +
7597 EVERGREEN_CRTC5_REGISTER_OFFSET);
7600 if (rdev->irq.stat_regs.cik.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
7601 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET,
7602 GRPH_PFLIP_INT_CLEAR);
7603 if (rdev->irq.stat_regs.cik.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
7604 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET,
7605 GRPH_PFLIP_INT_CLEAR);
7606 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
7607 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
7608 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
7609 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
7610 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
7611 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
7612 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
7613 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
7615 if (rdev->num_crtc >= 4) {
7616 if (rdev->irq.stat_regs.cik.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
7617 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET,
7618 GRPH_PFLIP_INT_CLEAR);
7619 if (rdev->irq.stat_regs.cik.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
7620 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET,
7621 GRPH_PFLIP_INT_CLEAR);
7622 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
7623 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
7624 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
7625 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
7626 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
7627 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
7628 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
7629 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
7632 if (rdev->num_crtc >= 6) {
7633 if (rdev->irq.stat_regs.cik.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
7634 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET,
7635 GRPH_PFLIP_INT_CLEAR);
7636 if (rdev->irq.stat_regs.cik.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
7637 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET,
7638 GRPH_PFLIP_INT_CLEAR);
7639 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
7640 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
7641 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
7642 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
7643 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
7644 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
7645 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
7646 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
7649 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7650 tmp = RREG32(DC_HPD1_INT_CONTROL);
7651 tmp |= DC_HPDx_INT_ACK;
7652 WREG32(DC_HPD1_INT_CONTROL, tmp);
7654 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7655 tmp = RREG32(DC_HPD2_INT_CONTROL);
7656 tmp |= DC_HPDx_INT_ACK;
7657 WREG32(DC_HPD2_INT_CONTROL, tmp);
7659 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7660 tmp = RREG32(DC_HPD3_INT_CONTROL);
7661 tmp |= DC_HPDx_INT_ACK;
7662 WREG32(DC_HPD3_INT_CONTROL, tmp);
7664 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7665 tmp = RREG32(DC_HPD4_INT_CONTROL);
7666 tmp |= DC_HPDx_INT_ACK;
7667 WREG32(DC_HPD4_INT_CONTROL, tmp);
7669 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7670 tmp = RREG32(DC_HPD5_INT_CONTROL);
7671 tmp |= DC_HPDx_INT_ACK;
7672 WREG32(DC_HPD5_INT_CONTROL, tmp);
7674 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7675 tmp = RREG32(DC_HPD5_INT_CONTROL);
7676 tmp |= DC_HPDx_INT_ACK;
7677 WREG32(DC_HPD6_INT_CONTROL, tmp);
7682 * cik_irq_disable - disable interrupts
7684 * @rdev: radeon_device pointer
7686 * Disable interrupts on the hw (CIK).
7688 static void cik_irq_disable(struct radeon_device *rdev)
7690 cik_disable_interrupts(rdev);
7691 /* Wait and acknowledge irq */
7694 cik_disable_interrupt_state(rdev);
7698 * cik_irq_disable - disable interrupts for suspend
7700 * @rdev: radeon_device pointer
7702 * Disable interrupts and stop the RLC (CIK).
7705 static void cik_irq_suspend(struct radeon_device *rdev)
7707 cik_irq_disable(rdev);
7712 * cik_irq_fini - tear down interrupt support
7714 * @rdev: radeon_device pointer
7716 * Disable interrupts on the hw and free the IH ring
7718 * Used for driver unload.
7720 static void cik_irq_fini(struct radeon_device *rdev)
7722 cik_irq_suspend(rdev);
7723 r600_ih_ring_fini(rdev);
7727 * cik_get_ih_wptr - get the IH ring buffer wptr
7729 * @rdev: radeon_device pointer
7731 * Get the IH ring buffer wptr from either the register
7732 * or the writeback memory buffer (CIK). Also check for
7733 * ring buffer overflow and deal with it.
7734 * Used by cik_irq_process().
7735 * Returns the value of the wptr.
7737 static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
7741 if (rdev->wb.enabled)
7742 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
7744 wptr = RREG32(IH_RB_WPTR);
7746 if (wptr & RB_OVERFLOW) {
7747 /* When a ring buffer overflow happen start parsing interrupt
7748 * from the last not overwritten vector (wptr + 16). Hopefully
7749 * this should allow us to catchup.
7751 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
7752 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
7753 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
7754 tmp = RREG32(IH_RB_CNTL);
7755 tmp |= IH_WPTR_OVERFLOW_CLEAR;
7756 WREG32(IH_RB_CNTL, tmp);
7757 wptr &= ~RB_OVERFLOW;
7759 return (wptr & rdev->ih.ptr_mask);
7763 * Each IV ring entry is 128 bits:
7764 * [7:0] - interrupt source id
7766 * [59:32] - interrupt source data
7767 * [63:60] - reserved
7770 * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
7771 * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
7772 * - for gfx, hw shader state (0=PS...5=LS, 6=CS)
7773 * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
7774 * PIPE_ID - ME0 0=3D
7775 * - ME1&2 compute dispatcher (4 pipes each)
7777 * INSTANCE_ID [1:0], QUEUE_ID[1:0]
7778 * INSTANCE_ID - 0 = sdma0, 1 = sdma1
7779 * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
7782 * [127:96] - reserved
7785 * cik_irq_process - interrupt handler
7787 * @rdev: radeon_device pointer
7789 * Interrupt hander (CIK). Walk the IH ring,
7790 * ack interrupts and schedule work to handle
7792 * Returns irq process return code.
7794 int cik_irq_process(struct radeon_device *rdev)
7796 struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7797 struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7800 u32 src_id, src_data, ring_id;
7801 u8 me_id, pipe_id, queue_id;
7803 bool queue_hotplug = false;
7804 bool queue_reset = false;
7805 u32 addr, status, mc_client;
7806 bool queue_thermal = false;
7808 if (!rdev->ih.enabled || rdev->shutdown)
7811 wptr = cik_get_ih_wptr(rdev);
7814 /* is somebody else already processing irqs? */
7815 if (atomic_xchg(&rdev->ih.lock, 1))
7818 rptr = rdev->ih.rptr;
7819 DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
7821 /* Order reading of wptr vs. reading of IH ring data */
7824 /* display interrupts */
7827 while (rptr != wptr) {
7828 /* wptr/rptr are in bytes! */
7829 ring_index = rptr / 4;
7830 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
7831 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
7832 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
7835 case 1: /* D1 vblank/vline */
7837 case 0: /* D1 vblank */
7838 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) {
7839 if (rdev->irq.crtc_vblank_int[0]) {
7840 drm_handle_vblank(rdev->ddev, 0);
7841 rdev->pm.vblank_sync = true;
7842 wake_up(&rdev->irq.vblank_queue);
7844 if (atomic_read(&rdev->irq.pflip[0]))
7845 radeon_crtc_handle_vblank(rdev, 0);
7846 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
7847 DRM_DEBUG("IH: D1 vblank\n");
7850 case 1: /* D1 vline */
7851 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) {
7852 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
7853 DRM_DEBUG("IH: D1 vline\n");
7857 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7861 case 2: /* D2 vblank/vline */
7863 case 0: /* D2 vblank */
7864 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
7865 if (rdev->irq.crtc_vblank_int[1]) {
7866 drm_handle_vblank(rdev->ddev, 1);
7867 rdev->pm.vblank_sync = true;
7868 wake_up(&rdev->irq.vblank_queue);
7870 if (atomic_read(&rdev->irq.pflip[1]))
7871 radeon_crtc_handle_vblank(rdev, 1);
7872 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
7873 DRM_DEBUG("IH: D2 vblank\n");
7876 case 1: /* D2 vline */
7877 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
7878 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
7879 DRM_DEBUG("IH: D2 vline\n");
7883 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7887 case 3: /* D3 vblank/vline */
7889 case 0: /* D3 vblank */
7890 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
7891 if (rdev->irq.crtc_vblank_int[2]) {
7892 drm_handle_vblank(rdev->ddev, 2);
7893 rdev->pm.vblank_sync = true;
7894 wake_up(&rdev->irq.vblank_queue);
7896 if (atomic_read(&rdev->irq.pflip[2]))
7897 radeon_crtc_handle_vblank(rdev, 2);
7898 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
7899 DRM_DEBUG("IH: D3 vblank\n");
7902 case 1: /* D3 vline */
7903 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
7904 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
7905 DRM_DEBUG("IH: D3 vline\n");
7909 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7913 case 4: /* D4 vblank/vline */
7915 case 0: /* D4 vblank */
7916 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
7917 if (rdev->irq.crtc_vblank_int[3]) {
7918 drm_handle_vblank(rdev->ddev, 3);
7919 rdev->pm.vblank_sync = true;
7920 wake_up(&rdev->irq.vblank_queue);
7922 if (atomic_read(&rdev->irq.pflip[3]))
7923 radeon_crtc_handle_vblank(rdev, 3);
7924 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
7925 DRM_DEBUG("IH: D4 vblank\n");
7928 case 1: /* D4 vline */
7929 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
7930 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
7931 DRM_DEBUG("IH: D4 vline\n");
7935 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7939 case 5: /* D5 vblank/vline */
7941 case 0: /* D5 vblank */
7942 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
7943 if (rdev->irq.crtc_vblank_int[4]) {
7944 drm_handle_vblank(rdev->ddev, 4);
7945 rdev->pm.vblank_sync = true;
7946 wake_up(&rdev->irq.vblank_queue);
7948 if (atomic_read(&rdev->irq.pflip[4]))
7949 radeon_crtc_handle_vblank(rdev, 4);
7950 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
7951 DRM_DEBUG("IH: D5 vblank\n");
7954 case 1: /* D5 vline */
7955 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
7956 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
7957 DRM_DEBUG("IH: D5 vline\n");
7961 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7965 case 6: /* D6 vblank/vline */
7967 case 0: /* D6 vblank */
7968 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
7969 if (rdev->irq.crtc_vblank_int[5]) {
7970 drm_handle_vblank(rdev->ddev, 5);
7971 rdev->pm.vblank_sync = true;
7972 wake_up(&rdev->irq.vblank_queue);
7974 if (atomic_read(&rdev->irq.pflip[5]))
7975 radeon_crtc_handle_vblank(rdev, 5);
7976 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
7977 DRM_DEBUG("IH: D6 vblank\n");
7980 case 1: /* D6 vline */
7981 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
7982 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
7983 DRM_DEBUG("IH: D6 vline\n");
7987 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7991 case 8: /* D1 page flip */
7992 case 10: /* D2 page flip */
7993 case 12: /* D3 page flip */
7994 case 14: /* D4 page flip */
7995 case 16: /* D5 page flip */
7996 case 18: /* D6 page flip */
7997 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
7998 if (radeon_use_pflipirq > 0)
7999 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
8001 case 42: /* HPD hotplug */
8004 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
8005 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
8006 queue_hotplug = true;
8007 DRM_DEBUG("IH: HPD1\n");
8011 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
8012 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
8013 queue_hotplug = true;
8014 DRM_DEBUG("IH: HPD2\n");
8018 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
8019 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
8020 queue_hotplug = true;
8021 DRM_DEBUG("IH: HPD3\n");
8025 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
8026 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
8027 queue_hotplug = true;
8028 DRM_DEBUG("IH: HPD4\n");
8032 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
8033 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
8034 queue_hotplug = true;
8035 DRM_DEBUG("IH: HPD5\n");
8039 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
8040 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
8041 queue_hotplug = true;
8042 DRM_DEBUG("IH: HPD6\n");
8046 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
8051 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
8052 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
8056 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
8057 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
8058 mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
8059 /* reset addr and status */
8060 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
8061 if (addr == 0x0 && status == 0x0)
8063 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
8064 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
8066 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
8068 cik_vm_decode_fault(rdev, status, addr, mc_client);
8071 DRM_DEBUG("IH: VCE int: 0x%08x\n", src_data);
8074 radeon_fence_process(rdev, TN_RING_TYPE_VCE1_INDEX);
8077 radeon_fence_process(rdev, TN_RING_TYPE_VCE2_INDEX);
8080 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
8084 case 176: /* GFX RB CP_INT */
8085 case 177: /* GFX IB CP_INT */
8086 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
8088 case 181: /* CP EOP event */
8089 DRM_DEBUG("IH: CP EOP\n");
8090 /* XXX check the bitfield order! */
8091 me_id = (ring_id & 0x60) >> 5;
8092 pipe_id = (ring_id & 0x18) >> 3;
8093 queue_id = (ring_id & 0x7) >> 0;
8096 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
8100 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
8101 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
8102 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
8103 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
8107 case 184: /* CP Privileged reg access */
8108 DRM_ERROR("Illegal register access in command stream\n");
8109 /* XXX check the bitfield order! */
8110 me_id = (ring_id & 0x60) >> 5;
8111 pipe_id = (ring_id & 0x18) >> 3;
8112 queue_id = (ring_id & 0x7) >> 0;
8115 /* This results in a full GPU reset, but all we need to do is soft
8116 * reset the CP for gfx
8130 case 185: /* CP Privileged inst */
8131 DRM_ERROR("Illegal instruction in command stream\n");
8132 /* XXX check the bitfield order! */
8133 me_id = (ring_id & 0x60) >> 5;
8134 pipe_id = (ring_id & 0x18) >> 3;
8135 queue_id = (ring_id & 0x7) >> 0;
8138 /* This results in a full GPU reset, but all we need to do is soft
8139 * reset the CP for gfx
8153 case 224: /* SDMA trap event */
8154 /* XXX check the bitfield order! */
8155 me_id = (ring_id & 0x3) >> 0;
8156 queue_id = (ring_id & 0xc) >> 2;
8157 DRM_DEBUG("IH: SDMA trap\n");
8162 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
8175 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
8187 case 230: /* thermal low to high */
8188 DRM_DEBUG("IH: thermal low to high\n");
8189 rdev->pm.dpm.thermal.high_to_low = false;
8190 queue_thermal = true;
8192 case 231: /* thermal high to low */
8193 DRM_DEBUG("IH: thermal high to low\n");
8194 rdev->pm.dpm.thermal.high_to_low = true;
8195 queue_thermal = true;
8197 case 233: /* GUI IDLE */
8198 DRM_DEBUG("IH: GUI idle\n");
8200 case 241: /* SDMA Privileged inst */
8201 case 247: /* SDMA Privileged inst */
8202 DRM_ERROR("Illegal instruction in SDMA command stream\n");
8203 /* XXX check the bitfield order! */
8204 me_id = (ring_id & 0x3) >> 0;
8205 queue_id = (ring_id & 0xc) >> 2;
8240 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
8244 /* wptr/rptr are in bytes! */
8246 rptr &= rdev->ih.ptr_mask;
8249 schedule_work(&rdev->hotplug_work);
8251 rdev->needs_reset = true;
8252 wake_up_all(&rdev->fence_queue);
8255 schedule_work(&rdev->pm.dpm.thermal.work);
8256 rdev->ih.rptr = rptr;
8257 WREG32(IH_RB_RPTR, rdev->ih.rptr);
8258 atomic_set(&rdev->ih.lock, 0);
8260 /* make sure wptr hasn't changed while processing */
8261 wptr = cik_get_ih_wptr(rdev);
8269 * startup/shutdown callbacks
8272 * cik_startup - program the asic to a functional state
8274 * @rdev: radeon_device pointer
8276 * Programs the asic to a functional state (CIK).
8277 * Called by cik_init() and cik_resume().
8278 * Returns 0 for success, error for failure.
8280 static int cik_startup(struct radeon_device *rdev)
8282 struct radeon_ring *ring;
8286 /* enable pcie gen2/3 link */
8287 cik_pcie_gen3_enable(rdev);
8289 cik_program_aspm(rdev);
8291 /* scratch needs to be initialized before MC */
8292 r = r600_vram_scratch_init(rdev);
8296 cik_mc_program(rdev);
8298 if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
8299 r = ci_mc_load_microcode(rdev);
8301 DRM_ERROR("Failed to load MC firmware!\n");
8306 r = cik_pcie_gart_enable(rdev);
8311 /* allocate rlc buffers */
8312 if (rdev->flags & RADEON_IS_IGP) {
8313 if (rdev->family == CHIP_KAVERI) {
8314 rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;
8315 rdev->rlc.reg_list_size =
8316 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list);
8318 rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;
8319 rdev->rlc.reg_list_size =
8320 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list);
8323 rdev->rlc.cs_data = ci_cs_data;
8324 rdev->rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4;
8325 r = sumo_rlc_init(rdev);
8327 DRM_ERROR("Failed to init rlc BOs!\n");
8331 /* allocate wb buffer */
8332 r = radeon_wb_init(rdev);
8336 /* allocate mec buffers */
8337 r = cik_mec_init(rdev);
8339 DRM_ERROR("Failed to init MEC BOs!\n");
8343 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
8345 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8349 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
8351 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8355 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
8357 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8361 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
8363 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
8367 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
8369 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
8373 r = radeon_uvd_resume(rdev);
8375 r = uvd_v4_2_resume(rdev);
8377 r = radeon_fence_driver_start_ring(rdev,
8378 R600_RING_TYPE_UVD_INDEX);
8380 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
8384 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
8386 r = radeon_vce_resume(rdev);
8388 r = vce_v2_0_resume(rdev);
8390 r = radeon_fence_driver_start_ring(rdev,
8391 TN_RING_TYPE_VCE1_INDEX);
8393 r = radeon_fence_driver_start_ring(rdev,
8394 TN_RING_TYPE_VCE2_INDEX);
8397 dev_err(rdev->dev, "VCE init error (%d).\n", r);
8398 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
8399 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
8403 if (!rdev->irq.installed) {
8404 r = radeon_irq_kms_init(rdev);
8409 r = cik_irq_init(rdev);
8411 DRM_ERROR("radeon: IH init failed (%d).\n", r);
8412 radeon_irq_kms_fini(rdev);
8417 if (rdev->family == CHIP_HAWAII) {
8419 nop = PACKET3(PACKET3_NOP, 0x3FFF);
8421 nop = RADEON_CP_PACKET2;
8423 nop = PACKET3(PACKET3_NOP, 0x3FFF);
8426 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8427 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
8432 /* set up the compute queues */
8433 /* type-2 packets are deprecated on MEC, use type-3 instead */
8434 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8435 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
8439 ring->me = 1; /* first MEC */
8440 ring->pipe = 0; /* first pipe */
8441 ring->queue = 0; /* first queue */
8442 ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
8444 /* type-2 packets are deprecated on MEC, use type-3 instead */
8445 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8446 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
8450 /* dGPU only have 1 MEC */
8451 ring->me = 1; /* first MEC */
8452 ring->pipe = 0; /* first pipe */
8453 ring->queue = 1; /* second queue */
8454 ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
8456 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8457 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
8458 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
8462 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8463 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
8464 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
8468 r = cik_cp_resume(rdev);
8472 r = cik_sdma_resume(rdev);
8476 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8477 if (ring->ring_size) {
8478 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8481 r = uvd_v1_0_init(rdev);
8483 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
8488 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
8489 if (ring->ring_size)
8490 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8493 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
8494 if (ring->ring_size)
8495 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8499 r = vce_v1_0_init(rdev);
8500 else if (r != -ENOENT)
8501 DRM_ERROR("radeon: failed initializing VCE (%d).\n", r);
8503 r = radeon_ib_pool_init(rdev);
8505 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
8509 r = radeon_vm_manager_init(rdev);
8511 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
8515 r = dce6_audio_init(rdev);
8523 * cik_resume - resume the asic to a functional state
8525 * @rdev: radeon_device pointer
8527 * Programs the asic to a functional state (CIK).
8529 * Returns 0 for success, error for failure.
8531 int cik_resume(struct radeon_device *rdev)
8536 atom_asic_init(rdev->mode_info.atom_context);
8538 /* init golden registers */
8539 cik_init_golden_registers(rdev);
8541 if (rdev->pm.pm_method == PM_METHOD_DPM)
8542 radeon_pm_resume(rdev);
8544 rdev->accel_working = true;
8545 r = cik_startup(rdev);
8547 DRM_ERROR("cik startup failed on resume\n");
8548 rdev->accel_working = false;
8557 * cik_suspend - suspend the asic
8559 * @rdev: radeon_device pointer
8561 * Bring the chip into a state suitable for suspend (CIK).
8562 * Called at suspend.
8563 * Returns 0 for success.
8565 int cik_suspend(struct radeon_device *rdev)
8567 radeon_pm_suspend(rdev);
8568 dce6_audio_fini(rdev);
8569 radeon_vm_manager_fini(rdev);
8570 cik_cp_enable(rdev, false);
8571 cik_sdma_enable(rdev, false);
8572 uvd_v1_0_fini(rdev);
8573 radeon_uvd_suspend(rdev);
8574 radeon_vce_suspend(rdev);
8577 cik_irq_suspend(rdev);
8578 radeon_wb_disable(rdev);
8579 cik_pcie_gart_disable(rdev);
8583 /* Plan is to move initialization in that function and use
8584 * helper function so that radeon_device_init pretty much
8585 * do nothing more than calling asic specific function. This
8586 * should also allow to remove a bunch of callback function
8590 * cik_init - asic specific driver and hw init
8592 * @rdev: radeon_device pointer
8594 * Setup asic specific driver variables and program the hw
8595 * to a functional state (CIK).
8596 * Called at driver startup.
8597 * Returns 0 for success, errors for failure.
8599 int cik_init(struct radeon_device *rdev)
8601 struct radeon_ring *ring;
8605 if (!radeon_get_bios(rdev)) {
8606 if (ASIC_IS_AVIVO(rdev))
8609 /* Must be an ATOMBIOS */
8610 if (!rdev->is_atom_bios) {
8611 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
8614 r = radeon_atombios_init(rdev);
8618 /* Post card if necessary */
8619 if (!radeon_card_posted(rdev)) {
8621 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
8624 DRM_INFO("GPU not posted. posting now...\n");
8625 atom_asic_init(rdev->mode_info.atom_context);
8627 /* init golden registers */
8628 cik_init_golden_registers(rdev);
8629 /* Initialize scratch registers */
8630 cik_scratch_init(rdev);
8631 /* Initialize surface registers */
8632 radeon_surface_init(rdev);
8633 /* Initialize clocks */
8634 radeon_get_clock_info(rdev->ddev);
8637 r = radeon_fence_driver_init(rdev);
8641 /* initialize memory controller */
8642 r = cik_mc_init(rdev);
8645 /* Memory manager */
8646 r = radeon_bo_init(rdev);
8650 if (rdev->flags & RADEON_IS_IGP) {
8651 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8652 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
8653 r = cik_init_microcode(rdev);
8655 DRM_ERROR("Failed to load firmware!\n");
8660 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8661 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
8663 r = cik_init_microcode(rdev);
8665 DRM_ERROR("Failed to load firmware!\n");
8671 /* Initialize power management */
8672 radeon_pm_init(rdev);
8674 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8675 ring->ring_obj = NULL;
8676 r600_ring_init(rdev, ring, 1024 * 1024);
8678 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8679 ring->ring_obj = NULL;
8680 r600_ring_init(rdev, ring, 1024 * 1024);
8681 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
8685 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8686 ring->ring_obj = NULL;
8687 r600_ring_init(rdev, ring, 1024 * 1024);
8688 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
8692 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8693 ring->ring_obj = NULL;
8694 r600_ring_init(rdev, ring, 256 * 1024);
8696 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8697 ring->ring_obj = NULL;
8698 r600_ring_init(rdev, ring, 256 * 1024);
8700 r = radeon_uvd_init(rdev);
8702 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8703 ring->ring_obj = NULL;
8704 r600_ring_init(rdev, ring, 4096);
8707 r = radeon_vce_init(rdev);
8709 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
8710 ring->ring_obj = NULL;
8711 r600_ring_init(rdev, ring, 4096);
8713 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
8714 ring->ring_obj = NULL;
8715 r600_ring_init(rdev, ring, 4096);
8718 rdev->ih.ring_obj = NULL;
8719 r600_ih_ring_init(rdev, 64 * 1024);
8721 r = r600_pcie_gart_init(rdev);
8725 rdev->accel_working = true;
8726 r = cik_startup(rdev);
8728 dev_err(rdev->dev, "disabling GPU acceleration\n");
8730 cik_sdma_fini(rdev);
8732 sumo_rlc_fini(rdev);
8734 radeon_wb_fini(rdev);
8735 radeon_ib_pool_fini(rdev);
8736 radeon_vm_manager_fini(rdev);
8737 radeon_irq_kms_fini(rdev);
8738 cik_pcie_gart_fini(rdev);
8739 rdev->accel_working = false;
8742 /* Don't start up if the MC ucode is missing.
8743 * The default clocks and voltages before the MC ucode
8744 * is loaded are not suffient for advanced operations.
8746 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
8747 DRM_ERROR("radeon: MC ucode required for NI+.\n");
8755 * cik_fini - asic specific driver and hw fini
8757 * @rdev: radeon_device pointer
8759 * Tear down the asic specific driver variables and program the hw
8760 * to an idle state (CIK).
8761 * Called at driver unload.
8763 void cik_fini(struct radeon_device *rdev)
8765 radeon_pm_fini(rdev);
8767 cik_sdma_fini(rdev);
8771 sumo_rlc_fini(rdev);
8773 radeon_wb_fini(rdev);
8774 radeon_vm_manager_fini(rdev);
8775 radeon_ib_pool_fini(rdev);
8776 radeon_irq_kms_fini(rdev);
8777 uvd_v1_0_fini(rdev);
8778 radeon_uvd_fini(rdev);
8779 radeon_vce_fini(rdev);
8780 cik_pcie_gart_fini(rdev);
8781 r600_vram_scratch_fini(rdev);
8782 radeon_gem_fini(rdev);
8783 radeon_fence_driver_fini(rdev);
8784 radeon_bo_fini(rdev);
8785 radeon_atombios_fini(rdev);
8790 void dce8_program_fmt(struct drm_encoder *encoder)
8792 struct drm_device *dev = encoder->dev;
8793 struct radeon_device *rdev = dev->dev_private;
8794 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
8795 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
8796 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
8799 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
8802 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
8803 bpc = radeon_get_monitor_bpc(connector);
8804 dither = radeon_connector->dither;
8807 /* LVDS/eDP FMT is set up by atom */
8808 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
8811 /* not needed for analog */
8812 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
8813 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
8821 if (dither == RADEON_FMT_DITHER_ENABLE)
8822 /* XXX sort out optimal dither settings */
8823 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8824 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0));
8826 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0));
8829 if (dither == RADEON_FMT_DITHER_ENABLE)
8830 /* XXX sort out optimal dither settings */
8831 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8832 FMT_RGB_RANDOM_ENABLE |
8833 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1));
8835 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1));
8838 if (dither == RADEON_FMT_DITHER_ENABLE)
8839 /* XXX sort out optimal dither settings */
8840 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8841 FMT_RGB_RANDOM_ENABLE |
8842 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2));
8844 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2));
8851 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
8854 /* display watermark setup */
8856 * dce8_line_buffer_adjust - Set up the line buffer
8858 * @rdev: radeon_device pointer
8859 * @radeon_crtc: the selected display controller
8860 * @mode: the current display mode on the selected display
8863 * Setup up the line buffer allocation for
8864 * the selected display controller (CIK).
8865 * Returns the line buffer size in pixels.
8867 static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
8868 struct radeon_crtc *radeon_crtc,
8869 struct drm_display_mode *mode)
8871 u32 tmp, buffer_alloc, i;
8872 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
8875 * There are 6 line buffers, one for each display controllers.
8876 * There are 3 partitions per LB. Select the number of partitions
8877 * to enable based on the display width. For display widths larger
8878 * than 4096, you need use to use 2 display controllers and combine
8879 * them using the stereo blender.
8881 if (radeon_crtc->base.enabled && mode) {
8882 if (mode->crtc_hdisplay < 1920) {
8885 } else if (mode->crtc_hdisplay < 2560) {
8888 } else if (mode->crtc_hdisplay < 4096) {
8890 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8892 DRM_DEBUG_KMS("Mode too big for LB!\n");
8894 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8901 WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
8902 LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
8904 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
8905 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
8906 for (i = 0; i < rdev->usec_timeout; i++) {
8907 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
8908 DMIF_BUFFERS_ALLOCATED_COMPLETED)
8913 if (radeon_crtc->base.enabled && mode) {
8925 /* controller not enabled, so no lb used */
8930 * cik_get_number_of_dram_channels - get the number of dram channels
8932 * @rdev: radeon_device pointer
8934 * Look up the number of video ram channels (CIK).
8935 * Used for display watermark bandwidth calculations
8936 * Returns the number of dram channels
8938 static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
8940 u32 tmp = RREG32(MC_SHARED_CHMAP);
8942 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
8965 struct dce8_wm_params {
8966 u32 dram_channels; /* number of dram channels */
8967 u32 yclk; /* bandwidth per dram data pin in kHz */
8968 u32 sclk; /* engine clock in kHz */
8969 u32 disp_clk; /* display clock in kHz */
8970 u32 src_width; /* viewport width */
8971 u32 active_time; /* active display time in ns */
8972 u32 blank_time; /* blank time in ns */
8973 bool interlaced; /* mode is interlaced */
8974 fixed20_12 vsc; /* vertical scale ratio */
8975 u32 num_heads; /* number of active crtcs */
8976 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
8977 u32 lb_size; /* line buffer allocated to pipe */
8978 u32 vtaps; /* vertical scaler taps */
8982 * dce8_dram_bandwidth - get the dram bandwidth
8984 * @wm: watermark calculation data
8986 * Calculate the raw dram bandwidth (CIK).
8987 * Used for display watermark bandwidth calculations
8988 * Returns the dram bandwidth in MBytes/s
8990 static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
8992 /* Calculate raw DRAM Bandwidth */
8993 fixed20_12 dram_efficiency; /* 0.7 */
8994 fixed20_12 yclk, dram_channels, bandwidth;
8997 a.full = dfixed_const(1000);
8998 yclk.full = dfixed_const(wm->yclk);
8999 yclk.full = dfixed_div(yclk, a);
9000 dram_channels.full = dfixed_const(wm->dram_channels * 4);
9001 a.full = dfixed_const(10);
9002 dram_efficiency.full = dfixed_const(7);
9003 dram_efficiency.full = dfixed_div(dram_efficiency, a);
9004 bandwidth.full = dfixed_mul(dram_channels, yclk);
9005 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
9007 return dfixed_trunc(bandwidth);
9011 * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
9013 * @wm: watermark calculation data
9015 * Calculate the dram bandwidth used for display (CIK).
9016 * Used for display watermark bandwidth calculations
9017 * Returns the dram bandwidth for display in MBytes/s
9019 static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
9021 /* Calculate DRAM Bandwidth and the part allocated to display. */
9022 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
9023 fixed20_12 yclk, dram_channels, bandwidth;
9026 a.full = dfixed_const(1000);
9027 yclk.full = dfixed_const(wm->yclk);
9028 yclk.full = dfixed_div(yclk, a);
9029 dram_channels.full = dfixed_const(wm->dram_channels * 4);
9030 a.full = dfixed_const(10);
9031 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
9032 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
9033 bandwidth.full = dfixed_mul(dram_channels, yclk);
9034 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
9036 return dfixed_trunc(bandwidth);
9040 * dce8_data_return_bandwidth - get the data return bandwidth
9042 * @wm: watermark calculation data
9044 * Calculate the data return bandwidth used for display (CIK).
9045 * Used for display watermark bandwidth calculations
9046 * Returns the data return bandwidth in MBytes/s
9048 static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
9050 /* Calculate the display Data return Bandwidth */
9051 fixed20_12 return_efficiency; /* 0.8 */
9052 fixed20_12 sclk, bandwidth;
9055 a.full = dfixed_const(1000);
9056 sclk.full = dfixed_const(wm->sclk);
9057 sclk.full = dfixed_div(sclk, a);
9058 a.full = dfixed_const(10);
9059 return_efficiency.full = dfixed_const(8);
9060 return_efficiency.full = dfixed_div(return_efficiency, a);
9061 a.full = dfixed_const(32);
9062 bandwidth.full = dfixed_mul(a, sclk);
9063 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
9065 return dfixed_trunc(bandwidth);
9069 * dce8_dmif_request_bandwidth - get the dmif bandwidth
9071 * @wm: watermark calculation data
9073 * Calculate the dmif bandwidth used for display (CIK).
9074 * Used for display watermark bandwidth calculations
9075 * Returns the dmif bandwidth in MBytes/s
9077 static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
9079 /* Calculate the DMIF Request Bandwidth */
9080 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
9081 fixed20_12 disp_clk, bandwidth;
9084 a.full = dfixed_const(1000);
9085 disp_clk.full = dfixed_const(wm->disp_clk);
9086 disp_clk.full = dfixed_div(disp_clk, a);
9087 a.full = dfixed_const(32);
9088 b.full = dfixed_mul(a, disp_clk);
9090 a.full = dfixed_const(10);
9091 disp_clk_request_efficiency.full = dfixed_const(8);
9092 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
9094 bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
9096 return dfixed_trunc(bandwidth);
9100 * dce8_available_bandwidth - get the min available bandwidth
9102 * @wm: watermark calculation data
9104 * Calculate the min available bandwidth used for display (CIK).
9105 * Used for display watermark bandwidth calculations
9106 * Returns the min available bandwidth in MBytes/s
9108 static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
9110 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
9111 u32 dram_bandwidth = dce8_dram_bandwidth(wm);
9112 u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
9113 u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
9115 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
9119 * dce8_average_bandwidth - get the average available bandwidth
9121 * @wm: watermark calculation data
9123 * Calculate the average available bandwidth used for display (CIK).
9124 * Used for display watermark bandwidth calculations
9125 * Returns the average available bandwidth in MBytes/s
9127 static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
9129 /* Calculate the display mode Average Bandwidth
9130 * DisplayMode should contain the source and destination dimensions,
9134 fixed20_12 line_time;
9135 fixed20_12 src_width;
9136 fixed20_12 bandwidth;
9139 a.full = dfixed_const(1000);
9140 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
9141 line_time.full = dfixed_div(line_time, a);
9142 bpp.full = dfixed_const(wm->bytes_per_pixel);
9143 src_width.full = dfixed_const(wm->src_width);
9144 bandwidth.full = dfixed_mul(src_width, bpp);
9145 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
9146 bandwidth.full = dfixed_div(bandwidth, line_time);
9148 return dfixed_trunc(bandwidth);
9152 * dce8_latency_watermark - get the latency watermark
9154 * @wm: watermark calculation data
9156 * Calculate the latency watermark (CIK).
9157 * Used for display watermark bandwidth calculations
9158 * Returns the latency watermark in ns
9160 static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
9162 /* First calculate the latency in ns */
9163 u32 mc_latency = 2000; /* 2000 ns. */
9164 u32 available_bandwidth = dce8_available_bandwidth(wm);
9165 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
9166 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
9167 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
9168 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
9169 (wm->num_heads * cursor_line_pair_return_time);
9170 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
9171 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
9172 u32 tmp, dmif_size = 12288;
9175 if (wm->num_heads == 0)
9178 a.full = dfixed_const(2);
9179 b.full = dfixed_const(1);
9180 if ((wm->vsc.full > a.full) ||
9181 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
9183 ((wm->vsc.full >= a.full) && wm->interlaced))
9184 max_src_lines_per_dst_line = 4;
9186 max_src_lines_per_dst_line = 2;
9188 a.full = dfixed_const(available_bandwidth);
9189 b.full = dfixed_const(wm->num_heads);
9190 a.full = dfixed_div(a, b);
9192 b.full = dfixed_const(mc_latency + 512);
9193 c.full = dfixed_const(wm->disp_clk);
9194 b.full = dfixed_div(b, c);
9196 c.full = dfixed_const(dmif_size);
9197 b.full = dfixed_div(c, b);
9199 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
9201 b.full = dfixed_const(1000);
9202 c.full = dfixed_const(wm->disp_clk);
9203 b.full = dfixed_div(c, b);
9204 c.full = dfixed_const(wm->bytes_per_pixel);
9205 b.full = dfixed_mul(b, c);
9207 lb_fill_bw = min(tmp, dfixed_trunc(b));
9209 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
9210 b.full = dfixed_const(1000);
9211 c.full = dfixed_const(lb_fill_bw);
9212 b.full = dfixed_div(c, b);
9213 a.full = dfixed_div(a, b);
9214 line_fill_time = dfixed_trunc(a);
9216 if (line_fill_time < wm->active_time)
9219 return latency + (line_fill_time - wm->active_time);
9224 * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
9225 * average and available dram bandwidth
9227 * @wm: watermark calculation data
9229 * Check if the display average bandwidth fits in the display
9230 * dram bandwidth (CIK).
9231 * Used for display watermark bandwidth calculations
9232 * Returns true if the display fits, false if not.
9234 static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
9236 if (dce8_average_bandwidth(wm) <=
9237 (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
9244 * dce8_average_bandwidth_vs_available_bandwidth - check
9245 * average and available bandwidth
9247 * @wm: watermark calculation data
9249 * Check if the display average bandwidth fits in the display
9250 * available bandwidth (CIK).
9251 * Used for display watermark bandwidth calculations
9252 * Returns true if the display fits, false if not.
9254 static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
9256 if (dce8_average_bandwidth(wm) <=
9257 (dce8_available_bandwidth(wm) / wm->num_heads))
9264 * dce8_check_latency_hiding - check latency hiding
9266 * @wm: watermark calculation data
9268 * Check latency hiding (CIK).
9269 * Used for display watermark bandwidth calculations
9270 * Returns true if the display fits, false if not.
9272 static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
9274 u32 lb_partitions = wm->lb_size / wm->src_width;
9275 u32 line_time = wm->active_time + wm->blank_time;
9276 u32 latency_tolerant_lines;
9280 a.full = dfixed_const(1);
9281 if (wm->vsc.full > a.full)
9282 latency_tolerant_lines = 1;
9284 if (lb_partitions <= (wm->vtaps + 1))
9285 latency_tolerant_lines = 1;
9287 latency_tolerant_lines = 2;
9290 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
9292 if (dce8_latency_watermark(wm) <= latency_hiding)
9299 * dce8_program_watermarks - program display watermarks
9301 * @rdev: radeon_device pointer
9302 * @radeon_crtc: the selected display controller
9303 * @lb_size: line buffer size
9304 * @num_heads: number of display controllers in use
9306 * Calculate and program the display watermarks for the
9307 * selected display controller (CIK).
9309 static void dce8_program_watermarks(struct radeon_device *rdev,
9310 struct radeon_crtc *radeon_crtc,
9311 u32 lb_size, u32 num_heads)
9313 struct drm_display_mode *mode = &radeon_crtc->base.mode;
9314 struct dce8_wm_params wm_low, wm_high;
9317 u32 latency_watermark_a = 0, latency_watermark_b = 0;
9320 if (radeon_crtc->base.enabled && num_heads && mode) {
9321 pixel_period = 1000000 / (u32)mode->clock;
9322 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
9324 /* watermark for high clocks */
9325 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
9326 rdev->pm.dpm_enabled) {
9328 radeon_dpm_get_mclk(rdev, false) * 10;
9330 radeon_dpm_get_sclk(rdev, false) * 10;
9332 wm_high.yclk = rdev->pm.current_mclk * 10;
9333 wm_high.sclk = rdev->pm.current_sclk * 10;
9336 wm_high.disp_clk = mode->clock;
9337 wm_high.src_width = mode->crtc_hdisplay;
9338 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
9339 wm_high.blank_time = line_time - wm_high.active_time;
9340 wm_high.interlaced = false;
9341 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
9342 wm_high.interlaced = true;
9343 wm_high.vsc = radeon_crtc->vsc;
9345 if (radeon_crtc->rmx_type != RMX_OFF)
9347 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
9348 wm_high.lb_size = lb_size;
9349 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev);
9350 wm_high.num_heads = num_heads;
9352 /* set for high clocks */
9353 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535);
9355 /* possibly force display priority to high */
9356 /* should really do this at mode validation time... */
9357 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
9358 !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) ||
9359 !dce8_check_latency_hiding(&wm_high) ||
9360 (rdev->disp_priority == 2)) {
9361 DRM_DEBUG_KMS("force priority to high\n");
9364 /* watermark for low clocks */
9365 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
9366 rdev->pm.dpm_enabled) {
9368 radeon_dpm_get_mclk(rdev, true) * 10;
9370 radeon_dpm_get_sclk(rdev, true) * 10;
9372 wm_low.yclk = rdev->pm.current_mclk * 10;
9373 wm_low.sclk = rdev->pm.current_sclk * 10;
9376 wm_low.disp_clk = mode->clock;
9377 wm_low.src_width = mode->crtc_hdisplay;
9378 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
9379 wm_low.blank_time = line_time - wm_low.active_time;
9380 wm_low.interlaced = false;
9381 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
9382 wm_low.interlaced = true;
9383 wm_low.vsc = radeon_crtc->vsc;
9385 if (radeon_crtc->rmx_type != RMX_OFF)
9387 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
9388 wm_low.lb_size = lb_size;
9389 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev);
9390 wm_low.num_heads = num_heads;
9392 /* set for low clocks */
9393 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535);
9395 /* possibly force display priority to high */
9396 /* should really do this at mode validation time... */
9397 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
9398 !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) ||
9399 !dce8_check_latency_hiding(&wm_low) ||
9400 (rdev->disp_priority == 2)) {
9401 DRM_DEBUG_KMS("force priority to high\n");
9406 wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9408 tmp &= ~LATENCY_WATERMARK_MASK(3);
9409 tmp |= LATENCY_WATERMARK_MASK(1);
9410 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9411 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9412 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
9413 LATENCY_HIGH_WATERMARK(line_time)));
9415 tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9416 tmp &= ~LATENCY_WATERMARK_MASK(3);
9417 tmp |= LATENCY_WATERMARK_MASK(2);
9418 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9419 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9420 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
9421 LATENCY_HIGH_WATERMARK(line_time)));
9422 /* restore original selection */
9423 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
9425 /* save values for DPM */
9426 radeon_crtc->line_time = line_time;
9427 radeon_crtc->wm_high = latency_watermark_a;
9428 radeon_crtc->wm_low = latency_watermark_b;
9432 * dce8_bandwidth_update - program display watermarks
9434 * @rdev: radeon_device pointer
9436 * Calculate and program the display watermarks and line
9437 * buffer allocation (CIK).
9439 void dce8_bandwidth_update(struct radeon_device *rdev)
9441 struct drm_display_mode *mode = NULL;
9442 u32 num_heads = 0, lb_size;
9445 radeon_update_display_priority(rdev);
9447 for (i = 0; i < rdev->num_crtc; i++) {
9448 if (rdev->mode_info.crtcs[i]->base.enabled)
9451 for (i = 0; i < rdev->num_crtc; i++) {
9452 mode = &rdev->mode_info.crtcs[i]->base.mode;
9453 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
9454 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
9459 * cik_get_gpu_clock_counter - return GPU clock counter snapshot
9461 * @rdev: radeon_device pointer
9463 * Fetches a GPU clock counter snapshot (SI).
9464 * Returns the 64 bit clock counter snapshot.
9466 uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
9470 mutex_lock(&rdev->gpu_clock_mutex);
9471 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
9472 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
9473 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
9474 mutex_unlock(&rdev->gpu_clock_mutex);
9478 static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
9479 u32 cntl_reg, u32 status_reg)
9482 struct atom_clock_dividers dividers;
9485 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9486 clock, false, ÷rs);
9490 tmp = RREG32_SMC(cntl_reg);
9491 tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
9492 tmp |= dividers.post_divider;
9493 WREG32_SMC(cntl_reg, tmp);
9495 for (i = 0; i < 100; i++) {
9496 if (RREG32_SMC(status_reg) & DCLK_STATUS)
9506 int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
9510 r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
9514 r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
9518 int cik_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
9521 struct atom_clock_dividers dividers;
9524 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9525 ecclk, false, ÷rs);
9529 for (i = 0; i < 100; i++) {
9530 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9537 tmp = RREG32_SMC(CG_ECLK_CNTL);
9538 tmp &= ~(ECLK_DIR_CNTL_EN|ECLK_DIVIDER_MASK);
9539 tmp |= dividers.post_divider;
9540 WREG32_SMC(CG_ECLK_CNTL, tmp);
9542 for (i = 0; i < 100; i++) {
9543 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9553 static void cik_pcie_gen3_enable(struct radeon_device *rdev)
9555 struct pci_dev *root = rdev->pdev->bus->self;
9556 int bridge_pos, gpu_pos;
9557 u32 speed_cntl, mask, current_data_rate;
9561 if (radeon_pcie_gen2 == 0)
9564 if (rdev->flags & RADEON_IS_IGP)
9567 if (!(rdev->flags & RADEON_IS_PCIE))
9570 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
9574 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
9577 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9578 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
9579 LC_CURRENT_DATA_RATE_SHIFT;
9580 if (mask & DRM_PCIE_SPEED_80) {
9581 if (current_data_rate == 2) {
9582 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
9585 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
9586 } else if (mask & DRM_PCIE_SPEED_50) {
9587 if (current_data_rate == 1) {
9588 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
9591 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
9594 bridge_pos = pci_pcie_cap(root);
9598 gpu_pos = pci_pcie_cap(rdev->pdev);
9602 if (mask & DRM_PCIE_SPEED_80) {
9603 /* re-try equalization if gen3 is not already enabled */
9604 if (current_data_rate != 2) {
9605 u16 bridge_cfg, gpu_cfg;
9606 u16 bridge_cfg2, gpu_cfg2;
9607 u32 max_lw, current_lw, tmp;
9609 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9610 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9612 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
9613 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9615 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
9616 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9618 tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9619 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
9620 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
9622 if (current_lw < max_lw) {
9623 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9624 if (tmp & LC_RENEGOTIATION_SUPPORT) {
9625 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
9626 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
9627 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
9628 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
9632 for (i = 0; i < 10; i++) {
9634 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
9635 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
9638 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9639 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9641 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
9642 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
9644 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9645 tmp |= LC_SET_QUIESCE;
9646 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9648 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9650 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9655 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
9656 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9657 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
9658 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9660 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
9661 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9662 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
9663 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9666 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
9667 tmp16 &= ~((1 << 4) | (7 << 9));
9668 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
9669 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
9671 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9672 tmp16 &= ~((1 << 4) | (7 << 9));
9673 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
9674 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9676 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9677 tmp &= ~LC_SET_QUIESCE;
9678 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9683 /* set the link speed */
9684 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
9685 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
9686 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9688 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9690 if (mask & DRM_PCIE_SPEED_80)
9691 tmp16 |= 3; /* gen3 */
9692 else if (mask & DRM_PCIE_SPEED_50)
9693 tmp16 |= 2; /* gen2 */
9695 tmp16 |= 1; /* gen1 */
9696 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9698 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9699 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
9700 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9702 for (i = 0; i < rdev->usec_timeout; i++) {
9703 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9704 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
9710 static void cik_program_aspm(struct radeon_device *rdev)
9713 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
9714 bool disable_clkreq = false;
9716 if (radeon_aspm == 0)
9719 /* XXX double check IGPs */
9720 if (rdev->flags & RADEON_IS_IGP)
9723 if (!(rdev->flags & RADEON_IS_PCIE))
9726 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9727 data &= ~LC_XMIT_N_FTS_MASK;
9728 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
9730 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
9732 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
9733 data |= LC_GO_TO_RECOVERY;
9735 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
9737 orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL);
9738 data |= P_IGNORE_EDB_ERR;
9740 WREG32_PCIE_PORT(PCIE_P_CNTL, data);
9742 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9743 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
9744 data |= LC_PMI_TO_L1_DIS;
9746 data |= LC_L0S_INACTIVITY(7);
9749 data |= LC_L1_INACTIVITY(7);
9750 data &= ~LC_PMI_TO_L1_DIS;
9752 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9754 if (!disable_plloff_in_l1) {
9755 bool clk_req_support;
9757 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0);
9758 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9759 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9761 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data);
9763 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1);
9764 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9765 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9767 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data);
9769 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0);
9770 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9771 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9773 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data);
9775 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1);
9776 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9777 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9779 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data);
9781 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9782 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
9783 data |= LC_DYN_LANES_PWR_STATE(3);
9785 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
9787 if (!disable_clkreq) {
9788 struct pci_dev *root = rdev->pdev->bus->self;
9791 clk_req_support = false;
9792 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
9793 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
9794 clk_req_support = true;
9796 clk_req_support = false;
9799 if (clk_req_support) {
9800 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
9801 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
9803 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
9805 orig = data = RREG32_SMC(THM_CLK_CNTL);
9806 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
9807 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
9809 WREG32_SMC(THM_CLK_CNTL, data);
9811 orig = data = RREG32_SMC(MISC_CLK_CTRL);
9812 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
9813 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
9815 WREG32_SMC(MISC_CLK_CTRL, data);
9817 orig = data = RREG32_SMC(CG_CLKPIN_CNTL);
9818 data &= ~BCLK_AS_XCLK;
9820 WREG32_SMC(CG_CLKPIN_CNTL, data);
9822 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2);
9823 data &= ~FORCE_BIF_REFCLK_EN;
9825 WREG32_SMC(CG_CLKPIN_CNTL_2, data);
9827 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL);
9828 data &= ~MPLL_CLKOUT_SEL_MASK;
9829 data |= MPLL_CLKOUT_SEL(4);
9831 WREG32_SMC(MPLL_BYPASSCLK_SEL, data);
9836 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9839 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
9840 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
9842 WREG32_PCIE_PORT(PCIE_CNTL2, data);
9845 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9846 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
9847 data = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9848 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
9849 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9850 data &= ~LC_L0S_INACTIVITY_MASK;
9852 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);