drivers/gpu/drm/radeon/cik.c

   1 /*
   2  * Copyright 2012 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24 #include <linux/firmware.h>
  25 #include <linux/slab.h>
  26 #include <linux/module.h>
  27 #include "drmP.h"
  28 #include "radeon.h"
  29 #include "radeon_asic.h"
  30 #include "cikd.h"
  31 #include "atom.h"
  32 #include "cik_blit_shaders.h"
  33 #include "radeon_ucode.h"
  34 #include "clearstate_ci.h"
  35
  36 MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
  37 MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
  38 MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
  39 MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
  40 MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
  41 MODULE_FIRMWARE("radeon/BONAIRE_mc2.bin");
  42 MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
  43 MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
  44 MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
  45
  46 MODULE_FIRMWARE("radeon/bonaire_pfp.bin");
  47 MODULE_FIRMWARE("radeon/bonaire_me.bin");
  48 MODULE_FIRMWARE("radeon/bonaire_ce.bin");
  49 MODULE_FIRMWARE("radeon/bonaire_mec.bin");
  50 MODULE_FIRMWARE("radeon/bonaire_mc.bin");
  51 MODULE_FIRMWARE("radeon/bonaire_rlc.bin");
  52 MODULE_FIRMWARE("radeon/bonaire_sdma.bin");
  53 MODULE_FIRMWARE("radeon/bonaire_smc.bin");
  54
  55 MODULE_FIRMWARE("radeon/HAWAII_pfp.bin");
  56 MODULE_FIRMWARE("radeon/HAWAII_me.bin");
  57 MODULE_FIRMWARE("radeon/HAWAII_ce.bin");
  58 MODULE_FIRMWARE("radeon/HAWAII_mec.bin");
  59 MODULE_FIRMWARE("radeon/HAWAII_mc.bin");
  60 MODULE_FIRMWARE("radeon/HAWAII_mc2.bin");
  61 MODULE_FIRMWARE("radeon/HAWAII_rlc.bin");
  62 MODULE_FIRMWARE("radeon/HAWAII_sdma.bin");
  63 MODULE_FIRMWARE("radeon/HAWAII_smc.bin");
  64
  65 MODULE_FIRMWARE("radeon/hawaii_pfp.bin");
  66 MODULE_FIRMWARE("radeon/hawaii_me.bin");
  67 MODULE_FIRMWARE("radeon/hawaii_ce.bin");
  68 MODULE_FIRMWARE("radeon/hawaii_mec.bin");
  69 MODULE_FIRMWARE("radeon/hawaii_mc.bin");
  70 MODULE_FIRMWARE("radeon/hawaii_rlc.bin");
  71 MODULE_FIRMWARE("radeon/hawaii_sdma.bin");
  72 MODULE_FIRMWARE("radeon/hawaii_smc.bin");
  73
  74 MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
  75 MODULE_FIRMWARE("radeon/KAVERI_me.bin");
  76 MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
  77 MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
  78 MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
  79 MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
  80
  81 MODULE_FIRMWARE("radeon/kaveri_pfp.bin");
  82 MODULE_FIRMWARE("radeon/kaveri_me.bin");
  83 MODULE_FIRMWARE("radeon/kaveri_ce.bin");
  84 MODULE_FIRMWARE("radeon/kaveri_mec.bin");
  85 MODULE_FIRMWARE("radeon/kaveri_mec2.bin");
  86 MODULE_FIRMWARE("radeon/kaveri_rlc.bin");
  87 MODULE_FIRMWARE("radeon/kaveri_sdma.bin");
  88
  89 MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
  90 MODULE_FIRMWARE("radeon/KABINI_me.bin");
  91 MODULE_FIRMWARE("radeon/KABINI_ce.bin");
  92 MODULE_FIRMWARE("radeon/KABINI_mec.bin");
  93 MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
  94 MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
  95
  96 MODULE_FIRMWARE("radeon/kabini_pfp.bin");
  97 MODULE_FIRMWARE("radeon/kabini_me.bin");
  98 MODULE_FIRMWARE("radeon/kabini_ce.bin");
  99 MODULE_FIRMWARE("radeon/kabini_mec.bin");
 100 MODULE_FIRMWARE("radeon/kabini_rlc.bin");
 101 MODULE_FIRMWARE("radeon/kabini_sdma.bin");
 102
 103 MODULE_FIRMWARE("radeon/MULLINS_pfp.bin");
 104 MODULE_FIRMWARE("radeon/MULLINS_me.bin");
 105 MODULE_FIRMWARE("radeon/MULLINS_ce.bin");
 106 MODULE_FIRMWARE("radeon/MULLINS_mec.bin");
 107 MODULE_FIRMWARE("radeon/MULLINS_rlc.bin");
 108 MODULE_FIRMWARE("radeon/MULLINS_sdma.bin");
 109
 110 MODULE_FIRMWARE("radeon/mullins_pfp.bin");
 111 MODULE_FIRMWARE("radeon/mullins_me.bin");
 112 MODULE_FIRMWARE("radeon/mullins_ce.bin");
 113 MODULE_FIRMWARE("radeon/mullins_mec.bin");
 114 MODULE_FIRMWARE("radeon/mullins_rlc.bin");
 115 MODULE_FIRMWARE("radeon/mullins_sdma.bin");
 116
 117 extern int r600_ih_ring_alloc(struct radeon_device *rdev);
 118 extern void r600_ih_ring_fini(struct radeon_device *rdev);
 119 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
 120 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
 121 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
 122 extern void sumo_rlc_fini(struct radeon_device *rdev);
 123 extern int sumo_rlc_init(struct radeon_device *rdev);
 124 extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
 125 extern void si_rlc_reset(struct radeon_device *rdev);
 126 extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
 127 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
 128 extern int cik_sdma_resume(struct radeon_device *rdev);
 129 extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
 130 extern void cik_sdma_fini(struct radeon_device *rdev);
 131 extern void vce_v2_0_enable_mgcg(struct radeon_device *rdev, bool enable);
 132 static void cik_rlc_stop(struct radeon_device *rdev);
 133 static void cik_pcie_gen3_enable(struct radeon_device *rdev);
 134 static void cik_program_aspm(struct radeon_device *rdev);
 135 static void cik_init_pg(struct radeon_device *rdev);
 136 static void cik_init_cg(struct radeon_device *rdev);
 137 static void cik_fini_pg(struct radeon_device *rdev);
 138 static void cik_fini_cg(struct radeon_device *rdev);
 139 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
 140                                           bool enable);
 141
 142 /* get temperature in millidegrees */
 143 int ci_get_temp(struct radeon_device *rdev)
 144 {
 145         u32 temp;
 146         int actual_temp = 0;
 147
 148         temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
 149                 CTF_TEMP_SHIFT;
 150
 151         if (temp & 0x200)
 152                 actual_temp = 255;
 153         else
 154                 actual_temp = temp & 0x1ff;
 155
 156         actual_temp = actual_temp * 1000;
 157
 158         return actual_temp;
 159 }
 160
 161 /* get temperature in millidegrees */
 162 int kv_get_temp(struct radeon_device *rdev)
 163 {
 164         u32 temp;
 165         int actual_temp = 0;
 166
 167         temp = RREG32_SMC(0xC0300E0C);
 168
 169         if (temp)
 170                 actual_temp = (temp / 8) - 49;
 171         else
 172                 actual_temp = 0;
 173
 174         actual_temp = actual_temp * 1000;
 175
 176         return actual_temp;
 177 }
 178
 179 /*
 180  * Indirect registers accessor
 181  */
 182 u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
 183 {
 184         unsigned long flags;
 185         u32 r;
 186
 187         spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
 188         WREG32(PCIE_INDEX, reg);
 189         (void)RREG32(PCIE_INDEX);
 190         r = RREG32(PCIE_DATA);
 191         spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
 192         return r;
 193 }
 194
 195 void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
 196 {
 197         unsigned long flags;
 198
 199         spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
 200         WREG32(PCIE_INDEX, reg);
 201         (void)RREG32(PCIE_INDEX);
 202         WREG32(PCIE_DATA, v);
 203         (void)RREG32(PCIE_DATA);
 204         spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
 205 }
 206
 207 static const u32 spectre_rlc_save_restore_register_list[] =
 208 {
 209         (0x0e00 << 16) | (0xc12c >> 2),
 210         0x00000000,
 211         (0x0e00 << 16) | (0xc140 >> 2),
 212         0x00000000,
 213         (0x0e00 << 16) | (0xc150 >> 2),
 214         0x00000000,
 215         (0x0e00 << 16) | (0xc15c >> 2),
 216         0x00000000,
 217         (0x0e00 << 16) | (0xc168 >> 2),
 218         0x00000000,
 219         (0x0e00 << 16) | (0xc170 >> 2),
 220         0x00000000,
 221         (0x0e00 << 16) | (0xc178 >> 2),
 222         0x00000000,
 223         (0x0e00 << 16) | (0xc204 >> 2),
 224         0x00000000,
 225         (0x0e00 << 16) | (0xc2b4 >> 2),
 226         0x00000000,
 227         (0x0e00 << 16) | (0xc2b8 >> 2),
 228         0x00000000,
 229         (0x0e00 << 16) | (0xc2bc >> 2),
 230         0x00000000,
 231         (0x0e00 << 16) | (0xc2c0 >> 2),
 232         0x00000000,
 233         (0x0e00 << 16) | (0x8228 >> 2),
 234         0x00000000,
 235         (0x0e00 << 16) | (0x829c >> 2),
 236         0x00000000,
 237         (0x0e00 << 16) | (0x869c >> 2),
 238         0x00000000,
 239         (0x0600 << 16) | (0x98f4 >> 2),
 240         0x00000000,
 241         (0x0e00 << 16) | (0x98f8 >> 2),
 242         0x00000000,
 243         (0x0e00 << 16) | (0x9900 >> 2),
 244         0x00000000,
 245         (0x0e00 << 16) | (0xc260 >> 2),
 246         0x00000000,
 247         (0x0e00 << 16) | (0x90e8 >> 2),
 248         0x00000000,
 249         (0x0e00 << 16) | (0x3c000 >> 2),
 250         0x00000000,
 251         (0x0e00 << 16) | (0x3c00c >> 2),
 252         0x00000000,
 253         (0x0e00 << 16) | (0x8c1c >> 2),
 254         0x00000000,
 255         (0x0e00 << 16) | (0x9700 >> 2),
 256         0x00000000,
 257         (0x0e00 << 16) | (0xcd20 >> 2),
 258         0x00000000,
 259         (0x4e00 << 16) | (0xcd20 >> 2),
 260         0x00000000,
 261         (0x5e00 << 16) | (0xcd20 >> 2),
 262         0x00000000,
 263         (0x6e00 << 16) | (0xcd20 >> 2),
 264         0x00000000,
 265         (0x7e00 << 16) | (0xcd20 >> 2),
 266         0x00000000,
 267         (0x8e00 << 16) | (0xcd20 >> 2),
 268         0x00000000,
 269         (0x9e00 << 16) | (0xcd20 >> 2),
 270         0x00000000,
 271         (0xae00 << 16) | (0xcd20 >> 2),
 272         0x00000000,
 273         (0xbe00 << 16) | (0xcd20 >> 2),
 274         0x00000000,
 275         (0x0e00 << 16) | (0x89bc >> 2),
 276         0x00000000,
 277         (0x0e00 << 16) | (0x8900 >> 2),
 278         0x00000000,
 279         0x3,
 280         (0x0e00 << 16) | (0xc130 >> 2),
 281         0x00000000,
 282         (0x0e00 << 16) | (0xc134 >> 2),
 283         0x00000000,
 284         (0x0e00 << 16) | (0xc1fc >> 2),
 285         0x00000000,
 286         (0x0e00 << 16) | (0xc208 >> 2),
 287         0x00000000,
 288         (0x0e00 << 16) | (0xc264 >> 2),
 289         0x00000000,
 290         (0x0e00 << 16) | (0xc268 >> 2),
 291         0x00000000,
 292         (0x0e00 << 16) | (0xc26c >> 2),
 293         0x00000000,
 294         (0x0e00 << 16) | (0xc270 >> 2),
 295         0x00000000,
 296         (0x0e00 << 16) | (0xc274 >> 2),
 297         0x00000000,
 298         (0x0e00 << 16) | (0xc278 >> 2),
 299         0x00000000,
 300         (0x0e00 << 16) | (0xc27c >> 2),
 301         0x00000000,
 302         (0x0e00 << 16) | (0xc280 >> 2),
 303         0x00000000,
 304         (0x0e00 << 16) | (0xc284 >> 2),
 305         0x00000000,
 306         (0x0e00 << 16) | (0xc288 >> 2),
 307         0x00000000,
 308         (0x0e00 << 16) | (0xc28c >> 2),
 309         0x00000000,
 310         (0x0e00 << 16) | (0xc290 >> 2),
 311         0x00000000,
 312         (0x0e00 << 16) | (0xc294 >> 2),
 313         0x00000000,
 314         (0x0e00 << 16) | (0xc298 >> 2),
 315         0x00000000,
 316         (0x0e00 << 16) | (0xc29c >> 2),
 317         0x00000000,
 318         (0x0e00 << 16) | (0xc2a0 >> 2),
 319         0x00000000,
 320         (0x0e00 << 16) | (0xc2a4 >> 2),
 321         0x00000000,
 322         (0x0e00 << 16) | (0xc2a8 >> 2),
 323         0x00000000,
 324         (0x0e00 << 16) | (0xc2ac  >> 2),
 325         0x00000000,
 326         (0x0e00 << 16) | (0xc2b0 >> 2),
 327         0x00000000,
 328         (0x0e00 << 16) | (0x301d0 >> 2),
 329         0x00000000,
 330         (0x0e00 << 16) | (0x30238 >> 2),
 331         0x00000000,
 332         (0x0e00 << 16) | (0x30250 >> 2),
 333         0x00000000,
 334         (0x0e00 << 16) | (0x30254 >> 2),
 335         0x00000000,
 336         (0x0e00 << 16) | (0x30258 >> 2),
 337         0x00000000,
 338         (0x0e00 << 16) | (0x3025c >> 2),
 339         0x00000000,
 340         (0x4e00 << 16) | (0xc900 >> 2),
 341         0x00000000,
 342         (0x5e00 << 16) | (0xc900 >> 2),
 343         0x00000000,
 344         (0x6e00 << 16) | (0xc900 >> 2),
 345         0x00000000,
 346         (0x7e00 << 16) | (0xc900 >> 2),
 347         0x00000000,
 348         (0x8e00 << 16) | (0xc900 >> 2),
 349         0x00000000,
 350         (0x9e00 << 16) | (0xc900 >> 2),
 351         0x00000000,
 352         (0xae00 << 16) | (0xc900 >> 2),
 353         0x00000000,
 354         (0xbe00 << 16) | (0xc900 >> 2),
 355         0x00000000,
 356         (0x4e00 << 16) | (0xc904 >> 2),
 357         0x00000000,
 358         (0x5e00 << 16) | (0xc904 >> 2),
 359         0x00000000,
 360         (0x6e00 << 16) | (0xc904 >> 2),
 361         0x00000000,
 362         (0x7e00 << 16) | (0xc904 >> 2),
 363         0x00000000,
 364         (0x8e00 << 16) | (0xc904 >> 2),
 365         0x00000000,
 366         (0x9e00 << 16) | (0xc904 >> 2),
 367         0x00000000,
 368         (0xae00 << 16) | (0xc904 >> 2),
 369         0x00000000,
 370         (0xbe00 << 16) | (0xc904 >> 2),
 371         0x00000000,
 372         (0x4e00 << 16) | (0xc908 >> 2),
 373         0x00000000,
 374         (0x5e00 << 16) | (0xc908 >> 2),
 375         0x00000000,
 376         (0x6e00 << 16) | (0xc908 >> 2),
 377         0x00000000,
 378         (0x7e00 << 16) | (0xc908 >> 2),
 379         0x00000000,
 380         (0x8e00 << 16) | (0xc908 >> 2),
 381         0x00000000,
 382         (0x9e00 << 16) | (0xc908 >> 2),
 383         0x00000000,
 384         (0xae00 << 16) | (0xc908 >> 2),
 385         0x00000000,
 386         (0xbe00 << 16) | (0xc908 >> 2),
 387         0x00000000,
 388         (0x4e00 << 16) | (0xc90c >> 2),
 389         0x00000000,
 390         (0x5e00 << 16) | (0xc90c >> 2),
 391         0x00000000,
 392         (0x6e00 << 16) | (0xc90c >> 2),
 393         0x00000000,
 394         (0x7e00 << 16) | (0xc90c >> 2),
 395         0x00000000,
 396         (0x8e00 << 16) | (0xc90c >> 2),
 397         0x00000000,
 398         (0x9e00 << 16) | (0xc90c >> 2),
 399         0x00000000,
 400         (0xae00 << 16) | (0xc90c >> 2),
 401         0x00000000,
 402         (0xbe00 << 16) | (0xc90c >> 2),
 403         0x00000000,
 404         (0x4e00 << 16) | (0xc910 >> 2),
 405         0x00000000,
 406         (0x5e00 << 16) | (0xc910 >> 2),
 407         0x00000000,
 408         (0x6e00 << 16) | (0xc910 >> 2),
 409         0x00000000,
 410         (0x7e00 << 16) | (0xc910 >> 2),
 411         0x00000000,
 412         (0x8e00 << 16) | (0xc910 >> 2),
 413         0x00000000,
 414         (0x9e00 << 16) | (0xc910 >> 2),
 415         0x00000000,
 416         (0xae00 << 16) | (0xc910 >> 2),
 417         0x00000000,
 418         (0xbe00 << 16) | (0xc910 >> 2),
 419         0x00000000,
 420         (0x0e00 << 16) | (0xc99c >> 2),
 421         0x00000000,
 422         (0x0e00 << 16) | (0x9834 >> 2),
 423         0x00000000,
 424         (0x0000 << 16) | (0x30f00 >> 2),
 425         0x00000000,
 426         (0x0001 << 16) | (0x30f00 >> 2),
 427         0x00000000,
 428         (0x0000 << 16) | (0x30f04 >> 2),
 429         0x00000000,
 430         (0x0001 << 16) | (0x30f04 >> 2),
 431         0x00000000,
 432         (0x0000 << 16) | (0x30f08 >> 2),
 433         0x00000000,
 434         (0x0001 << 16) | (0x30f08 >> 2),
 435         0x00000000,
 436         (0x0000 << 16) | (0x30f0c >> 2),
 437         0x00000000,
 438         (0x0001 << 16) | (0x30f0c >> 2),
 439         0x00000000,
 440         (0x0600 << 16) | (0x9b7c >> 2),
 441         0x00000000,
 442         (0x0e00 << 16) | (0x8a14 >> 2),
 443         0x00000000,
 444         (0x0e00 << 16) | (0x8a18 >> 2),
 445         0x00000000,
 446         (0x0600 << 16) | (0x30a00 >> 2),
 447         0x00000000,
 448         (0x0e00 << 16) | (0x8bf0 >> 2),
 449         0x00000000,
 450         (0x0e00 << 16) | (0x8bcc >> 2),
 451         0x00000000,
 452         (0x0e00 << 16) | (0x8b24 >> 2),
 453         0x00000000,
 454         (0x0e00 << 16) | (0x30a04 >> 2),
 455         0x00000000,
 456         (0x0600 << 16) | (0x30a10 >> 2),
 457         0x00000000,
 458         (0x0600 << 16) | (0x30a14 >> 2),
 459         0x00000000,
 460         (0x0600 << 16) | (0x30a18 >> 2),
 461         0x00000000,
 462         (0x0600 << 16) | (0x30a2c >> 2),
 463         0x00000000,
 464         (0x0e00 << 16) | (0xc700 >> 2),
 465         0x00000000,
 466         (0x0e00 << 16) | (0xc704 >> 2),
 467         0x00000000,
 468         (0x0e00 << 16) | (0xc708 >> 2),
 469         0x00000000,
 470         (0x0e00 << 16) | (0xc768 >> 2),
 471         0x00000000,
 472         (0x0400 << 16) | (0xc770 >> 2),
 473         0x00000000,
 474         (0x0400 << 16) | (0xc774 >> 2),
 475         0x00000000,
 476         (0x0400 << 16) | (0xc778 >> 2),
 477         0x00000000,
 478         (0x0400 << 16) | (0xc77c >> 2),
 479         0x00000000,
 480         (0x0400 << 16) | (0xc780 >> 2),
 481         0x00000000,
 482         (0x0400 << 16) | (0xc784 >> 2),
 483         0x00000000,
 484         (0x0400 << 16) | (0xc788 >> 2),
 485         0x00000000,
 486         (0x0400 << 16) | (0xc78c >> 2),
 487         0x00000000,
 488         (0x0400 << 16) | (0xc798 >> 2),
 489         0x00000000,
 490         (0x0400 << 16) | (0xc79c >> 2),
 491         0x00000000,
 492         (0x0400 << 16) | (0xc7a0 >> 2),
 493         0x00000000,
 494         (0x0400 << 16) | (0xc7a4 >> 2),
 495         0x00000000,
 496         (0x0400 << 16) | (0xc7a8 >> 2),
 497         0x00000000,
 498         (0x0400 << 16) | (0xc7ac >> 2),
 499         0x00000000,
 500         (0x0400 << 16) | (0xc7b0 >> 2),
 501         0x00000000,
 502         (0x0400 << 16) | (0xc7b4 >> 2),
 503         0x00000000,
 504         (0x0e00 << 16) | (0x9100 >> 2),
 505         0x00000000,
 506         (0x0e00 << 16) | (0x3c010 >> 2),
 507         0x00000000,
 508         (0x0e00 << 16) | (0x92a8 >> 2),
 509         0x00000000,
 510         (0x0e00 << 16) | (0x92ac >> 2),
 511         0x00000000,
 512         (0x0e00 << 16) | (0x92b4 >> 2),
 513         0x00000000,
 514         (0x0e00 << 16) | (0x92b8 >> 2),
 515         0x00000000,
 516         (0x0e00 << 16) | (0x92bc >> 2),
 517         0x00000000,
 518         (0x0e00 << 16) | (0x92c0 >> 2),
 519         0x00000000,
 520         (0x0e00 << 16) | (0x92c4 >> 2),
 521         0x00000000,
 522         (0x0e00 << 16) | (0x92c8 >> 2),
 523         0x00000000,
 524         (0x0e00 << 16) | (0x92cc >> 2),
 525         0x00000000,
 526         (0x0e00 << 16) | (0x92d0 >> 2),
 527         0x00000000,
 528         (0x0e00 << 16) | (0x8c00 >> 2),
 529         0x00000000,
 530         (0x0e00 << 16) | (0x8c04 >> 2),
 531         0x00000000,
 532         (0x0e00 << 16) | (0x8c20 >> 2),
 533         0x00000000,
 534         (0x0e00 << 16) | (0x8c38 >> 2),
 535         0x00000000,
 536         (0x0e00 << 16) | (0x8c3c >> 2),
 537         0x00000000,
 538         (0x0e00 << 16) | (0xae00 >> 2),
 539         0x00000000,
 540         (0x0e00 << 16) | (0x9604 >> 2),
 541         0x00000000,
 542         (0x0e00 << 16) | (0xac08 >> 2),
 543         0x00000000,
 544         (0x0e00 << 16) | (0xac0c >> 2),
 545         0x00000000,
 546         (0x0e00 << 16) | (0xac10 >> 2),
 547         0x00000000,
 548         (0x0e00 << 16) | (0xac14 >> 2),
 549         0x00000000,
 550         (0x0e00 << 16) | (0xac58 >> 2),
 551         0x00000000,
 552         (0x0e00 << 16) | (0xac68 >> 2),
 553         0x00000000,
 554         (0x0e00 << 16) | (0xac6c >> 2),
 555         0x00000000,
 556         (0x0e00 << 16) | (0xac70 >> 2),
 557         0x00000000,
 558         (0x0e00 << 16) | (0xac74 >> 2),
 559         0x00000000,
 560         (0x0e00 << 16) | (0xac78 >> 2),
 561         0x00000000,
 562         (0x0e00 << 16) | (0xac7c >> 2),
 563         0x00000000,
 564         (0x0e00 << 16) | (0xac80 >> 2),
 565         0x00000000,
 566         (0x0e00 << 16) | (0xac84 >> 2),
 567         0x00000000,
 568         (0x0e00 << 16) | (0xac88 >> 2),
 569         0x00000000,
 570         (0x0e00 << 16) | (0xac8c >> 2),
 571         0x00000000,
 572         (0x0e00 << 16) | (0x970c >> 2),
 573         0x00000000,
 574         (0x0e00 << 16) | (0x9714 >> 2),
 575         0x00000000,
 576         (0x0e00 << 16) | (0x9718 >> 2),
 577         0x00000000,
 578         (0x0e00 << 16) | (0x971c >> 2),
 579         0x00000000,
 580         (0x0e00 << 16) | (0x31068 >> 2),
 581         0x00000000,
 582         (0x4e00 << 16) | (0x31068 >> 2),
 583         0x00000000,
 584         (0x5e00 << 16) | (0x31068 >> 2),
 585         0x00000000,
 586         (0x6e00 << 16) | (0x31068 >> 2),
 587         0x00000000,
 588         (0x7e00 << 16) | (0x31068 >> 2),
 589         0x00000000,
 590         (0x8e00 << 16) | (0x31068 >> 2),
 591         0x00000000,
 592         (0x9e00 << 16) | (0x31068 >> 2),
 593         0x00000000,
 594         (0xae00 << 16) | (0x31068 >> 2),
 595         0x00000000,
 596         (0xbe00 << 16) | (0x31068 >> 2),
 597         0x00000000,
 598         (0x0e00 << 16) | (0xcd10 >> 2),
 599         0x00000000,
 600         (0x0e00 << 16) | (0xcd14 >> 2),
 601         0x00000000,
 602         (0x0e00 << 16) | (0x88b0 >> 2),
 603         0x00000000,
 604         (0x0e00 << 16) | (0x88b4 >> 2),
 605         0x00000000,
 606         (0x0e00 << 16) | (0x88b8 >> 2),
 607         0x00000000,
 608         (0x0e00 << 16) | (0x88bc >> 2),
 609         0x00000000,
 610         (0x0400 << 16) | (0x89c0 >> 2),
 611         0x00000000,
 612         (0x0e00 << 16) | (0x88c4 >> 2),
 613         0x00000000,
 614         (0x0e00 << 16) | (0x88c8 >> 2),
 615         0x00000000,
 616         (0x0e00 << 16) | (0x88d0 >> 2),
 617         0x00000000,
 618         (0x0e00 << 16) | (0x88d4 >> 2),
 619         0x00000000,
 620         (0x0e00 << 16) | (0x88d8 >> 2),
 621         0x00000000,
 622         (0x0e00 << 16) | (0x8980 >> 2),
 623         0x00000000,
 624         (0x0e00 << 16) | (0x30938 >> 2),
 625         0x00000000,
 626         (0x0e00 << 16) | (0x3093c >> 2),
 627         0x00000000,
 628         (0x0e00 << 16) | (0x30940 >> 2),
 629         0x00000000,
 630         (0x0e00 << 16) | (0x89a0 >> 2),
 631         0x00000000,
 632         (0x0e00 << 16) | (0x30900 >> 2),
 633         0x00000000,
 634         (0x0e00 << 16) | (0x30904 >> 2),
 635         0x00000000,
 636         (0x0e00 << 16) | (0x89b4 >> 2),
 637         0x00000000,
 638         (0x0e00 << 16) | (0x3c210 >> 2),
 639         0x00000000,
 640         (0x0e00 << 16) | (0x3c214 >> 2),
 641         0x00000000,
 642         (0x0e00 << 16) | (0x3c218 >> 2),
 643         0x00000000,
 644         (0x0e00 << 16) | (0x8904 >> 2),
 645         0x00000000,
 646         0x5,
 647         (0x0e00 << 16) | (0x8c28 >> 2),
 648         (0x0e00 << 16) | (0x8c2c >> 2),
 649         (0x0e00 << 16) | (0x8c30 >> 2),
 650         (0x0e00 << 16) | (0x8c34 >> 2),
 651         (0x0e00 << 16) | (0x9600 >> 2),
 652 };
 653
 654 static const u32 kalindi_rlc_save_restore_register_list[] =
 655 {
 656         (0x0e00 << 16) | (0xc12c >> 2),
 657         0x00000000,
 658         (0x0e00 << 16) | (0xc140 >> 2),
 659         0x00000000,
 660         (0x0e00 << 16) | (0xc150 >> 2),
 661         0x00000000,
 662         (0x0e00 << 16) | (0xc15c >> 2),
 663         0x00000000,
 664         (0x0e00 << 16) | (0xc168 >> 2),
 665         0x00000000,
 666         (0x0e00 << 16) | (0xc170 >> 2),
 667         0x00000000,
 668         (0x0e00 << 16) | (0xc204 >> 2),
 669         0x00000000,
 670         (0x0e00 << 16) | (0xc2b4 >> 2),
 671         0x00000000,
 672         (0x0e00 << 16) | (0xc2b8 >> 2),
 673         0x00000000,
 674         (0x0e00 << 16) | (0xc2bc >> 2),
 675         0x00000000,
 676         (0x0e00 << 16) | (0xc2c0 >> 2),
 677         0x00000000,
 678         (0x0e00 << 16) | (0x8228 >> 2),
 679         0x00000000,
 680         (0x0e00 << 16) | (0x829c >> 2),
 681         0x00000000,
 682         (0x0e00 << 16) | (0x869c >> 2),
 683         0x00000000,
 684         (0x0600 << 16) | (0x98f4 >> 2),
 685         0x00000000,
 686         (0x0e00 << 16) | (0x98f8 >> 2),
 687         0x00000000,
 688         (0x0e00 << 16) | (0x9900 >> 2),
 689         0x00000000,
 690         (0x0e00 << 16) | (0xc260 >> 2),
 691         0x00000000,
 692         (0x0e00 << 16) | (0x90e8 >> 2),
 693         0x00000000,
 694         (0x0e00 << 16) | (0x3c000 >> 2),
 695         0x00000000,
 696         (0x0e00 << 16) | (0x3c00c >> 2),
 697         0x00000000,
 698         (0x0e00 << 16) | (0x8c1c >> 2),
 699         0x00000000,
 700         (0x0e00 << 16) | (0x9700 >> 2),
 701         0x00000000,
 702         (0x0e00 << 16) | (0xcd20 >> 2),
 703         0x00000000,
 704         (0x4e00 << 16) | (0xcd20 >> 2),
 705         0x00000000,
 706         (0x5e00 << 16) | (0xcd20 >> 2),
 707         0x00000000,
 708         (0x6e00 << 16) | (0xcd20 >> 2),
 709         0x00000000,
 710         (0x7e00 << 16) | (0xcd20 >> 2),
 711         0x00000000,
 712         (0x0e00 << 16) | (0x89bc >> 2),
 713         0x00000000,
 714         (0x0e00 << 16) | (0x8900 >> 2),
 715         0x00000000,
 716         0x3,
 717         (0x0e00 << 16) | (0xc130 >> 2),
 718         0x00000000,
 719         (0x0e00 << 16) | (0xc134 >> 2),
 720         0x00000000,
 721         (0x0e00 << 16) | (0xc1fc >> 2),
 722         0x00000000,
 723         (0x0e00 << 16) | (0xc208 >> 2),
 724         0x00000000,
 725         (0x0e00 << 16) | (0xc264 >> 2),
 726         0x00000000,
 727         (0x0e00 << 16) | (0xc268 >> 2),
 728         0x00000000,
 729         (0x0e00 << 16) | (0xc26c >> 2),
 730         0x00000000,
 731         (0x0e00 << 16) | (0xc270 >> 2),
 732         0x00000000,
 733         (0x0e00 << 16) | (0xc274 >> 2),
 734         0x00000000,
 735         (0x0e00 << 16) | (0xc28c >> 2),
 736         0x00000000,
 737         (0x0e00 << 16) | (0xc290 >> 2),
 738         0x00000000,
 739         (0x0e00 << 16) | (0xc294 >> 2),
 740         0x00000000,
 741         (0x0e00 << 16) | (0xc298 >> 2),
 742         0x00000000,
 743         (0x0e00 << 16) | (0xc2a0 >> 2),
 744         0x00000000,
 745         (0x0e00 << 16) | (0xc2a4 >> 2),
 746         0x00000000,
 747         (0x0e00 << 16) | (0xc2a8 >> 2),
 748         0x00000000,
 749         (0x0e00 << 16) | (0xc2ac >> 2),
 750         0x00000000,
 751         (0x0e00 << 16) | (0x301d0 >> 2),
 752         0x00000000,
 753         (0x0e00 << 16) | (0x30238 >> 2),
 754         0x00000000,
 755         (0x0e00 << 16) | (0x30250 >> 2),
 756         0x00000000,
 757         (0x0e00 << 16) | (0x30254 >> 2),
 758         0x00000000,
 759         (0x0e00 << 16) | (0x30258 >> 2),
 760         0x00000000,
 761         (0x0e00 << 16) | (0x3025c >> 2),
 762         0x00000000,
 763         (0x4e00 << 16) | (0xc900 >> 2),
 764         0x00000000,
 765         (0x5e00 << 16) | (0xc900 >> 2),
 766         0x00000000,
 767         (0x6e00 << 16) | (0xc900 >> 2),
 768         0x00000000,
 769         (0x7e00 << 16) | (0xc900 >> 2),
 770         0x00000000,
 771         (0x4e00 << 16) | (0xc904 >> 2),
 772         0x00000000,
 773         (0x5e00 << 16) | (0xc904 >> 2),
 774         0x00000000,
 775         (0x6e00 << 16) | (0xc904 >> 2),
 776         0x00000000,
 777         (0x7e00 << 16) | (0xc904 >> 2),
 778         0x00000000,
 779         (0x4e00 << 16) | (0xc908 >> 2),
 780         0x00000000,
 781         (0x5e00 << 16) | (0xc908 >> 2),
 782         0x00000000,
 783         (0x6e00 << 16) | (0xc908 >> 2),
 784         0x00000000,
 785         (0x7e00 << 16) | (0xc908 >> 2),
 786         0x00000000,
 787         (0x4e00 << 16) | (0xc90c >> 2),
 788         0x00000000,
 789         (0x5e00 << 16) | (0xc90c >> 2),
 790         0x00000000,
 791         (0x6e00 << 16) | (0xc90c >> 2),
 792         0x00000000,
 793         (0x7e00 << 16) | (0xc90c >> 2),
 794         0x00000000,
 795         (0x4e00 << 16) | (0xc910 >> 2),
 796         0x00000000,
 797         (0x5e00 << 16) | (0xc910 >> 2),
 798         0x00000000,
 799         (0x6e00 << 16) | (0xc910 >> 2),
 800         0x00000000,
 801         (0x7e00 << 16) | (0xc910 >> 2),
 802         0x00000000,
 803         (0x0e00 << 16) | (0xc99c >> 2),
 804         0x00000000,
 805         (0x0e00 << 16) | (0x9834 >> 2),
 806         0x00000000,
 807         (0x0000 << 16) | (0x30f00 >> 2),
 808         0x00000000,
 809         (0x0000 << 16) | (0x30f04 >> 2),
 810         0x00000000,
 811         (0x0000 << 16) | (0x30f08 >> 2),
 812         0x00000000,
 813         (0x0000 << 16) | (0x30f0c >> 2),
 814         0x00000000,
 815         (0x0600 << 16) | (0x9b7c >> 2),
 816         0x00000000,
 817         (0x0e00 << 16) | (0x8a14 >> 2),
 818         0x00000000,
 819         (0x0e00 << 16) | (0x8a18 >> 2),
 820         0x00000000,
 821         (0x0600 << 16) | (0x30a00 >> 2),
 822         0x00000000,
 823         (0x0e00 << 16) | (0x8bf0 >> 2),
 824         0x00000000,
 825         (0x0e00 << 16) | (0x8bcc >> 2),
 826         0x00000000,
 827         (0x0e00 << 16) | (0x8b24 >> 2),
 828         0x00000000,
 829         (0x0e00 << 16) | (0x30a04 >> 2),
 830         0x00000000,
 831         (0x0600 << 16) | (0x30a10 >> 2),
 832         0x00000000,
 833         (0x0600 << 16) | (0x30a14 >> 2),
 834         0x00000000,
 835         (0x0600 << 16) | (0x30a18 >> 2),
 836         0x00000000,
 837         (0x0600 << 16) | (0x30a2c >> 2),
 838         0x00000000,
 839         (0x0e00 << 16) | (0xc700 >> 2),
 840         0x00000000,
 841         (0x0e00 << 16) | (0xc704 >> 2),
 842         0x00000000,
 843         (0x0e00 << 16) | (0xc708 >> 2),
 844         0x00000000,
 845         (0x0e00 << 16) | (0xc768 >> 2),
 846         0x00000000,
 847         (0x0400 << 16) | (0xc770 >> 2),
 848         0x00000000,
 849         (0x0400 << 16) | (0xc774 >> 2),
 850         0x00000000,
 851         (0x0400 << 16) | (0xc798 >> 2),
 852         0x00000000,
 853         (0x0400 << 16) | (0xc79c >> 2),
 854         0x00000000,
 855         (0x0e00 << 16) | (0x9100 >> 2),
 856         0x00000000,
 857         (0x0e00 << 16) | (0x3c010 >> 2),
 858         0x00000000,
 859         (0x0e00 << 16) | (0x8c00 >> 2),
 860         0x00000000,
 861         (0x0e00 << 16) | (0x8c04 >> 2),
 862         0x00000000,
 863         (0x0e00 << 16) | (0x8c20 >> 2),
 864         0x00000000,
 865         (0x0e00 << 16) | (0x8c38 >> 2),
 866         0x00000000,
 867         (0x0e00 << 16) | (0x8c3c >> 2),
 868         0x00000000,
 869         (0x0e00 << 16) | (0xae00 >> 2),
 870         0x00000000,
 871         (0x0e00 << 16) | (0x9604 >> 2),
 872         0x00000000,
 873         (0x0e00 << 16) | (0xac08 >> 2),
 874         0x00000000,
 875         (0x0e00 << 16) | (0xac0c >> 2),
 876         0x00000000,
 877         (0x0e00 << 16) | (0xac10 >> 2),
 878         0x00000000,
 879         (0x0e00 << 16) | (0xac14 >> 2),
 880         0x00000000,
 881         (0x0e00 << 16) | (0xac58 >> 2),
 882         0x00000000,
 883         (0x0e00 << 16) | (0xac68 >> 2),
 884         0x00000000,
 885         (0x0e00 << 16) | (0xac6c >> 2),
 886         0x00000000,
 887         (0x0e00 << 16) | (0xac70 >> 2),
 888         0x00000000,
 889         (0x0e00 << 16) | (0xac74 >> 2),
 890         0x00000000,
 891         (0x0e00 << 16) | (0xac78 >> 2),
 892         0x00000000,
 893         (0x0e00 << 16) | (0xac7c >> 2),
 894         0x00000000,
 895         (0x0e00 << 16) | (0xac80 >> 2),
 896         0x00000000,
 897         (0x0e00 << 16) | (0xac84 >> 2),
 898         0x00000000,
 899         (0x0e00 << 16) | (0xac88 >> 2),
 900         0x00000000,
 901         (0x0e00 << 16) | (0xac8c >> 2),
 902         0x00000000,
 903         (0x0e00 << 16) | (0x970c >> 2),
 904         0x00000000,
 905         (0x0e00 << 16) | (0x9714 >> 2),
 906         0x00000000,
 907         (0x0e00 << 16) | (0x9718 >> 2),
 908         0x00000000,
 909         (0x0e00 << 16) | (0x971c >> 2),
 910         0x00000000,
 911         (0x0e00 << 16) | (0x31068 >> 2),
 912         0x00000000,
 913         (0x4e00 << 16) | (0x31068 >> 2),
 914         0x00000000,
 915         (0x5e00 << 16) | (0x31068 >> 2),
 916         0x00000000,
 917         (0x6e00 << 16) | (0x31068 >> 2),
 918         0x00000000,
 919         (0x7e00 << 16) | (0x31068 >> 2),
 920         0x00000000,
 921         (0x0e00 << 16) | (0xcd10 >> 2),
 922         0x00000000,
 923         (0x0e00 << 16) | (0xcd14 >> 2),
 924         0x00000000,
 925         (0x0e00 << 16) | (0x88b0 >> 2),
 926         0x00000000,
 927         (0x0e00 << 16) | (0x88b4 >> 2),
 928         0x00000000,
 929         (0x0e00 << 16) | (0x88b8 >> 2),
 930         0x00000000,
 931         (0x0e00 << 16) | (0x88bc >> 2),
 932         0x00000000,
 933         (0x0400 << 16) | (0x89c0 >> 2),
 934         0x00000000,
 935         (0x0e00 << 16) | (0x88c4 >> 2),
 936         0x00000000,
 937         (0x0e00 << 16) | (0x88c8 >> 2),
 938         0x00000000,
 939         (0x0e00 << 16) | (0x88d0 >> 2),
 940         0x00000000,
 941         (0x0e00 << 16) | (0x88d4 >> 2),
 942         0x00000000,
 943         (0x0e00 << 16) | (0x88d8 >> 2),
 944         0x00000000,
 945         (0x0e00 << 16) | (0x8980 >> 2),
 946         0x00000000,
 947         (0x0e00 << 16) | (0x30938 >> 2),
 948         0x00000000,
 949         (0x0e00 << 16) | (0x3093c >> 2),
 950         0x00000000,
 951         (0x0e00 << 16) | (0x30940 >> 2),
 952         0x00000000,
 953         (0x0e00 << 16) | (0x89a0 >> 2),
 954         0x00000000,
 955         (0x0e00 << 16) | (0x30900 >> 2),
 956         0x00000000,
 957         (0x0e00 << 16) | (0x30904 >> 2),
 958         0x00000000,
 959         (0x0e00 << 16) | (0x89b4 >> 2),
 960         0x00000000,
 961         (0x0e00 << 16) | (0x3e1fc >> 2),
 962         0x00000000,
 963         (0x0e00 << 16) | (0x3c210 >> 2),
 964         0x00000000,
 965         (0x0e00 << 16) | (0x3c214 >> 2),
 966         0x00000000,
 967         (0x0e00 << 16) | (0x3c218 >> 2),
 968         0x00000000,
 969         (0x0e00 << 16) | (0x8904 >> 2),
 970         0x00000000,
 971         0x5,
 972         (0x0e00 << 16) | (0x8c28 >> 2),
 973         (0x0e00 << 16) | (0x8c2c >> 2),
 974         (0x0e00 << 16) | (0x8c30 >> 2),
 975         (0x0e00 << 16) | (0x8c34 >> 2),
 976         (0x0e00 << 16) | (0x9600 >> 2),
 977 };
 978
 979 static const u32 bonaire_golden_spm_registers[] =
 980 {
 981         0x30800, 0xe0ffffff, 0xe0000000
 982 };
 983
 984 static const u32 bonaire_golden_common_registers[] =
 985 {
 986         0xc770, 0xffffffff, 0x00000800,
 987         0xc774, 0xffffffff, 0x00000800,
 988         0xc798, 0xffffffff, 0x00007fbf,
 989         0xc79c, 0xffffffff, 0x00007faf
 990 };
 991
 992 static const u32 bonaire_golden_registers[] =
 993 {
 994         0x3354, 0x00000333, 0x00000333,
 995         0x3350, 0x000c0fc0, 0x00040200,
 996         0x9a10, 0x00010000, 0x00058208,
 997         0x3c000, 0xffff1fff, 0x00140000,
 998         0x3c200, 0xfdfc0fff, 0x00000100,
 999         0x3c234, 0x40000000, 0x40000200,
1000         0x9830, 0xffffffff, 0x00000000,
1001         0x9834, 0xf00fffff, 0x00000400,
1002         0x9838, 0x0002021c, 0x00020200,
1003         0xc78, 0x00000080, 0x00000000,
1004         0x5bb0, 0x000000f0, 0x00000070,
1005         0x5bc0, 0xf0311fff, 0x80300000,
1006         0x98f8, 0x73773777, 0x12010001,
1007         0x350c, 0x00810000, 0x408af000,
1008         0x7030, 0x31000111, 0x00000011,
1009         0x2f48, 0x73773777, 0x12010001,
1010         0x220c, 0x00007fb6, 0x0021a1b1,
1011         0x2210, 0x00007fb6, 0x002021b1,
1012         0x2180, 0x00007fb6, 0x00002191,
1013         0x2218, 0x00007fb6, 0x002121b1,
1014         0x221c, 0x00007fb6, 0x002021b1,
1015         0x21dc, 0x00007fb6, 0x00002191,
1016         0x21e0, 0x00007fb6, 0x00002191,
1017         0x3628, 0x0000003f, 0x0000000a,
1018         0x362c, 0x0000003f, 0x0000000a,
1019         0x2ae4, 0x00073ffe, 0x000022a2,
1020         0x240c, 0x000007ff, 0x00000000,
1021         0x8a14, 0xf000003f, 0x00000007,
1022         0x8bf0, 0x00002001, 0x00000001,
1023         0x8b24, 0xffffffff, 0x00ffffff,
1024         0x30a04, 0x0000ff0f, 0x00000000,
1025         0x28a4c, 0x07ffffff, 0x06000000,
1026         0x4d8, 0x00000fff, 0x00000100,
1027         0x3e78, 0x00000001, 0x00000002,
1028         0x9100, 0x03000000, 0x0362c688,
1029         0x8c00, 0x000000ff, 0x00000001,
1030         0xe40, 0x00001fff, 0x00001fff,
1031         0x9060, 0x0000007f, 0x00000020,
1032         0x9508, 0x00010000, 0x00010000,
1033         0xac14, 0x000003ff, 0x000000f3,
1034         0xac0c, 0xffffffff, 0x00001032
1035 };
1036
1037 static const u32 bonaire_mgcg_cgcg_init[] =
1038 {
1039         0xc420, 0xffffffff, 0xfffffffc,
1040         0x30800, 0xffffffff, 0xe0000000,
1041         0x3c2a0, 0xffffffff, 0x00000100,
1042         0x3c208, 0xffffffff, 0x00000100,
1043         0x3c2c0, 0xffffffff, 0xc0000100,
1044         0x3c2c8, 0xffffffff, 0xc0000100,
1045         0x3c2c4, 0xffffffff, 0xc0000100,
1046         0x55e4, 0xffffffff, 0x00600100,
1047         0x3c280, 0xffffffff, 0x00000100,
1048         0x3c214, 0xffffffff, 0x06000100,
1049         0x3c220, 0xffffffff, 0x00000100,
1050         0x3c218, 0xffffffff, 0x06000100,
1051         0x3c204, 0xffffffff, 0x00000100,
1052         0x3c2e0, 0xffffffff, 0x00000100,
1053         0x3c224, 0xffffffff, 0x00000100,
1054         0x3c200, 0xffffffff, 0x00000100,
1055         0x3c230, 0xffffffff, 0x00000100,
1056         0x3c234, 0xffffffff, 0x00000100,
1057         0x3c250, 0xffffffff, 0x00000100,
1058         0x3c254, 0xffffffff, 0x00000100,
1059         0x3c258, 0xffffffff, 0x00000100,
1060         0x3c25c, 0xffffffff, 0x00000100,
1061         0x3c260, 0xffffffff, 0x00000100,
1062         0x3c27c, 0xffffffff, 0x00000100,
1063         0x3c278, 0xffffffff, 0x00000100,
1064         0x3c210, 0xffffffff, 0x06000100,
1065         0x3c290, 0xffffffff, 0x00000100,
1066         0x3c274, 0xffffffff, 0x00000100,
1067         0x3c2b4, 0xffffffff, 0x00000100,
1068         0x3c2b0, 0xffffffff, 0x00000100,
1069         0x3c270, 0xffffffff, 0x00000100,
1070         0x30800, 0xffffffff, 0xe0000000,
1071         0x3c020, 0xffffffff, 0x00010000,
1072         0x3c024, 0xffffffff, 0x00030002,
1073         0x3c028, 0xffffffff, 0x00040007,
1074         0x3c02c, 0xffffffff, 0x00060005,
1075         0x3c030, 0xffffffff, 0x00090008,
1076         0x3c034, 0xffffffff, 0x00010000,
1077         0x3c038, 0xffffffff, 0x00030002,
1078         0x3c03c, 0xffffffff, 0x00040007,
1079         0x3c040, 0xffffffff, 0x00060005,
1080         0x3c044, 0xffffffff, 0x00090008,
1081         0x3c048, 0xffffffff, 0x00010000,
1082         0x3c04c, 0xffffffff, 0x00030002,
1083         0x3c050, 0xffffffff, 0x00040007,
1084         0x3c054, 0xffffffff, 0x00060005,
1085         0x3c058, 0xffffffff, 0x00090008,
1086         0x3c05c, 0xffffffff, 0x00010000,
1087         0x3c060, 0xffffffff, 0x00030002,
1088         0x3c064, 0xffffffff, 0x00040007,
1089         0x3c068, 0xffffffff, 0x00060005,
1090         0x3c06c, 0xffffffff, 0x00090008,
1091         0x3c070, 0xffffffff, 0x00010000,
1092         0x3c074, 0xffffffff, 0x00030002,
1093         0x3c078, 0xffffffff, 0x00040007,
1094         0x3c07c, 0xffffffff, 0x00060005,
1095         0x3c080, 0xffffffff, 0x00090008,
1096         0x3c084, 0xffffffff, 0x00010000,
1097         0x3c088, 0xffffffff, 0x00030002,
1098         0x3c08c, 0xffffffff, 0x00040007,
1099         0x3c090, 0xffffffff, 0x00060005,
1100         0x3c094, 0xffffffff, 0x00090008,
1101         0x3c098, 0xffffffff, 0x00010000,
1102         0x3c09c, 0xffffffff, 0x00030002,
1103         0x3c0a0, 0xffffffff, 0x00040007,
1104         0x3c0a4, 0xffffffff, 0x00060005,
1105         0x3c0a8, 0xffffffff, 0x00090008,
1106         0x3c000, 0xffffffff, 0x96e00200,
1107         0x8708, 0xffffffff, 0x00900100,
1108         0xc424, 0xffffffff, 0x0020003f,
1109         0x38, 0xffffffff, 0x0140001c,
1110         0x3c, 0x000f0000, 0x000f0000,
1111         0x220, 0xffffffff, 0xC060000C,
1112         0x224, 0xc0000fff, 0x00000100,
1113         0xf90, 0xffffffff, 0x00000100,
1114         0xf98, 0x00000101, 0x00000000,
1115         0x20a8, 0xffffffff, 0x00000104,
1116         0x55e4, 0xff000fff, 0x00000100,
1117         0x30cc, 0xc0000fff, 0x00000104,
1118         0xc1e4, 0x00000001, 0x00000001,
1119         0xd00c, 0xff000ff0, 0x00000100,
1120         0xd80c, 0xff000ff0, 0x00000100
1121 };
1122
1123 static const u32 spectre_golden_spm_registers[] =
1124 {
1125         0x30800, 0xe0ffffff, 0xe0000000
1126 };
1127
1128 static const u32 spectre_golden_common_registers[] =
1129 {
1130         0xc770, 0xffffffff, 0x00000800,
1131         0xc774, 0xffffffff, 0x00000800,
1132         0xc798, 0xffffffff, 0x00007fbf,
1133         0xc79c, 0xffffffff, 0x00007faf
1134 };
1135
1136 static const u32 spectre_golden_registers[] =
1137 {
1138         0x3c000, 0xffff1fff, 0x96940200,
1139         0x3c00c, 0xffff0001, 0xff000000,
1140         0x3c200, 0xfffc0fff, 0x00000100,
1141         0x6ed8, 0x00010101, 0x00010000,
1142         0x9834, 0xf00fffff, 0x00000400,
1143         0x9838, 0xfffffffc, 0x00020200,
1144         0x5bb0, 0x000000f0, 0x00000070,
1145         0x5bc0, 0xf0311fff, 0x80300000,
1146         0x98f8, 0x73773777, 0x12010001,
1147         0x9b7c, 0x00ff0000, 0x00fc0000,
1148         0x2f48, 0x73773777, 0x12010001,
1149         0x8a14, 0xf000003f, 0x00000007,
1150         0x8b24, 0xffffffff, 0x00ffffff,
1151         0x28350, 0x3f3f3fff, 0x00000082,
1152         0x28354, 0x0000003f, 0x00000000,
1153         0x3e78, 0x00000001, 0x00000002,
1154         0x913c, 0xffff03df, 0x00000004,
1155         0xc768, 0x00000008, 0x00000008,
1156         0x8c00, 0x000008ff, 0x00000800,
1157         0x9508, 0x00010000, 0x00010000,
1158         0xac0c, 0xffffffff, 0x54763210,
1159         0x214f8, 0x01ff01ff, 0x00000002,
1160         0x21498, 0x007ff800, 0x00200000,
1161         0x2015c, 0xffffffff, 0x00000f40,
1162         0x30934, 0xffffffff, 0x00000001
1163 };
1164
1165 static const u32 spectre_mgcg_cgcg_init[] =
1166 {
1167         0xc420, 0xffffffff, 0xfffffffc,
1168         0x30800, 0xffffffff, 0xe0000000,
1169         0x3c2a0, 0xffffffff, 0x00000100,
1170         0x3c208, 0xffffffff, 0x00000100,
1171         0x3c2c0, 0xffffffff, 0x00000100,
1172         0x3c2c8, 0xffffffff, 0x00000100,
1173         0x3c2c4, 0xffffffff, 0x00000100,
1174         0x55e4, 0xffffffff, 0x00600100,
1175         0x3c280, 0xffffffff, 0x00000100,
1176         0x3c214, 0xffffffff, 0x06000100,
1177         0x3c220, 0xffffffff, 0x00000100,
1178         0x3c218, 0xffffffff, 0x06000100,
1179         0x3c204, 0xffffffff, 0x00000100,
1180         0x3c2e0, 0xffffffff, 0x00000100,
1181         0x3c224, 0xffffffff, 0x00000100,
1182         0x3c200, 0xffffffff, 0x00000100,
1183         0x3c230, 0xffffffff, 0x00000100,
1184         0x3c234, 0xffffffff, 0x00000100,
1185         0x3c250, 0xffffffff, 0x00000100,
1186         0x3c254, 0xffffffff, 0x00000100,
1187         0x3c258, 0xffffffff, 0x00000100,
1188         0x3c25c, 0xffffffff, 0x00000100,
1189         0x3c260, 0xffffffff, 0x00000100,
1190         0x3c27c, 0xffffffff, 0x00000100,
1191         0x3c278, 0xffffffff, 0x00000100,
1192         0x3c210, 0xffffffff, 0x06000100,
1193         0x3c290, 0xffffffff, 0x00000100,
1194         0x3c274, 0xffffffff, 0x00000100,
1195         0x3c2b4, 0xffffffff, 0x00000100,
1196         0x3c2b0, 0xffffffff, 0x00000100,
1197         0x3c270, 0xffffffff, 0x00000100,
1198         0x30800, 0xffffffff, 0xe0000000,
1199         0x3c020, 0xffffffff, 0x00010000,
1200         0x3c024, 0xffffffff, 0x00030002,
1201         0x3c028, 0xffffffff, 0x00040007,
1202         0x3c02c, 0xffffffff, 0x00060005,
1203         0x3c030, 0xffffffff, 0x00090008,
1204         0x3c034, 0xffffffff, 0x00010000,
1205         0x3c038, 0xffffffff, 0x00030002,
1206         0x3c03c, 0xffffffff, 0x00040007,
1207         0x3c040, 0xffffffff, 0x00060005,
1208         0x3c044, 0xffffffff, 0x00090008,
1209         0x3c048, 0xffffffff, 0x00010000,
1210         0x3c04c, 0xffffffff, 0x00030002,
1211         0x3c050, 0xffffffff, 0x00040007,
1212         0x3c054, 0xffffffff, 0x00060005,
1213         0x3c058, 0xffffffff, 0x00090008,
1214         0x3c05c, 0xffffffff, 0x00010000,
1215         0x3c060, 0xffffffff, 0x00030002,
1216         0x3c064, 0xffffffff, 0x00040007,
1217         0x3c068, 0xffffffff, 0x00060005,
1218         0x3c06c, 0xffffffff, 0x00090008,
1219         0x3c070, 0xffffffff, 0x00010000,
1220         0x3c074, 0xffffffff, 0x00030002,
1221         0x3c078, 0xffffffff, 0x00040007,
1222         0x3c07c, 0xffffffff, 0x00060005,
1223         0x3c080, 0xffffffff, 0x00090008,
1224         0x3c084, 0xffffffff, 0x00010000,
1225         0x3c088, 0xffffffff, 0x00030002,
1226         0x3c08c, 0xffffffff, 0x00040007,
1227         0x3c090, 0xffffffff, 0x00060005,
1228         0x3c094, 0xffffffff, 0x00090008,
1229         0x3c098, 0xffffffff, 0x00010000,
1230         0x3c09c, 0xffffffff, 0x00030002,
1231         0x3c0a0, 0xffffffff, 0x00040007,
1232         0x3c0a4, 0xffffffff, 0x00060005,
1233         0x3c0a8, 0xffffffff, 0x00090008,
1234         0x3c0ac, 0xffffffff, 0x00010000,
1235         0x3c0b0, 0xffffffff, 0x00030002,
1236         0x3c0b4, 0xffffffff, 0x00040007,
1237         0x3c0b8, 0xffffffff, 0x00060005,
1238         0x3c0bc, 0xffffffff, 0x00090008,
1239         0x3c000, 0xffffffff, 0x96e00200,
1240         0x8708, 0xffffffff, 0x00900100,
1241         0xc424, 0xffffffff, 0x0020003f,
1242         0x38, 0xffffffff, 0x0140001c,
1243         0x3c, 0x000f0000, 0x000f0000,
1244         0x220, 0xffffffff, 0xC060000C,
1245         0x224, 0xc0000fff, 0x00000100,
1246         0xf90, 0xffffffff, 0x00000100,
1247         0xf98, 0x00000101, 0x00000000,
1248         0x20a8, 0xffffffff, 0x00000104,
1249         0x55e4, 0xff000fff, 0x00000100,
1250         0x30cc, 0xc0000fff, 0x00000104,
1251         0xc1e4, 0x00000001, 0x00000001,
1252         0xd00c, 0xff000ff0, 0x00000100,
1253         0xd80c, 0xff000ff0, 0x00000100
1254 };
1255
1256 static const u32 kalindi_golden_spm_registers[] =
1257 {
1258         0x30800, 0xe0ffffff, 0xe0000000
1259 };
1260
1261 static const u32 kalindi_golden_common_registers[] =
1262 {
1263         0xc770, 0xffffffff, 0x00000800,
1264         0xc774, 0xffffffff, 0x00000800,
1265         0xc798, 0xffffffff, 0x00007fbf,
1266         0xc79c, 0xffffffff, 0x00007faf
1267 };
1268
1269 static const u32 kalindi_golden_registers[] =
1270 {
1271         0x3c000, 0xffffdfff, 0x6e944040,
1272         0x55e4, 0xff607fff, 0xfc000100,
1273         0x3c220, 0xff000fff, 0x00000100,
1274         0x3c224, 0xff000fff, 0x00000100,
1275         0x3c200, 0xfffc0fff, 0x00000100,
1276         0x6ed8, 0x00010101, 0x00010000,
1277         0x9830, 0xffffffff, 0x00000000,
1278         0x9834, 0xf00fffff, 0x00000400,
1279         0x5bb0, 0x000000f0, 0x00000070,
1280         0x5bc0, 0xf0311fff, 0x80300000,
1281         0x98f8, 0x73773777, 0x12010001,
1282         0x98fc, 0xffffffff, 0x00000010,
1283         0x9b7c, 0x00ff0000, 0x00fc0000,
1284         0x8030, 0x00001f0f, 0x0000100a,
1285         0x2f48, 0x73773777, 0x12010001,
1286         0x2408, 0x000fffff, 0x000c007f,
1287         0x8a14, 0xf000003f, 0x00000007,
1288         0x8b24, 0x3fff3fff, 0x00ffcfff,
1289         0x30a04, 0x0000ff0f, 0x00000000,
1290         0x28a4c, 0x07ffffff, 0x06000000,
1291         0x4d8, 0x00000fff, 0x00000100,
1292         0x3e78, 0x00000001, 0x00000002,
1293         0xc768, 0x00000008, 0x00000008,
1294         0x8c00, 0x000000ff, 0x00000003,
1295         0x214f8, 0x01ff01ff, 0x00000002,
1296         0x21498, 0x007ff800, 0x00200000,
1297         0x2015c, 0xffffffff, 0x00000f40,
1298         0x88c4, 0x001f3ae3, 0x00000082,
1299         0x88d4, 0x0000001f, 0x00000010,
1300         0x30934, 0xffffffff, 0x00000000
1301 };
1302
1303 static const u32 kalindi_mgcg_cgcg_init[] =
1304 {
1305         0xc420, 0xffffffff, 0xfffffffc,
1306         0x30800, 0xffffffff, 0xe0000000,
1307         0x3c2a0, 0xffffffff, 0x00000100,
1308         0x3c208, 0xffffffff, 0x00000100,
1309         0x3c2c0, 0xffffffff, 0x00000100,
1310         0x3c2c8, 0xffffffff, 0x00000100,
1311         0x3c2c4, 0xffffffff, 0x00000100,
1312         0x55e4, 0xffffffff, 0x00600100,
1313         0x3c280, 0xffffffff, 0x00000100,
1314         0x3c214, 0xffffffff, 0x06000100,
1315         0x3c220, 0xffffffff, 0x00000100,
1316         0x3c218, 0xffffffff, 0x06000100,
1317         0x3c204, 0xffffffff, 0x00000100,
1318         0x3c2e0, 0xffffffff, 0x00000100,
1319         0x3c224, 0xffffffff, 0x00000100,
1320         0x3c200, 0xffffffff, 0x00000100,
1321         0x3c230, 0xffffffff, 0x00000100,
1322         0x3c234, 0xffffffff, 0x00000100,
1323         0x3c250, 0xffffffff, 0x00000100,
1324         0x3c254, 0xffffffff, 0x00000100,
1325         0x3c258, 0xffffffff, 0x00000100,
1326         0x3c25c, 0xffffffff, 0x00000100,
1327         0x3c260, 0xffffffff, 0x00000100,
1328         0x3c27c, 0xffffffff, 0x00000100,
1329         0x3c278, 0xffffffff, 0x00000100,
1330         0x3c210, 0xffffffff, 0x06000100,
1331         0x3c290, 0xffffffff, 0x00000100,
1332         0x3c274, 0xffffffff, 0x00000100,
1333         0x3c2b4, 0xffffffff, 0x00000100,
1334         0x3c2b0, 0xffffffff, 0x00000100,
1335         0x3c270, 0xffffffff, 0x00000100,
1336         0x30800, 0xffffffff, 0xe0000000,
1337         0x3c020, 0xffffffff, 0x00010000,
1338         0x3c024, 0xffffffff, 0x00030002,
1339         0x3c028, 0xffffffff, 0x00040007,
1340         0x3c02c, 0xffffffff, 0x00060005,
1341         0x3c030, 0xffffffff, 0x00090008,
1342         0x3c034, 0xffffffff, 0x00010000,
1343         0x3c038, 0xffffffff, 0x00030002,
1344         0x3c03c, 0xffffffff, 0x00040007,
1345         0x3c040, 0xffffffff, 0x00060005,
1346         0x3c044, 0xffffffff, 0x00090008,
1347         0x3c000, 0xffffffff, 0x96e00200,
1348         0x8708, 0xffffffff, 0x00900100,
1349         0xc424, 0xffffffff, 0x0020003f,
1350         0x38, 0xffffffff, 0x0140001c,
1351         0x3c, 0x000f0000, 0x000f0000,
1352         0x220, 0xffffffff, 0xC060000C,
1353         0x224, 0xc0000fff, 0x00000100,
1354         0x20a8, 0xffffffff, 0x00000104,
1355         0x55e4, 0xff000fff, 0x00000100,
1356         0x30cc, 0xc0000fff, 0x00000104,
1357         0xc1e4, 0x00000001, 0x00000001,
1358         0xd00c, 0xff000ff0, 0x00000100,
1359         0xd80c, 0xff000ff0, 0x00000100
1360 };
1361
1362 static const u32 hawaii_golden_spm_registers[] =
1363 {
1364         0x30800, 0xe0ffffff, 0xe0000000
1365 };
1366
1367 static const u32 hawaii_golden_common_registers[] =
1368 {
1369         0x30800, 0xffffffff, 0xe0000000,
1370         0x28350, 0xffffffff, 0x3a00161a,
1371         0x28354, 0xffffffff, 0x0000002e,
1372         0x9a10, 0xffffffff, 0x00018208,
1373         0x98f8, 0xffffffff, 0x12011003
1374 };
1375
1376 static const u32 hawaii_golden_registers[] =
1377 {
1378         0x3354, 0x00000333, 0x00000333,
1379         0x9a10, 0x00010000, 0x00058208,
1380         0x9830, 0xffffffff, 0x00000000,
1381         0x9834, 0xf00fffff, 0x00000400,
1382         0x9838, 0x0002021c, 0x00020200,
1383         0xc78, 0x00000080, 0x00000000,
1384         0x5bb0, 0x000000f0, 0x00000070,
1385         0x5bc0, 0xf0311fff, 0x80300000,
1386         0x350c, 0x00810000, 0x408af000,
1387         0x7030, 0x31000111, 0x00000011,
1388         0x2f48, 0x73773777, 0x12010001,
1389         0x2120, 0x0000007f, 0x0000001b,
1390         0x21dc, 0x00007fb6, 0x00002191,
1391         0x3628, 0x0000003f, 0x0000000a,
1392         0x362c, 0x0000003f, 0x0000000a,
1393         0x2ae4, 0x00073ffe, 0x000022a2,
1394         0x240c, 0x000007ff, 0x00000000,
1395         0x8bf0, 0x00002001, 0x00000001,
1396         0x8b24, 0xffffffff, 0x00ffffff,
1397         0x30a04, 0x0000ff0f, 0x00000000,
1398         0x28a4c, 0x07ffffff, 0x06000000,
1399         0x3e78, 0x00000001, 0x00000002,
1400         0xc768, 0x00000008, 0x00000008,
1401         0xc770, 0x00000f00, 0x00000800,
1402         0xc774, 0x00000f00, 0x00000800,
1403         0xc798, 0x00ffffff, 0x00ff7fbf,
1404         0xc79c, 0x00ffffff, 0x00ff7faf,
1405         0x8c00, 0x000000ff, 0x00000800,
1406         0xe40, 0x00001fff, 0x00001fff,
1407         0x9060, 0x0000007f, 0x00000020,
1408         0x9508, 0x00010000, 0x00010000,
1409         0xae00, 0x00100000, 0x000ff07c,
1410         0xac14, 0x000003ff, 0x0000000f,
1411         0xac10, 0xffffffff, 0x7564fdec,
1412         0xac0c, 0xffffffff, 0x3120b9a8,
1413         0xac08, 0x20000000, 0x0f9c0000
1414 };
1415
1416 static const u32 hawaii_mgcg_cgcg_init[] =
1417 {
1418         0xc420, 0xffffffff, 0xfffffffd,
1419         0x30800, 0xffffffff, 0xe0000000,
1420         0x3c2a0, 0xffffffff, 0x00000100,
1421         0x3c208, 0xffffffff, 0x00000100,
1422         0x3c2c0, 0xffffffff, 0x00000100,
1423         0x3c2c8, 0xffffffff, 0x00000100,
1424         0x3c2c4, 0xffffffff, 0x00000100,
1425         0x55e4, 0xffffffff, 0x00200100,
1426         0x3c280, 0xffffffff, 0x00000100,
1427         0x3c214, 0xffffffff, 0x06000100,
1428         0x3c220, 0xffffffff, 0x00000100,
1429         0x3c218, 0xffffffff, 0x06000100,
1430         0x3c204, 0xffffffff, 0x00000100,
1431         0x3c2e0, 0xffffffff, 0x00000100,
1432         0x3c224, 0xffffffff, 0x00000100,
1433         0x3c200, 0xffffffff, 0x00000100,
1434         0x3c230, 0xffffffff, 0x00000100,
1435         0x3c234, 0xffffffff, 0x00000100,
1436         0x3c250, 0xffffffff, 0x00000100,
1437         0x3c254, 0xffffffff, 0x00000100,
1438         0x3c258, 0xffffffff, 0x00000100,
1439         0x3c25c, 0xffffffff, 0x00000100,
1440         0x3c260, 0xffffffff, 0x00000100,
1441         0x3c27c, 0xffffffff, 0x00000100,
1442         0x3c278, 0xffffffff, 0x00000100,
1443         0x3c210, 0xffffffff, 0x06000100,
1444         0x3c290, 0xffffffff, 0x00000100,
1445         0x3c274, 0xffffffff, 0x00000100,
1446         0x3c2b4, 0xffffffff, 0x00000100,
1447         0x3c2b0, 0xffffffff, 0x00000100,
1448         0x3c270, 0xffffffff, 0x00000100,
1449         0x30800, 0xffffffff, 0xe0000000,
1450         0x3c020, 0xffffffff, 0x00010000,
1451         0x3c024, 0xffffffff, 0x00030002,
1452         0x3c028, 0xffffffff, 0x00040007,
1453         0x3c02c, 0xffffffff, 0x00060005,
1454         0x3c030, 0xffffffff, 0x00090008,
1455         0x3c034, 0xffffffff, 0x00010000,
1456         0x3c038, 0xffffffff, 0x00030002,
1457         0x3c03c, 0xffffffff, 0x00040007,
1458         0x3c040, 0xffffffff, 0x00060005,
1459         0x3c044, 0xffffffff, 0x00090008,
1460         0x3c048, 0xffffffff, 0x00010000,
1461         0x3c04c, 0xffffffff, 0x00030002,
1462         0x3c050, 0xffffffff, 0x00040007,
1463         0x3c054, 0xffffffff, 0x00060005,
1464         0x3c058, 0xffffffff, 0x00090008,
1465         0x3c05c, 0xffffffff, 0x00010000,
1466         0x3c060, 0xffffffff, 0x00030002,
1467         0x3c064, 0xffffffff, 0x00040007,
1468         0x3c068, 0xffffffff, 0x00060005,
1469         0x3c06c, 0xffffffff, 0x00090008,
1470         0x3c070, 0xffffffff, 0x00010000,
1471         0x3c074, 0xffffffff, 0x00030002,
1472         0x3c078, 0xffffffff, 0x00040007,
1473         0x3c07c, 0xffffffff, 0x00060005,
1474         0x3c080, 0xffffffff, 0x00090008,
1475         0x3c084, 0xffffffff, 0x00010000,
1476         0x3c088, 0xffffffff, 0x00030002,
1477         0x3c08c, 0xffffffff, 0x00040007,
1478         0x3c090, 0xffffffff, 0x00060005,
1479         0x3c094, 0xffffffff, 0x00090008,
1480         0x3c098, 0xffffffff, 0x00010000,
1481         0x3c09c, 0xffffffff, 0x00030002,
1482         0x3c0a0, 0xffffffff, 0x00040007,
1483         0x3c0a4, 0xffffffff, 0x00060005,
1484         0x3c0a8, 0xffffffff, 0x00090008,
1485         0x3c0ac, 0xffffffff, 0x00010000,
1486         0x3c0b0, 0xffffffff, 0x00030002,
1487         0x3c0b4, 0xffffffff, 0x00040007,
1488         0x3c0b8, 0xffffffff, 0x00060005,
1489         0x3c0bc, 0xffffffff, 0x00090008,
1490         0x3c0c0, 0xffffffff, 0x00010000,
1491         0x3c0c4, 0xffffffff, 0x00030002,
1492         0x3c0c8, 0xffffffff, 0x00040007,
1493         0x3c0cc, 0xffffffff, 0x00060005,
1494         0x3c0d0, 0xffffffff, 0x00090008,
1495         0x3c0d4, 0xffffffff, 0x00010000,
1496         0x3c0d8, 0xffffffff, 0x00030002,
1497         0x3c0dc, 0xffffffff, 0x00040007,
1498         0x3c0e0, 0xffffffff, 0x00060005,
1499         0x3c0e4, 0xffffffff, 0x00090008,
1500         0x3c0e8, 0xffffffff, 0x00010000,
1501         0x3c0ec, 0xffffffff, 0x00030002,
1502         0x3c0f0, 0xffffffff, 0x00040007,
1503         0x3c0f4, 0xffffffff, 0x00060005,
1504         0x3c0f8, 0xffffffff, 0x00090008,
1505         0xc318, 0xffffffff, 0x00020200,
1506         0x3350, 0xffffffff, 0x00000200,
1507         0x15c0, 0xffffffff, 0x00000400,
1508         0x55e8, 0xffffffff, 0x00000000,
1509         0x2f50, 0xffffffff, 0x00000902,
1510         0x3c000, 0xffffffff, 0x96940200,
1511         0x8708, 0xffffffff, 0x00900100,
1512         0xc424, 0xffffffff, 0x0020003f,
1513         0x38, 0xffffffff, 0x0140001c,
1514         0x3c, 0x000f0000, 0x000f0000,
1515         0x220, 0xffffffff, 0xc060000c,
1516         0x224, 0xc0000fff, 0x00000100,
1517         0xf90, 0xffffffff, 0x00000100,
1518         0xf98, 0x00000101, 0x00000000,
1519         0x20a8, 0xffffffff, 0x00000104,
1520         0x55e4, 0xff000fff, 0x00000100,
1521         0x30cc, 0xc0000fff, 0x00000104,
1522         0xc1e4, 0x00000001, 0x00000001,
1523         0xd00c, 0xff000ff0, 0x00000100,
1524         0xd80c, 0xff000ff0, 0x00000100
1525 };
1526
1527 static const u32 godavari_golden_registers[] =
1528 {
1529         0x55e4, 0xff607fff, 0xfc000100,
1530         0x6ed8, 0x00010101, 0x00010000,
1531         0x9830, 0xffffffff, 0x00000000,
1532         0x98302, 0xf00fffff, 0x00000400,
1533         0x6130, 0xffffffff, 0x00010000,
1534         0x5bb0, 0x000000f0, 0x00000070,
1535         0x5bc0, 0xf0311fff, 0x80300000,
1536         0x98f8, 0x73773777, 0x12010001,
1537         0x98fc, 0xffffffff, 0x00000010,
1538         0x8030, 0x00001f0f, 0x0000100a,
1539         0x2f48, 0x73773777, 0x12010001,
1540         0x2408, 0x000fffff, 0x000c007f,
1541         0x8a14, 0xf000003f, 0x00000007,
1542         0x8b24, 0xffffffff, 0x00ff0fff,
1543         0x30a04, 0x0000ff0f, 0x00000000,
1544         0x28a4c, 0x07ffffff, 0x06000000,
1545         0x4d8, 0x00000fff, 0x00000100,
1546         0xd014, 0x00010000, 0x00810001,
1547         0xd814, 0x00010000, 0x00810001,
1548         0x3e78, 0x00000001, 0x00000002,
1549         0xc768, 0x00000008, 0x00000008,
1550         0xc770, 0x00000f00, 0x00000800,
1551         0xc774, 0x00000f00, 0x00000800,
1552         0xc798, 0x00ffffff, 0x00ff7fbf,
1553         0xc79c, 0x00ffffff, 0x00ff7faf,
1554         0x8c00, 0x000000ff, 0x00000001,
1555         0x214f8, 0x01ff01ff, 0x00000002,
1556         0x21498, 0x007ff800, 0x00200000,
1557         0x2015c, 0xffffffff, 0x00000f40,
1558         0x88c4, 0x001f3ae3, 0x00000082,
1559         0x88d4, 0x0000001f, 0x00000010,
1560         0x30934, 0xffffffff, 0x00000000
1561 };
1562
1563
1564 static void cik_init_golden_registers(struct radeon_device *rdev)
1565 {
1566         switch (rdev->family) {
1567         case CHIP_BONAIRE:
1568                 radeon_program_register_sequence(rdev,
1569                                                  bonaire_mgcg_cgcg_init,
1570                                                  (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
1571                 radeon_program_register_sequence(rdev,
1572                                                  bonaire_golden_registers,
1573                                                  (const u32)ARRAY_SIZE(bonaire_golden_registers));
1574                 radeon_program_register_sequence(rdev,
1575                                                  bonaire_golden_common_registers,
1576                                                  (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
1577                 radeon_program_register_sequence(rdev,
1578                                                  bonaire_golden_spm_registers,
1579                                                  (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
1580                 break;
1581         case CHIP_KABINI:
1582                 radeon_program_register_sequence(rdev,
1583                                                  kalindi_mgcg_cgcg_init,
1584                                                  (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1585                 radeon_program_register_sequence(rdev,
1586                                                  kalindi_golden_registers,
1587                                                  (const u32)ARRAY_SIZE(kalindi_golden_registers));
1588                 radeon_program_register_sequence(rdev,
1589                                                  kalindi_golden_common_registers,
1590                                                  (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1591                 radeon_program_register_sequence(rdev,
1592                                                  kalindi_golden_spm_registers,
1593                                                  (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1594                 break;
1595         case CHIP_MULLINS:
1596                 radeon_program_register_sequence(rdev,
1597                                                  kalindi_mgcg_cgcg_init,
1598                                                  (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1599                 radeon_program_register_sequence(rdev,
1600                                                  godavari_golden_registers,
1601                                                  (const u32)ARRAY_SIZE(godavari_golden_registers));
1602                 radeon_program_register_sequence(rdev,
1603                                                  kalindi_golden_common_registers,
1604                                                  (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1605                 radeon_program_register_sequence(rdev,
1606                                                  kalindi_golden_spm_registers,
1607                                                  (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1608                 break;
1609         case CHIP_KAVERI:
1610                 radeon_program_register_sequence(rdev,
1611                                                  spectre_mgcg_cgcg_init,
1612                                                  (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
1613                 radeon_program_register_sequence(rdev,
1614                                                  spectre_golden_registers,
1615                                                  (const u32)ARRAY_SIZE(spectre_golden_registers));
1616                 radeon_program_register_sequence(rdev,
1617                                                  spectre_golden_common_registers,
1618                                                  (const u32)ARRAY_SIZE(spectre_golden_common_registers));
1619                 radeon_program_register_sequence(rdev,
1620                                                  spectre_golden_spm_registers,
1621                                                  (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
1622                 break;
1623         case CHIP_HAWAII:
1624                 radeon_program_register_sequence(rdev,
1625                                                  hawaii_mgcg_cgcg_init,
1626                                                  (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
1627                 radeon_program_register_sequence(rdev,
1628                                                  hawaii_golden_registers,
1629                                                  (const u32)ARRAY_SIZE(hawaii_golden_registers));
1630                 radeon_program_register_sequence(rdev,
1631                                                  hawaii_golden_common_registers,
1632                                                  (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
1633                 radeon_program_register_sequence(rdev,
1634                                                  hawaii_golden_spm_registers,
1635                                                  (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
1636                 break;
1637         default:
1638                 break;
1639         }
1640 }
1641
1642 /**
1643  * cik_get_xclk - get the xclk
1644  *
1645  * @rdev: radeon_device pointer
1646  *
1647  * Returns the reference clock used by the gfx engine
1648  * (CIK).
1649  */
1650 u32 cik_get_xclk(struct radeon_device *rdev)
1651 {
1652         u32 reference_clock = rdev->clock.spll.reference_freq;
1653
1654         if (rdev->flags & RADEON_IS_IGP) {
1655                 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
1656                         return reference_clock / 2;
1657         } else {
1658                 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
1659                         return reference_clock / 4;
1660         }
1661         return reference_clock;
1662 }
1663
1664 /**
1665  * cik_mm_rdoorbell - read a doorbell dword
1666  *
1667  * @rdev: radeon_device pointer
1668  * @index: doorbell index
1669  *
1670  * Returns the value in the doorbell aperture at the
1671  * requested doorbell index (CIK).
1672  */
1673 u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index)
1674 {
1675         if (index < rdev->doorbell.num_doorbells) {
1676                 return readl(rdev->doorbell.ptr + index);
1677         } else {
1678                 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
1679                 return 0;
1680         }
1681 }
1682
1683 /**
1684  * cik_mm_wdoorbell - write a doorbell dword
1685  *
1686  * @rdev: radeon_device pointer
1687  * @index: doorbell index
1688  * @v: value to write
1689  *
1690  * Writes @v to the doorbell aperture at the
1691  * requested doorbell index (CIK).
1692  */
1693 void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v)
1694 {
1695         if (index < rdev->doorbell.num_doorbells) {
1696                 writel(v, rdev->doorbell.ptr + index);
1697         } else {
1698                 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
1699         }
1700 }
1701
1702 #define BONAIRE_IO_MC_REGS_SIZE 36
1703
1704 static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
1705 {
1706         {0x00000070, 0x04400000},
1707         {0x00000071, 0x80c01803},
1708         {0x00000072, 0x00004004},
1709         {0x00000073, 0x00000100},
1710         {0x00000074, 0x00ff0000},
1711         {0x00000075, 0x34000000},
1712         {0x00000076, 0x08000014},
1713         {0x00000077, 0x00cc08ec},
1714         {0x00000078, 0x00000400},
1715         {0x00000079, 0x00000000},
1716         {0x0000007a, 0x04090000},
1717         {0x0000007c, 0x00000000},
1718         {0x0000007e, 0x4408a8e8},
1719         {0x0000007f, 0x00000304},
1720         {0x00000080, 0x00000000},
1721         {0x00000082, 0x00000001},
1722         {0x00000083, 0x00000002},
1723         {0x00000084, 0xf3e4f400},
1724         {0x00000085, 0x052024e3},
1725         {0x00000087, 0x00000000},
1726         {0x00000088, 0x01000000},
1727         {0x0000008a, 0x1c0a0000},
1728         {0x0000008b, 0xff010000},
1729         {0x0000008d, 0xffffefff},
1730         {0x0000008e, 0xfff3efff},
1731         {0x0000008f, 0xfff3efbf},
1732         {0x00000092, 0xf7ffffff},
1733         {0x00000093, 0xffffff7f},
1734         {0x00000095, 0x00101101},
1735         {0x00000096, 0x00000fff},
1736         {0x00000097, 0x00116fff},
1737         {0x00000098, 0x60010000},
1738         {0x00000099, 0x10010000},
1739         {0x0000009a, 0x00006000},
1740         {0x0000009b, 0x00001000},
1741         {0x0000009f, 0x00b48000}
1742 };
1743
1744 #define HAWAII_IO_MC_REGS_SIZE 22
1745
1746 static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] =
1747 {
1748         {0x0000007d, 0x40000000},
1749         {0x0000007e, 0x40180304},
1750         {0x0000007f, 0x0000ff00},
1751         {0x00000081, 0x00000000},
1752         {0x00000083, 0x00000800},
1753         {0x00000086, 0x00000000},
1754         {0x00000087, 0x00000100},
1755         {0x00000088, 0x00020100},
1756         {0x00000089, 0x00000000},
1757         {0x0000008b, 0x00040000},
1758         {0x0000008c, 0x00000100},
1759         {0x0000008e, 0xff010000},
1760         {0x00000090, 0xffffefff},
1761         {0x00000091, 0xfff3efff},
1762         {0x00000092, 0xfff3efbf},
1763         {0x00000093, 0xf7ffffff},
1764         {0x00000094, 0xffffff7f},
1765         {0x00000095, 0x00000fff},
1766         {0x00000096, 0x00116fff},
1767         {0x00000097, 0x60010000},
1768         {0x00000098, 0x10010000},
1769         {0x0000009f, 0x00c79000}
1770 };
1771
1772
1773 /**
1774  * cik_srbm_select - select specific register instances
1775  *
1776  * @rdev: radeon_device pointer
1777  * @me: selected ME (micro engine)
1778  * @pipe: pipe
1779  * @queue: queue
1780  * @vmid: VMID
1781  *
1782  * Switches the currently active registers instances.  Some
1783  * registers are instanced per VMID, others are instanced per
1784  * me/pipe/queue combination.
1785  */
1786 static void cik_srbm_select(struct radeon_device *rdev,
1787                             u32 me, u32 pipe, u32 queue, u32 vmid)
1788 {
1789         u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
1790                              MEID(me & 0x3) |
1791                              VMID(vmid & 0xf) |
1792                              QUEUEID(queue & 0x7));
1793         WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
1794 }
1795
1796 /* ucode loading */
1797 /**
1798  * ci_mc_load_microcode - load MC ucode into the hw
1799  *
1800  * @rdev: radeon_device pointer
1801  *
1802  * Load the GDDR MC ucode into the hw (CIK).
1803  * Returns 0 on success, error on failure.
1804  */
1805 int ci_mc_load_microcode(struct radeon_device *rdev)
1806 {
1807         const __be32 *fw_data = NULL;
1808         const __le32 *new_fw_data = NULL;
1809         u32 running, blackout = 0;
1810         u32 *io_mc_regs = NULL;
1811         const __le32 *new_io_mc_regs = NULL;
1812         int i, regs_size, ucode_size;
1813
1814         if (!rdev->mc_fw)
1815                 return -EINVAL;
1816
1817         if (rdev->new_fw) {
1818                 const struct mc_firmware_header_v1_0 *hdr =
1819                         (const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data;
1820
1821                 radeon_ucode_print_mc_hdr(&hdr->header);
1822
1823                 regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
1824                 new_io_mc_regs = (const __le32 *)
1825                         (rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
1826                 ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
1827                 new_fw_data = (const __le32 *)
1828                         (rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1829         } else {
1830                 ucode_size = rdev->mc_fw->size / 4;
1831
1832                 switch (rdev->family) {
1833                 case CHIP_BONAIRE:
1834                         io_mc_regs = (u32 *)&bonaire_io_mc_regs;
1835                         regs_size = BONAIRE_IO_MC_REGS_SIZE;
1836                         break;
1837                 case CHIP_HAWAII:
1838                         io_mc_regs = (u32 *)&hawaii_io_mc_regs;
1839                         regs_size = HAWAII_IO_MC_REGS_SIZE;
1840                         break;
1841                 default:
1842                         return -EINVAL;
1843                 }
1844                 fw_data = (const __be32 *)rdev->mc_fw->data;
1845         }
1846
1847         running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1848
1849         if (running == 0) {
1850                 if (running) {
1851                         blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1852                         WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1853                 }
1854
1855                 /* reset the engine and set to writable */
1856                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1857                 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1858
1859                 /* load mc io regs */
1860                 for (i = 0; i < regs_size; i++) {
1861                         if (rdev->new_fw) {
1862                                 WREG32(MC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++));
1863                                 WREG32(MC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++));
1864                         } else {
1865                                 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1866                                 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1867                         }
1868                 }
1869                 /* load the MC ucode */
1870                 for (i = 0; i < ucode_size; i++) {
1871                         if (rdev->new_fw)
1872                                 WREG32(MC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++));
1873                         else
1874                                 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1875                 }
1876
1877                 /* put the engine back into the active state */
1878                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1879                 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1880                 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1881
1882                 /* wait for training to complete */
1883                 for (i = 0; i < rdev->usec_timeout; i++) {
1884                         if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1885                                 break;
1886                         udelay(1);
1887                 }
1888                 for (i = 0; i < rdev->usec_timeout; i++) {
1889                         if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1890                                 break;
1891                         udelay(1);
1892                 }
1893
1894                 if (running)
1895                         WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1896         }
1897
1898         return 0;
1899 }
1900
1901 /**
1902  * cik_init_microcode - load ucode images from disk
1903  *
1904  * @rdev: radeon_device pointer
1905  *
1906  * Use the firmware interface to load the ucode images into
1907  * the driver (not loaded into hw).
1908  * Returns 0 on success, error on failure.
1909  */
1910 static int cik_init_microcode(struct radeon_device *rdev)
1911 {
1912         const char *chip_name;
1913         const char *new_chip_name;
1914         size_t pfp_req_size, me_req_size, ce_req_size,
1915                 mec_req_size, rlc_req_size, mc_req_size = 0,
1916                 sdma_req_size, smc_req_size = 0, mc2_req_size = 0;
1917         char fw_name[30];
1918         int new_fw = 0;
1919         int err;
1920         int num_fw;
1921
1922         DRM_DEBUG("\n");
1923
1924         switch (rdev->family) {
1925         case CHIP_BONAIRE:
1926                 chip_name = "BONAIRE";
1927                 new_chip_name = "bonaire";
1928                 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1929                 me_req_size = CIK_ME_UCODE_SIZE * 4;
1930                 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1931                 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1932                 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1933                 mc_req_size = BONAIRE_MC_UCODE_SIZE * 4;
1934                 mc2_req_size = BONAIRE_MC2_UCODE_SIZE * 4;
1935                 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1936                 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
1937                 num_fw = 8;
1938                 break;
1939         case CHIP_HAWAII:
1940                 chip_name = "HAWAII";
1941                 new_chip_name = "hawaii";
1942                 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1943                 me_req_size = CIK_ME_UCODE_SIZE * 4;
1944                 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1945                 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1946                 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1947                 mc_req_size = HAWAII_MC_UCODE_SIZE * 4;
1948                 mc2_req_size = HAWAII_MC2_UCODE_SIZE * 4;
1949                 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1950                 smc_req_size = ALIGN(HAWAII_SMC_UCODE_SIZE, 4);
1951                 num_fw = 8;
1952                 break;
1953         case CHIP_KAVERI:
1954                 chip_name = "KAVERI";
1955                 new_chip_name = "kaveri";
1956                 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1957                 me_req_size = CIK_ME_UCODE_SIZE * 4;
1958                 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1959                 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1960                 rlc_req_size = KV_RLC_UCODE_SIZE * 4;
1961                 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1962                 num_fw = 7;
1963                 break;
1964         case CHIP_KABINI:
1965                 chip_name = "KABINI";
1966                 new_chip_name = "kabini";
1967                 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1968                 me_req_size = CIK_ME_UCODE_SIZE * 4;
1969                 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1970                 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1971                 rlc_req_size = KB_RLC_UCODE_SIZE * 4;
1972                 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1973                 num_fw = 6;
1974                 break;
1975         case CHIP_MULLINS:
1976                 chip_name = "MULLINS";
1977                 new_chip_name = "mullins";
1978                 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1979                 me_req_size = CIK_ME_UCODE_SIZE * 4;
1980                 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1981                 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1982                 rlc_req_size = ML_RLC_UCODE_SIZE * 4;
1983                 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1984                 num_fw = 6;
1985                 break;
1986         default: BUG();
1987         }
1988
1989         DRM_INFO("Loading %s Microcode\n", new_chip_name);
1990
1991         snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", new_chip_name);
1992         err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1993         if (err) {
1994                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1995                 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1996                 if (err)
1997                         goto out;
1998                 if (rdev->pfp_fw->size != pfp_req_size) {
1999                         printk(KERN_ERR
2000                                "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2001                                rdev->pfp_fw->size, fw_name);
2002                         err = -EINVAL;
2003                         goto out;
2004                 }
2005         } else {
2006                 err = radeon_ucode_validate(rdev->pfp_fw);
2007                 if (err) {
2008                         printk(KERN_ERR
2009                                "cik_fw: validation failed for firmware \"%s\"\n",
2010                                fw_name);
2011                         goto out;
2012                 } else {
2013                         new_fw++;
2014                 }
2015         }
2016
2017         snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", new_chip_name);
2018         err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
2019         if (err) {
2020                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
2021                 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
2022                 if (err)
2023                         goto out;
2024                 if (rdev->me_fw->size != me_req_size) {
2025                         printk(KERN_ERR
2026                                "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2027                                rdev->me_fw->size, fw_name);
2028                         err = -EINVAL;
2029                 }
2030         } else {
2031                 err = radeon_ucode_validate(rdev->me_fw);
2032                 if (err) {
2033                         printk(KERN_ERR
2034                                "cik_fw: validation failed for firmware \"%s\"\n",
2035                                fw_name);
2036                         goto out;
2037                 } else {
2038                         new_fw++;
2039                 }
2040         }
2041
2042         snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", new_chip_name);
2043         err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
2044         if (err) {
2045                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
2046                 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
2047                 if (err)
2048                         goto out;
2049                 if (rdev->ce_fw->size != ce_req_size) {
2050                         printk(KERN_ERR
2051                                "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2052                                rdev->ce_fw->size, fw_name);
2053                         err = -EINVAL;
2054                 }
2055         } else {
2056                 err = radeon_ucode_validate(rdev->ce_fw);
2057                 if (err) {
2058                         printk(KERN_ERR
2059                                "cik_fw: validation failed for firmware \"%s\"\n",
2060                                fw_name);
2061                         goto out;
2062                 } else {
2063                         new_fw++;
2064                 }
2065         }
2066
2067         snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", new_chip_name);
2068         err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
2069         if (err) {
2070                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
2071                 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
2072                 if (err)
2073                         goto out;
2074                 if (rdev->mec_fw->size != mec_req_size) {
2075                         printk(KERN_ERR
2076                                "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2077                                rdev->mec_fw->size, fw_name);
2078                         err = -EINVAL;
2079                 }
2080         } else {
2081                 err = radeon_ucode_validate(rdev->mec_fw);
2082                 if (err) {
2083                         printk(KERN_ERR
2084                                "cik_fw: validation failed for firmware \"%s\"\n",
2085                                fw_name);
2086                         goto out;
2087                 } else {
2088                         new_fw++;
2089                 }
2090         }
2091
2092         if (rdev->family == CHIP_KAVERI) {
2093                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec2.bin", new_chip_name);
2094                 err = request_firmware(&rdev->mec2_fw, fw_name, rdev->dev);
2095                 if (err) {
2096                         goto out;
2097                 } else {
2098                         err = radeon_ucode_validate(rdev->mec2_fw);
2099                         if (err) {
2100                                 goto out;
2101                         } else {
2102                                 new_fw++;
2103                         }
2104                 }
2105         }
2106
2107         snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", new_chip_name);
2108         err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
2109         if (err) {
2110                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
2111                 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
2112                 if (err)
2113                         goto out;
2114                 if (rdev->rlc_fw->size != rlc_req_size) {
2115                         printk(KERN_ERR
2116                                "cik_rlc: Bogus length %zu in firmware \"%s\"\n",
2117                                rdev->rlc_fw->size, fw_name);
2118                         err = -EINVAL;
2119                 }
2120         } else {
2121                 err = radeon_ucode_validate(rdev->rlc_fw);
2122                 if (err) {
2123                         printk(KERN_ERR
2124                                "cik_fw: validation failed for firmware \"%s\"\n",
2125                                fw_name);
2126                         goto out;
2127                 } else {
2128                         new_fw++;
2129                 }
2130         }
2131
2132         snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", new_chip_name);
2133         err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
2134         if (err) {
2135                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
2136                 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
2137                 if (err)
2138                         goto out;
2139                 if (rdev->sdma_fw->size != sdma_req_size) {
2140                         printk(KERN_ERR
2141                                "cik_sdma: Bogus length %zu in firmware \"%s\"\n",
2142                                rdev->sdma_fw->size, fw_name);
2143                         err = -EINVAL;
2144                 }
2145         } else {
2146                 err = radeon_ucode_validate(rdev->sdma_fw);
2147                 if (err) {
2148                         printk(KERN_ERR
2149                                "cik_fw: validation failed for firmware \"%s\"\n",
2150                                fw_name);
2151                         goto out;
2152                 } else {
2153                         new_fw++;
2154                 }
2155         }
2156
2157         /* No SMC, MC ucode on APUs */
2158         if (!(rdev->flags & RADEON_IS_IGP)) {
2159                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", new_chip_name);
2160                 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2161                 if (err) {
2162                         snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name);
2163                         err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2164                         if (err) {
2165                                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
2166                                 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2167                                 if (err)
2168                                         goto out;
2169                         }
2170                         if ((rdev->mc_fw->size != mc_req_size) &&
2171                             (rdev->mc_fw->size != mc2_req_size)){
2172                                 printk(KERN_ERR
2173                                        "cik_mc: Bogus length %zu in firmware \"%s\"\n",
2174                                        rdev->mc_fw->size, fw_name);
2175                                 err = -EINVAL;
2176                         }
2177                         DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);
2178                 } else {
2179                         err = radeon_ucode_validate(rdev->mc_fw);
2180                         if (err) {
2181                                 printk(KERN_ERR
2182                                        "cik_fw: validation failed for firmware \"%s\"\n",
2183                                        fw_name);
2184                                 goto out;
2185                         } else {
2186                                 new_fw++;
2187                         }
2188                 }
2189
2190                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", new_chip_name);
2191                 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
2192                 if (err) {
2193                         snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
2194                         err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
2195                         if (err) {
2196                                 printk(KERN_ERR
2197                                        "smc: error loading firmware \"%s\"\n",
2198                                        fw_name);
2199                                 release_firmware(rdev->smc_fw);
2200                                 rdev->smc_fw = NULL;
2201                                 err = 0;
2202                         } else if (rdev->smc_fw->size != smc_req_size) {
2203                                 printk(KERN_ERR
2204                                        "cik_smc: Bogus length %zu in firmware \"%s\"\n",
2205                                        rdev->smc_fw->size, fw_name);
2206                                 err = -EINVAL;
2207                         }
2208                 } else {
2209                         err = radeon_ucode_validate(rdev->smc_fw);
2210                         if (err) {
2211                                 printk(KERN_ERR
2212                                        "cik_fw: validation failed for firmware \"%s\"\n",
2213                                        fw_name);
2214                                 goto out;
2215                         } else {
2216                                 new_fw++;
2217                         }
2218                 }
2219         }
2220
2221         if (new_fw == 0) {
2222                 rdev->new_fw = false;
2223         } else if (new_fw < num_fw) {
2224                 printk(KERN_ERR "ci_fw: mixing new and old firmware!\n");
2225                 err = -EINVAL;
2226         } else {
2227                 rdev->new_fw = true;
2228         }
2229
2230 out:
2231         if (err) {
2232                 if (err != -EINVAL)
2233                         printk(KERN_ERR
2234                                "cik_cp: Failed to load firmware \"%s\"\n",
2235                                fw_name);
2236                 release_firmware(rdev->pfp_fw);
2237                 rdev->pfp_fw = NULL;
2238                 release_firmware(rdev->me_fw);
2239                 rdev->me_fw = NULL;
2240                 release_firmware(rdev->ce_fw);
2241                 rdev->ce_fw = NULL;
2242                 release_firmware(rdev->mec_fw);
2243                 rdev->mec_fw = NULL;
2244                 release_firmware(rdev->mec2_fw);
2245                 rdev->mec2_fw = NULL;
2246                 release_firmware(rdev->rlc_fw);
2247                 rdev->rlc_fw = NULL;
2248                 release_firmware(rdev->sdma_fw);
2249                 rdev->sdma_fw = NULL;
2250                 release_firmware(rdev->mc_fw);
2251                 rdev->mc_fw = NULL;
2252                 release_firmware(rdev->smc_fw);
2253                 rdev->smc_fw = NULL;
2254         }
2255         return err;
2256 }
2257
2258 /*
2259  * Core functions
2260  */
2261 /**
2262  * cik_tiling_mode_table_init - init the hw tiling table
2263  *
2264  * @rdev: radeon_device pointer
2265  *
2266  * Starting with SI, the tiling setup is done globally in a
2267  * set of 32 tiling modes.  Rather than selecting each set of
2268  * parameters per surface as on older asics, we just select
2269  * which index in the tiling table we want to use, and the
2270  * surface uses those parameters (CIK).
2271  */
2272 static void cik_tiling_mode_table_init(struct radeon_device *rdev)
2273 {
2274         const u32 num_tile_mode_states = 32;
2275         const u32 num_secondary_tile_mode_states = 16;
2276         u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
2277         u32 num_pipe_configs;
2278         u32 num_rbs = rdev->config.cik.max_backends_per_se *
2279                 rdev->config.cik.max_shader_engines;
2280
2281         switch (rdev->config.cik.mem_row_size_in_kb) {
2282         case 1:
2283                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2284                 break;
2285         case 2:
2286         default:
2287                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2288                 break;
2289         case 4:
2290                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2291                 break;
2292         }
2293
2294         num_pipe_configs = rdev->config.cik.max_tile_pipes;
2295         if (num_pipe_configs > 8)
2296                 num_pipe_configs = 16;
2297
2298         if (num_pipe_configs == 16) {
2299                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2300                         switch (reg_offset) {
2301                         case 0:
2302                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2303                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2304                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2305                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2306                                 break;
2307                         case 1:
2308                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2309                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2310                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2311                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2312                                 break;
2313                         case 2:
2314                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2315                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2316                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2317                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2318                                 break;
2319                         case 3:
2320                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2321                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2322                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2323                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2324                                 break;
2325                         case 4:
2326                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2327                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2328                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2329                                                  TILE_SPLIT(split_equal_to_row_size));
2330                                 break;
2331                         case 5:
2332                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2333                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2334                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2335                                 break;
2336                         case 6:
2337                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2338                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2339                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2340                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2341                                 break;
2342                         case 7:
2343                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2344                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2345                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2346                                                  TILE_SPLIT(split_equal_to_row_size));
2347                                 break;
2348                         case 8:
2349                                 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2350                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
2351                                 break;
2352                         case 9:
2353                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2354                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2355                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2356                                 break;
2357                         case 10:
2358                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2359                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2360                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2361                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2362                                 break;
2363                         case 11:
2364                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2365                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2366                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2367                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2368                                 break;
2369                         case 12:
2370                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2371                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2372                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2373                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2374                                 break;
2375                         case 13:
2376                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2377                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2378                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2379                                 break;
2380                         case 14:
2381                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2382                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2383                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2384                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2385                                 break;
2386                         case 16:
2387                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2388                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2389                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2390                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2391                                 break;
2392                         case 17:
2393                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2394                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2395                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2396                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2397                                 break;
2398                         case 27:
2399                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2400                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2401                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2402                                 break;
2403                         case 28:
2404                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2405                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2406                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2407                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2408                                 break;
2409                         case 29:
2410                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2411                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2412                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2413                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2414                                 break;
2415                         case 30:
2416                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2417                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2418                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2419                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2420                                 break;
2421                         default:
2422                                 gb_tile_moden = 0;
2423                                 break;
2424                         }
2425                         rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2426                         WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2427                 }
2428                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2429                         switch (reg_offset) {
2430                         case 0:
2431                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2432                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2433                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2434                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2435                                 break;
2436                         case 1:
2437                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2438                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2439                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2440                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2441                                 break;
2442                         case 2:
2443                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2444                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2445                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2446                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2447                                 break;
2448                         case 3:
2449                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2450                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2451                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2452                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2453                                 break;
2454                         case 4:
2455                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2456                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2457                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2458                                                  NUM_BANKS(ADDR_SURF_8_BANK));
2459                                 break;
2460                         case 5:
2461                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2462                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2463                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2464                                                  NUM_BANKS(ADDR_SURF_4_BANK));
2465                                 break;
2466                         case 6:
2467                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2468                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2469                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2470                                                  NUM_BANKS(ADDR_SURF_2_BANK));
2471                                 break;
2472                         case 8:
2473                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2474                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2475                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2476                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2477                                 break;
2478                         case 9:
2479                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2480                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2481                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2482                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2483                                 break;
2484                         case 10:
2485                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2486                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2487                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2488                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2489                                 break;
2490                         case 11:
2491                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2492                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2493                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2494                                                  NUM_BANKS(ADDR_SURF_8_BANK));
2495                                 break;
2496                         case 12:
2497                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2498                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2499                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2500                                                  NUM_BANKS(ADDR_SURF_4_BANK));
2501                                 break;
2502                         case 13:
2503                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2504                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2505                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2506                                                  NUM_BANKS(ADDR_SURF_2_BANK));
2507                                 break;
2508                         case 14:
2509                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2510                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2511                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2512                                                  NUM_BANKS(ADDR_SURF_2_BANK));
2513                                 break;
2514                         default:
2515                                 gb_tile_moden = 0;
2516                                 break;
2517                         }
2518                         rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
2519                         WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2520                 }
2521         } else if (num_pipe_configs == 8) {
2522                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2523                         switch (reg_offset) {
2524                         case 0:
2525                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2526                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2527                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2528                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2529                                 break;
2530                         case 1:
2531                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2532                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2533                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2534                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2535                                 break;
2536                         case 2:
2537                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2538                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2539                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2540                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2541                                 break;
2542                         case 3:
2543                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2544                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2545                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2546                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2547                                 break;
2548                         case 4:
2549                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2550                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2551                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2552                                                  TILE_SPLIT(split_equal_to_row_size));
2553                                 break;
2554                         case 5:
2555                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2556                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2557                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2558                                 break;
2559                         case 6:
2560                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2561                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2562                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2563                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2564                                 break;
2565                         case 7:
2566                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2567                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2568                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2569                                                  TILE_SPLIT(split_equal_to_row_size));
2570                                 break;
2571                         case 8:
2572                                 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2573                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2574                                 break;
2575                         case 9:
2576                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2577                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2578                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2579                                 break;
2580                         case 10:
2581                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2582                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2583                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2584                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2585                                 break;
2586                         case 11:
2587                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2588                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2589                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2590                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2591                                 break;
2592                         case 12:
2593                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2594                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2595                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2596                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2597                                 break;
2598                         case 13:
2599                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2600                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2601                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2602                                 break;
2603                         case 14:
2604                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2605                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2606                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2607                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2608                                 break;
2609                         case 16:
2610                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2611                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2612                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2613                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2614                                 break;
2615                         case 17:
2616                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2617                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2618                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2619                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2620                                 break;
2621                         case 27:
2622                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2623                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2624                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2625                                 break;
2626                         case 28:
2627                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2628                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2629                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2630                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2631                                 break;
2632                         case 29:
2633                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2634                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2635                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2636                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2637                                 break;
2638                         case 30:
2639                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2640                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2641                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2642                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2643                                 break;
2644                         default:
2645                                 gb_tile_moden = 0;
2646                                 break;
2647                         }
2648                         rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2649                         WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2650                 }
2651                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2652                         switch (reg_offset) {
2653                         case 0:
2654                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2655                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2656                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2657                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2658                                 break;
2659                         case 1:
2660                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2661                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2662                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2663                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2664                                 break;
2665                         case 2:
2666                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2667                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2668                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2669                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2670                                 break;
2671                         case 3:
2672                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2673                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2674                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2675                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2676                                 break;
2677                         case 4:
2678                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2679                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2680                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2681                                                  NUM_BANKS(ADDR_SURF_8_BANK));
2682                                 break;
2683                         case 5:
2684                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2685                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2686                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2687                                                  NUM_BANKS(ADDR_SURF_4_BANK));
2688                                 break;
2689                         case 6:
2690                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2691                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2692                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2693                                                  NUM_BANKS(ADDR_SURF_2_BANK));
2694                                 break;
2695                         case 8:
2696                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2697                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2698                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2699                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2700                                 break;
2701                         case 9:
2702                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2703                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2704                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2705                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2706                                 break;
2707                         case 10:
2708                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2709                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2710                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2711                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2712                                 break;
2713                         case 11:
2714                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2715                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2716                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2717                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2718                                 break;
2719                         case 12:
2720                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2721                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2722                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2723                                                  NUM_BANKS(ADDR_SURF_8_BANK));
2724                                 break;
2725                         case 13:
2726                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2727                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2728                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2729                                                  NUM_BANKS(ADDR_SURF_4_BANK));
2730                                 break;
2731                         case 14:
2732                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2733                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2734                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2735                                                  NUM_BANKS(ADDR_SURF_2_BANK));
2736                                 break;
2737                         default:
2738                                 gb_tile_moden = 0;
2739                                 break;
2740                         }
2741                         rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
2742                         WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2743                 }
2744         } else if (num_pipe_configs == 4) {
2745                 if (num_rbs == 4) {
2746                         for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2747                                 switch (reg_offset) {
2748                                 case 0:
2749                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2750                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2751                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2752                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2753                                         break;
2754                                 case 1:
2755                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2756                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2757                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2758                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2759                                         break;
2760                                 case 2:
2761                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2762                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2763                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2764                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2765                                         break;
2766                                 case 3:
2767                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2768                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2769                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2770                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2771                                         break;
2772                                 case 4:
2773                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2774                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2775                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2776                                                          TILE_SPLIT(split_equal_to_row_size));
2777                                         break;
2778                                 case 5:
2779                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2780                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2781                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2782                                         break;
2783                                 case 6:
2784                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2785                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2786                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2787                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2788                                         break;
2789                                 case 7:
2790                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2791                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2792                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2793                                                          TILE_SPLIT(split_equal_to_row_size));
2794                                         break;
2795                                 case 8:
2796                                         gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2797                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16));
2798                                         break;
2799                                 case 9:
2800                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2801                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2802                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2803                                         break;
2804                                 case 10:
2805                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2806                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2807                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2808                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2809                                         break;
2810                                 case 11:
2811                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2812                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2813                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2814                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2815                                         break;
2816                                 case 12:
2817                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2818                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2819                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2820                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2821                                         break;
2822                                 case 13:
2823                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2824                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2825                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2826                                         break;
2827                                 case 14:
2828                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2829                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2830                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2831                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2832                                         break;
2833                                 case 16:
2834                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2835                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2836                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2837                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2838                                         break;
2839                                 case 17:
2840                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2841                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2842                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2843                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2844                                         break;
2845                                 case 27:
2846                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2847                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2848                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2849                                         break;
2850                                 case 28:
2851                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2852                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2853                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2854                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2855                                         break;
2856                                 case 29:
2857                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2858                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2859                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2860                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2861                                         break;
2862                                 case 30:
2863                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2864                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2865                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2866                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2867                                         break;
2868                                 default:
2869                                         gb_tile_moden = 0;
2870                                         break;
2871                                 }
2872                                 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2873                                 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2874                         }
2875                 } else if (num_rbs < 4) {
2876                         for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2877                                 switch (reg_offset) {
2878                                 case 0:
2879                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2880                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2881                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2882                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2883                                         break;
2884                                 case 1:
2885                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2886                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2887                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2888                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2889                                         break;
2890                                 case 2:
2891                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2892                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2893                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2894                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2895                                         break;
2896                                 case 3:
2897                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2898                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2899                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2900                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2901                                         break;
2902                                 case 4:
2903                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2904                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2905                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2906                                                          TILE_SPLIT(split_equal_to_row_size));
2907                                         break;
2908                                 case 5:
2909                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2910                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2911                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2912                                         break;
2913                                 case 6:
2914                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2915                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2916                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2917                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2918                                         break;
2919                                 case 7:
2920                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2921                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2922                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2923                                                          TILE_SPLIT(split_equal_to_row_size));
2924                                         break;
2925                                 case 8:
2926                                         gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2927                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16));
2928                                         break;
2929                                 case 9:
2930                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2931                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2932                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2933                                         break;
2934                                 case 10:
2935                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2936                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2937                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2938                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2939                                         break;
2940                                 case 11:
2941                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2942                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2943                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2944                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2945                                         break;
2946                                 case 12:
2947                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2948                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2949                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2950                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2951                                         break;
2952                                 case 13:
2953                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2954                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2955                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2956                                         break;
2957                                 case 14:
2958                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2959                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2960                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2961                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2962                                         break;
2963                                 case 16:
2964                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2965                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2966                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2967                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2968                                         break;
2969                                 case 17:
2970                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2971                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2972                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2973                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2974                                         break;
2975                                 case 27:
2976                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2977                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2978                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2979                                         break;
2980                                 case 28:
2981                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2982                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2983                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2984                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2985                                         break;
2986                                 case 29:
2987                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2988                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2989                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2990                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2991                                         break;
2992                                 case 30:
2993                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2994                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2995                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2996                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2997                                         break;
2998                                 default:
2999                                         gb_tile_moden = 0;
3000                                         break;
3001                                 }
3002                                 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
3003                                 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3004                         }
3005                 }
3006                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
3007                         switch (reg_offset) {
3008                         case 0:
3009                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3010                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3011                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3012                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3013                                 break;
3014                         case 1:
3015                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3016                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3017                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3018                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3019                                 break;
3020                         case 2:
3021                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3022                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3023                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3024                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3025                                 break;
3026                         case 3:
3027                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3028                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3029                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3030                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3031                                 break;
3032                         case 4:
3033                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3034                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3035                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3036                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3037                                 break;
3038                         case 5:
3039                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3040                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3041                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3042                                                  NUM_BANKS(ADDR_SURF_8_BANK));
3043                                 break;
3044                         case 6:
3045                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3046                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3047                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3048                                                  NUM_BANKS(ADDR_SURF_4_BANK));
3049                                 break;
3050                         case 8:
3051                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3052                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3053                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3054                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3055                                 break;
3056                         case 9:
3057                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3058                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3059                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3060                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3061                                 break;
3062                         case 10:
3063                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3064                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3065                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3066                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3067                                 break;
3068                         case 11:
3069                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3070                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3071                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3072                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3073                                 break;
3074                         case 12:
3075                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3076                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3077                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3078                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3079                                 break;
3080                         case 13:
3081                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3082                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3083                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3084                                                  NUM_BANKS(ADDR_SURF_8_BANK));
3085                                 break;
3086                         case 14:
3087                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3088                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3089                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3090                                                  NUM_BANKS(ADDR_SURF_4_BANK));
3091                                 break;
3092                         default:
3093                                 gb_tile_moden = 0;
3094                                 break;
3095                         }
3096                         rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
3097                         WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3098                 }
3099         } else if (num_pipe_configs == 2) {
3100                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
3101                         switch (reg_offset) {
3102                         case 0:
3103                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3104                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3105                                                  PIPE_CONFIG(ADDR_SURF_P2) |
3106                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
3107                                 break;
3108                         case 1:
3109                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3110                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3111                                                  PIPE_CONFIG(ADDR_SURF_P2) |
3112                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
3113                                 break;
3114                         case 2:
3115                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3116                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3117                                                  PIPE_CONFIG(ADDR_SURF_P2) |
3118                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
3119                                 break;
3120                         case 3:
3121                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3122                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3123                                                  PIPE_CONFIG(ADDR_SURF_P2) |
3124                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
3125                                 break;
3126                         case 4:
3127                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3128                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3129                                                  PIPE_CONFIG(ADDR_SURF_P2) |
3130                                                  TILE_SPLIT(split_equal_to_row_size));
3131                                 break;
3132                         case 5:
3133                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3134                                                  PIPE_CONFIG(ADDR_SURF_P2) |
3135                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3136                                 break;
3137                         case 6:
3138                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3139                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3140                                                  PIPE_CONFIG(ADDR_SURF_P2) |
3141                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
3142                                 break;
3143                         case 7:
3144                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3145                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3146                                                  PIPE_CONFIG(ADDR_SURF_P2) |
3147                                                  TILE_SPLIT(split_equal_to_row_size));
3148                                 break;
3149                         case 8:
3150                                 gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
3151                                                 PIPE_CONFIG(ADDR_SURF_P2);
3152                                 break;
3153                         case 9:
3154                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3155                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3156                                                  PIPE_CONFIG(ADDR_SURF_P2));
3157                                 break;
3158                         case 10:
3159                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3160                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3161                                                  PIPE_CONFIG(ADDR_SURF_P2) |
3162                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3163                                 break;
3164                         case 11:
3165                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3166                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3167                                                  PIPE_CONFIG(ADDR_SURF_P2) |
3168                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3169                                 break;
3170                         case 12:
3171                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3172                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3173                                                  PIPE_CONFIG(ADDR_SURF_P2) |
3174                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3175                                 break;
3176                         case 13:
3177                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3178                                                  PIPE_CONFIG(ADDR_SURF_P2) |
3179                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
3180                                 break;
3181                         case 14:
3182                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3183                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3184                                                  PIPE_CONFIG(ADDR_SURF_P2) |
3185                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3186                                 break;
3187                         case 16:
3188                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3189                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3190                                                  PIPE_CONFIG(ADDR_SURF_P2) |
3191                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3192                                 break;
3193                         case 17:
3194                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3195                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3196                                                  PIPE_CONFIG(ADDR_SURF_P2) |
3197                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3198                                 break;
3199                         case 27:
3200                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3201                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3202                                                  PIPE_CONFIG(ADDR_SURF_P2));
3203                                 break;
3204                         case 28:
3205                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3206                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3207                                                  PIPE_CONFIG(ADDR_SURF_P2) |
3208                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3209                                 break;
3210                         case 29:
3211                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3212                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3213                                                  PIPE_CONFIG(ADDR_SURF_P2) |
3214                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3215                                 break;
3216                         case 30:
3217                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3218                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3219                                                  PIPE_CONFIG(ADDR_SURF_P2) |
3220                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3221                                 break;
3222                         default:
3223                                 gb_tile_moden = 0;
3224                                 break;
3225                         }
3226                         rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
3227                         WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3228                 }
3229                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
3230                         switch (reg_offset) {
3231                         case 0:
3232                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3233                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3234                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3235                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3236                                 break;
3237                         case 1:
3238                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3239                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3240                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3241                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3242                                 break;
3243                         case 2:
3244                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3245                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3246                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3247                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3248                                 break;
3249                         case 3:
3250                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3251                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3252                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3253                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3254                                 break;
3255                         case 4:
3256                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3257                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3258                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3259                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3260                                 break;
3261                         case 5:
3262                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3263                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3264                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3265                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3266                                 break;
3267                         case 6:
3268                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3269                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3270                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3271                                                  NUM_BANKS(ADDR_SURF_8_BANK));
3272                                 break;
3273                         case 8:
3274                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3275                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3276                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3277                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3278                                 break;
3279                         case 9:
3280                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3281                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3282                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3283                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3284                                 break;
3285                         case 10:
3286                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3287                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3288                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3289                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3290                                 break;
3291                         case 11:
3292                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3293                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3294                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3295                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3296                                 break;
3297                         case 12:
3298                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3299                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3300                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3301                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3302                                 break;
3303                         case 13:
3304                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3305                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3306                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3307                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3308                                 break;
3309                         case 14:
3310                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3311                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3312                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3313                                                  NUM_BANKS(ADDR_SURF_8_BANK));
3314                                 break;
3315                         default:
3316                                 gb_tile_moden = 0;
3317                                 break;
3318                         }
3319                         rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
3320                         WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3321                 }
3322         } else
3323                 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
3324 }
3325
3326 /**
3327  * cik_select_se_sh - select which SE, SH to address
3328  *
3329  * @rdev: radeon_device pointer
3330  * @se_num: shader engine to address
3331  * @sh_num: sh block to address
3332  *
3333  * Select which SE, SH combinations to address. Certain
3334  * registers are instanced per SE or SH.  0xffffffff means
3335  * broadcast to all SEs or SHs (CIK).
3336  */
3337 static void cik_select_se_sh(struct radeon_device *rdev,
3338                              u32 se_num, u32 sh_num)
3339 {
3340         u32 data = INSTANCE_BROADCAST_WRITES;
3341
3342         if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
3343                 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
3344         else if (se_num == 0xffffffff)
3345                 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
3346         else if (sh_num == 0xffffffff)
3347                 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
3348         else
3349                 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
3350         WREG32(GRBM_GFX_INDEX, data);
3351 }
3352
3353 /**
3354  * cik_create_bitmask - create a bitmask
3355  *
3356  * @bit_width: length of the mask
3357  *
3358  * create a variable length bit mask (CIK).
3359  * Returns the bitmask.
3360  */
3361 static u32 cik_create_bitmask(u32 bit_width)
3362 {
3363         u32 i, mask = 0;
3364
3365         for (i = 0; i < bit_width; i++) {
3366                 mask <<= 1;
3367                 mask |= 1;
3368         }
3369         return mask;
3370 }
3371
3372 /**
3373  * cik_get_rb_disabled - computes the mask of disabled RBs
3374  *
3375  * @rdev: radeon_device pointer
3376  * @max_rb_num: max RBs (render backends) for the asic
3377  * @se_num: number of SEs (shader engines) for the asic
3378  * @sh_per_se: number of SH blocks per SE for the asic
3379  *
3380  * Calculates the bitmask of disabled RBs (CIK).
3381  * Returns the disabled RB bitmask.
3382  */
3383 static u32 cik_get_rb_disabled(struct radeon_device *rdev,
3384                               u32 max_rb_num_per_se,
3385                               u32 sh_per_se)
3386 {
3387         u32 data, mask;
3388
3389         data = RREG32(CC_RB_BACKEND_DISABLE);
3390         if (data & 1)
3391                 data &= BACKEND_DISABLE_MASK;
3392         else
3393                 data = 0;
3394         data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3395
3396         data >>= BACKEND_DISABLE_SHIFT;
3397
3398         mask = cik_create_bitmask(max_rb_num_per_se / sh_per_se);
3399
3400         return data & mask;
3401 }
3402
3403 /**
3404  * cik_setup_rb - setup the RBs on the asic
3405  *
3406  * @rdev: radeon_device pointer
3407  * @se_num: number of SEs (shader engines) for the asic
3408  * @sh_per_se: number of SH blocks per SE for the asic
3409  * @max_rb_num: max RBs (render backends) for the asic
3410  *
3411  * Configures per-SE/SH RB registers (CIK).
3412  */
3413 static void cik_setup_rb(struct radeon_device *rdev,
3414                          u32 se_num, u32 sh_per_se,
3415                          u32 max_rb_num_per_se)
3416 {
3417         int i, j;
3418         u32 data, mask;
3419         u32 disabled_rbs = 0;
3420         u32 enabled_rbs = 0;
3421
3422         for (i = 0; i < se_num; i++) {
3423                 for (j = 0; j < sh_per_se; j++) {
3424                         cik_select_se_sh(rdev, i, j);
3425                         data = cik_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
3426                         if (rdev->family == CHIP_HAWAII)
3427                                 disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH);
3428                         else
3429                                 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
3430                 }
3431         }
3432         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3433
3434         mask = 1;
3435         for (i = 0; i < max_rb_num_per_se * se_num; i++) {
3436                 if (!(disabled_rbs & mask))
3437                         enabled_rbs |= mask;
3438                 mask <<= 1;
3439         }
3440
3441         rdev->config.cik.backend_enable_mask = enabled_rbs;
3442
3443         for (i = 0; i < se_num; i++) {
3444                 cik_select_se_sh(rdev, i, 0xffffffff);
3445                 data = 0;
3446                 for (j = 0; j < sh_per_se; j++) {
3447                         switch (enabled_rbs & 3) {
3448                         case 0:
3449                                 if (j == 0)
3450                                         data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3);
3451                                 else
3452                                         data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0);
3453                                 break;
3454                         case 1:
3455                                 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3456                                 break;
3457                         case 2:
3458                                 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3459                                 break;
3460                         case 3:
3461                         default:
3462                                 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3463                                 break;
3464                         }
3465                         enabled_rbs >>= 2;
3466                 }
3467                 WREG32(PA_SC_RASTER_CONFIG, data);
3468         }
3469         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3470 }
3471
3472 /**
3473  * cik_gpu_init - setup the 3D engine
3474  *
3475  * @rdev: radeon_device pointer
3476  *
3477  * Configures the 3D engine and tiling configuration
3478  * registers so that the 3D engine is usable.
3479  */
3480 static void cik_gpu_init(struct radeon_device *rdev)
3481 {
3482         u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
3483         u32 mc_shared_chmap, mc_arb_ramcfg;
3484         u32 hdp_host_path_cntl;
3485         u32 tmp;
3486         int i, j;
3487
3488         switch (rdev->family) {
3489         case CHIP_BONAIRE:
3490                 rdev->config.cik.max_shader_engines = 2;
3491                 rdev->config.cik.max_tile_pipes = 4;
3492                 rdev->config.cik.max_cu_per_sh = 7;
3493                 rdev->config.cik.max_sh_per_se = 1;
3494                 rdev->config.cik.max_backends_per_se = 2;
3495                 rdev->config.cik.max_texture_channel_caches = 4;
3496                 rdev->config.cik.max_gprs = 256;
3497                 rdev->config.cik.max_gs_threads = 32;
3498                 rdev->config.cik.max_hw_contexts = 8;
3499
3500                 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3501                 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3502                 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3503                 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3504                 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3505                 break;
3506         case CHIP_HAWAII:
3507                 rdev->config.cik.max_shader_engines = 4;
3508                 rdev->config.cik.max_tile_pipes = 16;
3509                 rdev->config.cik.max_cu_per_sh = 11;
3510                 rdev->config.cik.max_sh_per_se = 1;
3511                 rdev->config.cik.max_backends_per_se = 4;
3512                 rdev->config.cik.max_texture_channel_caches = 16;
3513                 rdev->config.cik.max_gprs = 256;
3514                 rdev->config.cik.max_gs_threads = 32;
3515                 rdev->config.cik.max_hw_contexts = 8;
3516
3517                 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3518                 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3519                 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3520                 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3521                 gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN;
3522                 break;
3523         case CHIP_KAVERI:
3524                 rdev->config.cik.max_shader_engines = 1;
3525                 rdev->config.cik.max_tile_pipes = 4;
3526                 if ((rdev->pdev->device == 0x1304) ||
3527                     (rdev->pdev->device == 0x1305) ||
3528                     (rdev->pdev->device == 0x130C) ||
3529                     (rdev->pdev->device == 0x130F) ||
3530                     (rdev->pdev->device == 0x1310) ||
3531                     (rdev->pdev->device == 0x1311) ||
3532                     (rdev->pdev->device == 0x131C)) {
3533                         rdev->config.cik.max_cu_per_sh = 8;
3534                         rdev->config.cik.max_backends_per_se = 2;
3535                 } else if ((rdev->pdev->device == 0x1309) ||
3536                            (rdev->pdev->device == 0x130A) ||
3537                            (rdev->pdev->device == 0x130D) ||
3538                            (rdev->pdev->device == 0x1313) ||
3539                            (rdev->pdev->device == 0x131D)) {
3540                         rdev->config.cik.max_cu_per_sh = 6;
3541                         rdev->config.cik.max_backends_per_se = 2;
3542                 } else if ((rdev->pdev->device == 0x1306) ||
3543                            (rdev->pdev->device == 0x1307) ||
3544                            (rdev->pdev->device == 0x130B) ||
3545                            (rdev->pdev->device == 0x130E) ||
3546                            (rdev->pdev->device == 0x1315) ||
3547                            (rdev->pdev->device == 0x1318) ||
3548                            (rdev->pdev->device == 0x131B)) {
3549                         rdev->config.cik.max_cu_per_sh = 4;
3550                         rdev->config.cik.max_backends_per_se = 1;
3551                 } else {
3552                         rdev->config.cik.max_cu_per_sh = 3;
3553                         rdev->config.cik.max_backends_per_se = 1;
3554                 }
3555                 rdev->config.cik.max_sh_per_se = 1;
3556                 rdev->config.cik.max_texture_channel_caches = 4;
3557                 rdev->config.cik.max_gprs = 256;
3558                 rdev->config.cik.max_gs_threads = 16;
3559                 rdev->config.cik.max_hw_contexts = 8;
3560
3561                 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3562                 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3563                 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3564                 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3565                 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3566                 break;
3567         case CHIP_KABINI:
3568         case CHIP_MULLINS:
3569         default:
3570                 rdev->config.cik.max_shader_engines = 1;
3571                 rdev->config.cik.max_tile_pipes = 2;
3572                 rdev->config.cik.max_cu_per_sh = 2;
3573                 rdev->config.cik.max_sh_per_se = 1;
3574                 rdev->config.cik.max_backends_per_se = 1;
3575                 rdev->config.cik.max_texture_channel_caches = 2;
3576                 rdev->config.cik.max_gprs = 256;
3577                 rdev->config.cik.max_gs_threads = 16;
3578                 rdev->config.cik.max_hw_contexts = 8;
3579
3580                 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3581                 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3582                 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3583                 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3584                 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3585                 break;
3586         }
3587
3588         /* Initialize HDP */
3589         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3590                 WREG32((0x2c14 + j), 0x00000000);
3591                 WREG32((0x2c18 + j), 0x00000000);
3592                 WREG32((0x2c1c + j), 0x00000000);
3593                 WREG32((0x2c20 + j), 0x00000000);
3594                 WREG32((0x2c24 + j), 0x00000000);
3595         }
3596
3597         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3598
3599         WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3600
3601         mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3602         mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3603
3604         rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
3605         rdev->config.cik.mem_max_burst_length_bytes = 256;
3606         tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3607         rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3608         if (rdev->config.cik.mem_row_size_in_kb > 4)
3609                 rdev->config.cik.mem_row_size_in_kb = 4;
3610         /* XXX use MC settings? */
3611         rdev->config.cik.shader_engine_tile_size = 32;
3612         rdev->config.cik.num_gpus = 1;
3613         rdev->config.cik.multi_gpu_tile_size = 64;
3614
3615         /* fix up row size */
3616         gb_addr_config &= ~ROW_SIZE_MASK;
3617         switch (rdev->config.cik.mem_row_size_in_kb) {
3618         case 1:
3619         default:
3620                 gb_addr_config |= ROW_SIZE(0);
3621                 break;
3622         case 2:
3623                 gb_addr_config |= ROW_SIZE(1);
3624                 break;
3625         case 4:
3626                 gb_addr_config |= ROW_SIZE(2);
3627                 break;
3628         }
3629
3630         /* setup tiling info dword.  gb_addr_config is not adequate since it does
3631          * not have bank info, so create a custom tiling dword.
3632          * bits 3:0   num_pipes
3633          * bits 7:4   num_banks
3634          * bits 11:8  group_size
3635          * bits 15:12 row_size
3636          */
3637         rdev->config.cik.tile_config = 0;
3638         switch (rdev->config.cik.num_tile_pipes) {
3639         case 1:
3640                 rdev->config.cik.tile_config |= (0 << 0);
3641                 break;
3642         case 2:
3643                 rdev->config.cik.tile_config |= (1 << 0);
3644                 break;
3645         case 4:
3646                 rdev->config.cik.tile_config |= (2 << 0);
3647                 break;
3648         case 8:
3649         default:
3650                 /* XXX what about 12? */
3651                 rdev->config.cik.tile_config |= (3 << 0);
3652                 break;
3653         }
3654         rdev->config.cik.tile_config |=
3655                 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
3656         rdev->config.cik.tile_config |=
3657                 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3658         rdev->config.cik.tile_config |=
3659                 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3660
3661         WREG32(GB_ADDR_CONFIG, gb_addr_config);
3662         WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3663         WREG32(DMIF_ADDR_CALC, gb_addr_config);
3664         WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
3665         WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
3666         WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3667         WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3668         WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3669
3670         cik_tiling_mode_table_init(rdev);
3671
3672         cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
3673                      rdev->config.cik.max_sh_per_se,
3674                      rdev->config.cik.max_backends_per_se);
3675
3676         rdev->config.cik.active_cus = 0;
3677         for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
3678                 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
3679                         rdev->config.cik.active_cus +=
3680                                 hweight32(cik_get_cu_active_bitmap(rdev, i, j));
3681                 }
3682         }
3683
3684         /* set HW defaults for 3D engine */
3685         WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3686
3687         WREG32(SX_DEBUG_1, 0x20);
3688
3689         WREG32(TA_CNTL_AUX, 0x00010000);
3690
3691         tmp = RREG32(SPI_CONFIG_CNTL);
3692         tmp |= 0x03000000;
3693         WREG32(SPI_CONFIG_CNTL, tmp);
3694
3695         WREG32(SQ_CONFIG, 1);
3696
3697         WREG32(DB_DEBUG, 0);
3698
3699         tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
3700         tmp |= 0x00000400;
3701         WREG32(DB_DEBUG2, tmp);
3702
3703         tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
3704         tmp |= 0x00020200;
3705         WREG32(DB_DEBUG3, tmp);
3706
3707         tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
3708         tmp |= 0x00018208;
3709         WREG32(CB_HW_CONTROL, tmp);
3710
3711         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3712
3713         WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
3714                                  SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
3715                                  SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
3716                                  SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
3717
3718         WREG32(VGT_NUM_INSTANCES, 1);
3719
3720         WREG32(CP_PERFMON_CNTL, 0);
3721
3722         WREG32(SQ_CONFIG, 0);
3723
3724         WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3725                                           FORCE_EOV_MAX_REZ_CNT(255)));
3726
3727         WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3728                AUTO_INVLD_EN(ES_AND_GS_AUTO));
3729
3730         WREG32(VGT_GS_VERTEX_REUSE, 16);
3731         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3732
3733         tmp = RREG32(HDP_MISC_CNTL);
3734         tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3735         WREG32(HDP_MISC_CNTL, tmp);
3736
3737         hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3738         WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3739
3740         WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3741         WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
3742
3743         udelay(50);
3744 }
3745
3746 /*
3747  * GPU scratch registers helpers function.
3748  */
3749 /**
3750  * cik_scratch_init - setup driver info for CP scratch regs
3751  *
3752  * @rdev: radeon_device pointer
3753  *
3754  * Set up the number and offset of the CP scratch registers.
3755  * NOTE: use of CP scratch registers is a legacy inferface and
3756  * is not used by default on newer asics (r6xx+).  On newer asics,
3757  * memory buffers are used for fences rather than scratch regs.
3758  */
3759 static void cik_scratch_init(struct radeon_device *rdev)
3760 {
3761         int i;
3762
3763         rdev->scratch.num_reg = 7;
3764         rdev->scratch.reg_base = SCRATCH_REG0;
3765         for (i = 0; i < rdev->scratch.num_reg; i++) {
3766                 rdev->scratch.free[i] = true;
3767                 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3768         }
3769 }
3770
3771 /**
3772  * cik_ring_test - basic gfx ring test
3773  *
3774  * @rdev: radeon_device pointer
3775  * @ring: radeon_ring structure holding ring information
3776  *
3777  * Allocate a scratch register and write to it using the gfx ring (CIK).
3778  * Provides a basic gfx ring test to verify that the ring is working.
3779  * Used by cik_cp_gfx_resume();
3780  * Returns 0 on success, error on failure.
3781  */
3782 int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
3783 {
3784         uint32_t scratch;
3785         uint32_t tmp = 0;
3786         unsigned i;
3787         int r;
3788
3789         r = radeon_scratch_get(rdev, &scratch);
3790         if (r) {
3791                 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
3792                 return r;
3793         }
3794         WREG32(scratch, 0xCAFEDEAD);
3795         r = radeon_ring_lock(rdev, ring, 3);
3796         if (r) {
3797                 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
3798                 radeon_scratch_free(rdev, scratch);
3799                 return r;
3800         }
3801         radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3802         radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
3803         radeon_ring_write(ring, 0xDEADBEEF);
3804         radeon_ring_unlock_commit(rdev, ring, false);
3805
3806         for (i = 0; i < rdev->usec_timeout; i++) {
3807                 tmp = RREG32(scratch);
3808                 if (tmp == 0xDEADBEEF)
3809                         break;
3810                 DRM_UDELAY(1);
3811         }
3812         if (i < rdev->usec_timeout) {
3813                 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3814         } else {
3815                 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
3816                           ring->idx, scratch, tmp);
3817                 r = -EINVAL;
3818         }
3819         radeon_scratch_free(rdev, scratch);
3820         return r;
3821 }
3822
3823 /**
3824  * cik_hdp_flush_cp_ring_emit - emit an hdp flush on the cp
3825  *
3826  * @rdev: radeon_device pointer
3827  * @ridx: radeon ring index
3828  *
3829  * Emits an hdp flush on the cp.
3830  */
3831 static void cik_hdp_flush_cp_ring_emit(struct radeon_device *rdev,
3832                                        int ridx)
3833 {
3834         struct radeon_ring *ring = &rdev->ring[ridx];
3835         u32 ref_and_mask;
3836
3837         switch (ring->idx) {
3838         case CAYMAN_RING_TYPE_CP1_INDEX:
3839         case CAYMAN_RING_TYPE_CP2_INDEX:
3840         default:
3841                 switch (ring->me) {
3842                 case 0:
3843                         ref_and_mask = CP2 << ring->pipe;
3844                         break;
3845                 case 1:
3846                         ref_and_mask = CP6 << ring->pipe;
3847                         break;
3848                 default:
3849                         return;
3850                 }
3851                 break;
3852         case RADEON_RING_TYPE_GFX_INDEX:
3853                 ref_and_mask = CP0;
3854                 break;
3855         }
3856
3857         radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
3858         radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */
3859                                  WAIT_REG_MEM_FUNCTION(3) |  /* == */
3860                                  WAIT_REG_MEM_ENGINE(1)));   /* pfp */
3861         radeon_ring_write(ring, GPU_HDP_FLUSH_REQ >> 2);
3862         radeon_ring_write(ring, GPU_HDP_FLUSH_DONE >> 2);
3863         radeon_ring_write(ring, ref_and_mask);
3864         radeon_ring_write(ring, ref_and_mask);
3865         radeon_ring_write(ring, 0x20); /* poll interval */
3866 }
3867
3868 /**
3869  * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
3870  *
3871  * @rdev: radeon_device pointer
3872  * @fence: radeon fence object
3873  *
3874  * Emits a fence sequnce number on the gfx ring and flushes
3875  * GPU caches.
3876  */
3877 void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
3878                              struct radeon_fence *fence)
3879 {
3880         struct radeon_ring *ring = &rdev->ring[fence->ring];
3881         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3882
3883         /* EVENT_WRITE_EOP - flush caches, send int */
3884         radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3885         radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3886                                  EOP_TC_ACTION_EN |
3887                                  EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3888                                  EVENT_INDEX(5)));
3889         radeon_ring_write(ring, addr & 0xfffffffc);
3890         radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3891         radeon_ring_write(ring, fence->seq);
3892         radeon_ring_write(ring, 0);
3893 }
3894
3895 /**
3896  * cik_fence_compute_ring_emit - emit a fence on the compute ring
3897  *
3898  * @rdev: radeon_device pointer
3899  * @fence: radeon fence object
3900  *
3901  * Emits a fence sequnce number on the compute ring and flushes
3902  * GPU caches.
3903  */
3904 void cik_fence_compute_ring_emit(struct radeon_device *rdev,
3905                                  struct radeon_fence *fence)
3906 {
3907         struct radeon_ring *ring = &rdev->ring[fence->ring];
3908         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3909
3910         /* RELEASE_MEM - flush caches, send int */
3911         radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
3912         radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3913                                  EOP_TC_ACTION_EN |
3914                                  EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3915                                  EVENT_INDEX(5)));
3916         radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
3917         radeon_ring_write(ring, addr & 0xfffffffc);
3918         radeon_ring_write(ring, upper_32_bits(addr));
3919         radeon_ring_write(ring, fence->seq);
3920         radeon_ring_write(ring, 0);
3921 }
3922
3923 /**
3924  * cik_semaphore_ring_emit - emit a semaphore on the CP ring
3925  *
3926  * @rdev: radeon_device pointer
3927  * @ring: radeon ring buffer object
3928  * @semaphore: radeon semaphore object
3929  * @emit_wait: Is this a sempahore wait?
3930  *
3931  * Emits a semaphore signal/wait packet to the CP ring and prevents the PFP
3932  * from running ahead of semaphore waits.
3933  */
3934 bool cik_semaphore_ring_emit(struct radeon_device *rdev,
3935                              struct radeon_ring *ring,
3936                              struct radeon_semaphore *semaphore,
3937                              bool emit_wait)
3938 {
3939         uint64_t addr = semaphore->gpu_addr;
3940         unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3941
3942         radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
3943         radeon_ring_write(ring, lower_32_bits(addr));
3944         radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
3945
3946         if (emit_wait && ring->idx == RADEON_RING_TYPE_GFX_INDEX) {
3947                 /* Prevent the PFP from running ahead of the semaphore wait */
3948                 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
3949                 radeon_ring_write(ring, 0x0);
3950         }
3951
3952         return true;
3953 }
3954
3955 /**
3956  * cik_copy_cpdma - copy pages using the CP DMA engine
3957  *
3958  * @rdev: radeon_device pointer
3959  * @src_offset: src GPU address
3960  * @dst_offset: dst GPU address
3961  * @num_gpu_pages: number of GPU pages to xfer
3962  * @resv: reservation object to sync to
3963  *
3964  * Copy GPU paging using the CP DMA engine (CIK+).
3965  * Used by the radeon ttm implementation to move pages if
3966  * registered as the asic copy callback.
3967  */
3968 struct radeon_fence *cik_copy_cpdma(struct radeon_device *rdev,
3969                                     uint64_t src_offset, uint64_t dst_offset,
3970                                     unsigned num_gpu_pages,
3971                                     struct reservation_object *resv)
3972 {
3973         struct radeon_semaphore *sem = NULL;
3974         struct radeon_fence *fence;
3975         int ring_index = rdev->asic->copy.blit_ring_index;
3976         struct radeon_ring *ring = &rdev->ring[ring_index];
3977         u32 size_in_bytes, cur_size_in_bytes, control;
3978         int i, num_loops;
3979         int r = 0;
3980
3981         r = radeon_semaphore_create(rdev, &sem);
3982         if (r) {
3983                 DRM_ERROR("radeon: moving bo (%d).\n", r);
3984                 return ERR_PTR(r);
3985         }
3986
3987         size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
3988         num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
3989         r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18);
3990         if (r) {
3991                 DRM_ERROR("radeon: moving bo (%d).\n", r);
3992                 radeon_semaphore_free(rdev, &sem, NULL);
3993                 return ERR_PTR(r);
3994         }
3995
3996         radeon_semaphore_sync_resv(sem, resv, false);
3997         radeon_semaphore_sync_rings(rdev, sem, ring->idx);
3998
3999         for (i = 0; i < num_loops; i++) {
4000                 cur_size_in_bytes = size_in_bytes;
4001                 if (cur_size_in_bytes > 0x1fffff)
4002                         cur_size_in_bytes = 0x1fffff;
4003                 size_in_bytes -= cur_size_in_bytes;
4004                 control = 0;
4005                 if (size_in_bytes == 0)
4006                         control |= PACKET3_DMA_DATA_CP_SYNC;
4007                 radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
4008                 radeon_ring_write(ring, control);
4009                 radeon_ring_write(ring, lower_32_bits(src_offset));
4010                 radeon_ring_write(ring, upper_32_bits(src_offset));
4011                 radeon_ring_write(ring, lower_32_bits(dst_offset));
4012                 radeon_ring_write(ring, upper_32_bits(dst_offset));
4013                 radeon_ring_write(ring, cur_size_in_bytes);
4014                 src_offset += cur_size_in_bytes;
4015                 dst_offset += cur_size_in_bytes;
4016         }
4017
4018         r = radeon_fence_emit(rdev, &fence, ring->idx);
4019         if (r) {
4020                 radeon_ring_unlock_undo(rdev, ring);
4021                 radeon_semaphore_free(rdev, &sem, NULL);
4022                 return ERR_PTR(r);
4023         }
4024
4025         radeon_ring_unlock_commit(rdev, ring, false);
4026         radeon_semaphore_free(rdev, &sem, fence);
4027
4028         return fence;
4029 }
4030
4031 /*
4032  * IB stuff
4033  */
4034 /**
4035  * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
4036  *
4037  * @rdev: radeon_device pointer
4038  * @ib: radeon indirect buffer object
4039  *
4040  * Emits an DE (drawing engine) or CE (constant engine) IB
4041  * on the gfx ring.  IBs are usually generated by userspace
4042  * acceleration drivers and submitted to the kernel for
4043  * sheduling on the ring.  This function schedules the IB
4044  * on the gfx ring for execution by the GPU.
4045  */
4046 void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
4047 {
4048         struct radeon_ring *ring = &rdev->ring[ib->ring];
4049         u32 header, control = INDIRECT_BUFFER_VALID;
4050
4051         if (ib->is_const_ib) {
4052                 /* set switch buffer packet before const IB */
4053                 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
4054                 radeon_ring_write(ring, 0);
4055
4056                 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
4057         } else {
4058                 u32 next_rptr;
4059                 if (ring->rptr_save_reg) {
4060                         next_rptr = ring->wptr + 3 + 4;
4061                         radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
4062                         radeon_ring_write(ring, ((ring->rptr_save_reg -
4063                                                   PACKET3_SET_UCONFIG_REG_START) >> 2));
4064                         radeon_ring_write(ring, next_rptr);
4065                 } else if (rdev->wb.enabled) {
4066                         next_rptr = ring->wptr + 5 + 4;
4067                         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4068                         radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
4069                         radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
4070                         radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr));
4071                         radeon_ring_write(ring, next_rptr);
4072                 }
4073
4074                 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
4075         }
4076
4077         control |= ib->length_dw |
4078                 (ib->vm ? (ib->vm->id << 24) : 0);
4079
4080         radeon_ring_write(ring, header);
4081         radeon_ring_write(ring,
4082 #ifdef __BIG_ENDIAN
4083                           (2 << 0) |
4084 #endif
4085                           (ib->gpu_addr & 0xFFFFFFFC));
4086         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
4087         radeon_ring_write(ring, control);
4088 }
4089
4090 /**
4091  * cik_ib_test - basic gfx ring IB test
4092  *
4093  * @rdev: radeon_device pointer
4094  * @ring: radeon_ring structure holding ring information
4095  *
4096  * Allocate an IB and execute it on the gfx ring (CIK).
4097  * Provides a basic gfx ring test to verify that IBs are working.
4098  * Returns 0 on success, error on failure.
4099  */
4100 int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
4101 {
4102         struct radeon_ib ib;
4103         uint32_t scratch;
4104         uint32_t tmp = 0;
4105         unsigned i;
4106         int r;
4107
4108         r = radeon_scratch_get(rdev, &scratch);
4109         if (r) {
4110                 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
4111                 return r;
4112         }
4113         WREG32(scratch, 0xCAFEDEAD);
4114         r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
4115         if (r) {
4116                 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
4117                 radeon_scratch_free(rdev, scratch);
4118                 return r;
4119         }
4120         ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
4121         ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
4122         ib.ptr[2] = 0xDEADBEEF;
4123         ib.length_dw = 3;
4124         r = radeon_ib_schedule(rdev, &ib, NULL, false);
4125         if (r) {
4126                 radeon_scratch_free(rdev, scratch);
4127                 radeon_ib_free(rdev, &ib);
4128                 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
4129                 return r;
4130         }
4131         r = radeon_fence_wait(ib.fence, false);
4132         if (r) {
4133                 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
4134                 radeon_scratch_free(rdev, scratch);
4135                 radeon_ib_free(rdev, &ib);
4136                 return r;
4137         }
4138         for (i = 0; i < rdev->usec_timeout; i++) {
4139                 tmp = RREG32(scratch);
4140                 if (tmp == 0xDEADBEEF)
4141                         break;
4142                 DRM_UDELAY(1);
4143         }
4144         if (i < rdev->usec_timeout) {
4145                 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
4146         } else {
4147                 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
4148                           scratch, tmp);
4149                 r = -EINVAL;
4150         }
4151         radeon_scratch_free(rdev, scratch);
4152         radeon_ib_free(rdev, &ib);
4153         return r;
4154 }
4155
4156 /*
4157  * CP.
4158  * On CIK, gfx and compute now have independant command processors.
4159  *
4160  * GFX
4161  * Gfx consists of a single ring and can process both gfx jobs and
4162  * compute jobs.  The gfx CP consists of three microengines (ME):
4163  * PFP - Pre-Fetch Parser
4164  * ME - Micro Engine
4165  * CE - Constant Engine
4166  * The PFP and ME make up what is considered the Drawing Engine (DE).
4167  * The CE is an asynchronous engine used for updating buffer desciptors
4168  * used by the DE so that they can be loaded into cache in parallel
4169  * while the DE is processing state update packets.
4170  *
4171  * Compute
4172  * The compute CP consists of two microengines (ME):
4173  * MEC1 - Compute MicroEngine 1
4174  * MEC2 - Compute MicroEngine 2
4175  * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
4176  * The queues are exposed to userspace and are programmed directly
4177  * by the compute runtime.
4178  */
4179 /**
4180  * cik_cp_gfx_enable - enable/disable the gfx CP MEs
4181  *
4182  * @rdev: radeon_device pointer
4183  * @enable: enable or disable the MEs
4184  *
4185  * Halts or unhalts the gfx MEs.
4186  */
4187 static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
4188 {
4189         if (enable)
4190                 WREG32(CP_ME_CNTL, 0);
4191         else {
4192                 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
4193                         radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
4194                 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
4195                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4196         }
4197         udelay(50);
4198 }
4199
4200 /**
4201  * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
4202  *
4203  * @rdev: radeon_device pointer
4204  *
4205  * Loads the gfx PFP, ME, and CE ucode.
4206  * Returns 0 for success, -EINVAL if the ucode is not available.
4207  */
4208 static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
4209 {
4210         int i;
4211
4212         if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
4213                 return -EINVAL;
4214
4215         cik_cp_gfx_enable(rdev, false);
4216
4217         if (rdev->new_fw) {
4218                 const struct gfx_firmware_header_v1_0 *pfp_hdr =
4219                         (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
4220                 const struct gfx_firmware_header_v1_0 *ce_hdr =
4221                         (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
4222                 const struct gfx_firmware_header_v1_0 *me_hdr =
4223                         (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
4224                 const __le32 *fw_data;
4225                 u32 fw_size;
4226
4227                 radeon_ucode_print_gfx_hdr(&pfp_hdr->header);
4228                 radeon_ucode_print_gfx_hdr(&ce_hdr->header);
4229                 radeon_ucode_print_gfx_hdr(&me_hdr->header);
4230
4231                 /* PFP */
4232                 fw_data = (const __le32 *)
4233                         (rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
4234                 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
4235                 WREG32(CP_PFP_UCODE_ADDR, 0);
4236                 for (i = 0; i < fw_size; i++)
4237                         WREG32(CP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
4238                 WREG32(CP_PFP_UCODE_ADDR, 0);
4239
4240                 /* CE */
4241                 fw_data = (const __le32 *)
4242                         (rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
4243                 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
4244                 WREG32(CP_CE_UCODE_ADDR, 0);
4245                 for (i = 0; i < fw_size; i++)
4246                         WREG32(CP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
4247                 WREG32(CP_CE_UCODE_ADDR, 0);
4248
4249                 /* ME */
4250                 fw_data = (const __be32 *)
4251                         (rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
4252                 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
4253                 WREG32(CP_ME_RAM_WADDR, 0);
4254                 for (i = 0; i < fw_size; i++)
4255                         WREG32(CP_ME_RAM_DATA, le32_to_cpup(fw_data++));
4256                 WREG32(CP_ME_RAM_WADDR, 0);
4257         } else {
4258                 const __be32 *fw_data;
4259
4260                 /* PFP */
4261                 fw_data = (const __be32 *)rdev->pfp_fw->data;
4262                 WREG32(CP_PFP_UCODE_ADDR, 0);
4263                 for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
4264                         WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
4265                 WREG32(CP_PFP_UCODE_ADDR, 0);
4266
4267                 /* CE */
4268                 fw_data = (const __be32 *)rdev->ce_fw->data;
4269                 WREG32(CP_CE_UCODE_ADDR, 0);
4270                 for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
4271                         WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
4272                 WREG32(CP_CE_UCODE_ADDR, 0);
4273
4274                 /* ME */
4275                 fw_data = (const __be32 *)rdev->me_fw->data;
4276                 WREG32(CP_ME_RAM_WADDR, 0);
4277                 for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
4278                         WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
4279                 WREG32(CP_ME_RAM_WADDR, 0);
4280         }
4281
4282         WREG32(CP_PFP_UCODE_ADDR, 0);
4283         WREG32(CP_CE_UCODE_ADDR, 0);
4284         WREG32(CP_ME_RAM_WADDR, 0);
4285         WREG32(CP_ME_RAM_RADDR, 0);
4286         return 0;
4287 }
4288
4289 /**
4290  * cik_cp_gfx_start - start the gfx ring
4291  *
4292  * @rdev: radeon_device pointer
4293  *
4294  * Enables the ring and loads the clear state context and other
4295  * packets required to init the ring.
4296  * Returns 0 for success, error for failure.
4297  */
4298 static int cik_cp_gfx_start(struct radeon_device *rdev)
4299 {
4300         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4301         int r, i;
4302
4303         /* init the CP */
4304         WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
4305         WREG32(CP_ENDIAN_SWAP, 0);
4306         WREG32(CP_DEVICE_ID, 1);
4307
4308         cik_cp_gfx_enable(rdev, true);
4309
4310         r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
4311         if (r) {
4312                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
4313                 return r;
4314         }
4315
4316         /* init the CE partitions.  CE only used for gfx on CIK */
4317         radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
4318         radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
4319         radeon_ring_write(ring, 0xc000);
4320         radeon_ring_write(ring, 0xc000);
4321
4322         /* setup clear context state */
4323         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4324         radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
4325
4326         radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
4327         radeon_ring_write(ring, 0x80000000);
4328         radeon_ring_write(ring, 0x80000000);
4329
4330         for (i = 0; i < cik_default_size; i++)
4331                 radeon_ring_write(ring, cik_default_state[i]);
4332
4333         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4334         radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
4335
4336         /* set clear context state */
4337         radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
4338         radeon_ring_write(ring, 0);
4339
4340         radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
4341         radeon_ring_write(ring, 0x00000316);
4342         radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
4343         radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
4344
4345         radeon_ring_unlock_commit(rdev, ring, false);
4346
4347         return 0;
4348 }
4349
4350 /**
4351  * cik_cp_gfx_fini - stop the gfx ring
4352  *
4353  * @rdev: radeon_device pointer
4354  *
4355  * Stop the gfx ring and tear down the driver ring
4356  * info.
4357  */
4358 static void cik_cp_gfx_fini(struct radeon_device *rdev)
4359 {
4360         cik_cp_gfx_enable(rdev, false);
4361         radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4362 }
4363
4364 /**
4365  * cik_cp_gfx_resume - setup the gfx ring buffer registers
4366  *
4367  * @rdev: radeon_device pointer
4368  *
4369  * Program the location and size of the gfx ring buffer
4370  * and test it to make sure it's working.
4371  * Returns 0 for success, error for failure.
4372  */
4373 static int cik_cp_gfx_resume(struct radeon_device *rdev)
4374 {
4375         struct radeon_ring *ring;
4376         u32 tmp;
4377         u32 rb_bufsz;
4378         u64 rb_addr;
4379         int r;
4380
4381         WREG32(CP_SEM_WAIT_TIMER, 0x0);
4382         if (rdev->family != CHIP_HAWAII)
4383                 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
4384
4385         /* Set the write pointer delay */
4386         WREG32(CP_RB_WPTR_DELAY, 0);
4387
4388         /* set the RB to use vmid 0 */
4389         WREG32(CP_RB_VMID, 0);
4390
4391         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
4392
4393         /* ring 0 - compute and gfx */
4394         /* Set ring buffer size */
4395         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4396         rb_bufsz = order_base_2(ring->ring_size / 8);
4397         tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
4398 #ifdef __BIG_ENDIAN
4399         tmp |= BUF_SWAP_32BIT;
4400 #endif
4401         WREG32(CP_RB0_CNTL, tmp);
4402
4403         /* Initialize the ring buffer's read and write pointers */
4404         WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
4405         ring->wptr = 0;
4406         WREG32(CP_RB0_WPTR, ring->wptr);
4407
4408         /* set the wb address wether it's enabled or not */
4409         WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
4410         WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
4411
4412         /* scratch register shadowing is no longer supported */
4413         WREG32(SCRATCH_UMSK, 0);
4414
4415         if (!rdev->wb.enabled)
4416                 tmp |= RB_NO_UPDATE;
4417
4418         mdelay(1);
4419         WREG32(CP_RB0_CNTL, tmp);
4420
4421         rb_addr = ring->gpu_addr >> 8;
4422         WREG32(CP_RB0_BASE, rb_addr);
4423         WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
4424
4425         /* start the ring */
4426         cik_cp_gfx_start(rdev);
4427         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
4428         r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4429         if (r) {
4430                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4431                 return r;
4432         }
4433
4434         if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
4435                 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
4436
4437         return 0;
4438 }
4439
4440 u32 cik_gfx_get_rptr(struct radeon_device *rdev,
4441                      struct radeon_ring *ring)
4442 {
4443         u32 rptr;
4444
4445         if (rdev->wb.enabled)
4446                 rptr = rdev->wb.wb[ring->rptr_offs/4];
4447         else
4448                 rptr = RREG32(CP_RB0_RPTR);
4449
4450         return rptr;
4451 }
4452
4453 u32 cik_gfx_get_wptr(struct radeon_device *rdev,
4454                      struct radeon_ring *ring)
4455 {
4456         u32 wptr;
4457
4458         wptr = RREG32(CP_RB0_WPTR);
4459
4460         return wptr;
4461 }
4462
4463 void cik_gfx_set_wptr(struct radeon_device *rdev,
4464                       struct radeon_ring *ring)
4465 {
4466         WREG32(CP_RB0_WPTR, ring->wptr);
4467         (void)RREG32(CP_RB0_WPTR);
4468 }
4469
4470 u32 cik_compute_get_rptr(struct radeon_device *rdev,
4471                          struct radeon_ring *ring)
4472 {
4473         u32 rptr;
4474
4475         if (rdev->wb.enabled) {
4476                 rptr = rdev->wb.wb[ring->rptr_offs/4];
4477         } else {
4478                 mutex_lock(&rdev->srbm_mutex);
4479                 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4480                 rptr = RREG32(CP_HQD_PQ_RPTR);
4481                 cik_srbm_select(rdev, 0, 0, 0, 0);
4482                 mutex_unlock(&rdev->srbm_mutex);
4483         }
4484
4485         return rptr;
4486 }
4487
4488 u32 cik_compute_get_wptr(struct radeon_device *rdev,
4489                          struct radeon_ring *ring)
4490 {
4491         u32 wptr;
4492
4493         if (rdev->wb.enabled) {
4494                 /* XXX check if swapping is necessary on BE */
4495                 wptr = rdev->wb.wb[ring->wptr_offs/4];
4496         } else {
4497                 mutex_lock(&rdev->srbm_mutex);
4498                 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4499                 wptr = RREG32(CP_HQD_PQ_WPTR);
4500                 cik_srbm_select(rdev, 0, 0, 0, 0);
4501                 mutex_unlock(&rdev->srbm_mutex);
4502         }
4503
4504         return wptr;
4505 }
4506
4507 void cik_compute_set_wptr(struct radeon_device *rdev,
4508                           struct radeon_ring *ring)
4509 {
4510         /* XXX check if swapping is necessary on BE */
4511         rdev->wb.wb[ring->wptr_offs/4] = ring->wptr;
4512         WDOORBELL32(ring->doorbell_index, ring->wptr);
4513 }
4514
4515 /**
4516  * cik_cp_compute_enable - enable/disable the compute CP MEs
4517  *
4518  * @rdev: radeon_device pointer
4519  * @enable: enable or disable the MEs
4520  *
4521  * Halts or unhalts the compute MEs.
4522  */
4523 static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
4524 {
4525         if (enable)
4526                 WREG32(CP_MEC_CNTL, 0);
4527         else {
4528                 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
4529                 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
4530                 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
4531         }
4532         udelay(50);
4533 }
4534
4535 /**
4536  * cik_cp_compute_load_microcode - load the compute CP ME ucode
4537  *
4538  * @rdev: radeon_device pointer
4539  *
4540  * Loads the compute MEC1&2 ucode.
4541  * Returns 0 for success, -EINVAL if the ucode is not available.
4542  */
4543 static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
4544 {
4545         int i;
4546
4547         if (!rdev->mec_fw)
4548                 return -EINVAL;
4549
4550         cik_cp_compute_enable(rdev, false);
4551
4552         if (rdev->new_fw) {
4553                 const struct gfx_firmware_header_v1_0 *mec_hdr =
4554                         (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;
4555                 const __le32 *fw_data;
4556                 u32 fw_size;
4557
4558                 radeon_ucode_print_gfx_hdr(&mec_hdr->header);
4559
4560                 /* MEC1 */
4561                 fw_data = (const __le32 *)
4562                         (rdev->mec_fw->data + le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
4563                 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes) / 4;
4564                 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4565                 for (i = 0; i < fw_size; i++)
4566                         WREG32(CP_MEC_ME1_UCODE_DATA, le32_to_cpup(fw_data++));
4567                 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4568
4569                 /* MEC2 */
4570                 if (rdev->family == CHIP_KAVERI) {
4571                         const struct gfx_firmware_header_v1_0 *mec2_hdr =
4572                                 (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;
4573
4574                         fw_data = (const __le32 *)
4575                                 (rdev->mec2_fw->data +
4576                                  le32_to_cpu(mec2_hdr->header.ucode_array_offset_bytes));
4577                         fw_size = le32_to_cpu(mec2_hdr->header.ucode_size_bytes) / 4;
4578                         WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4579                         for (i = 0; i < fw_size; i++)
4580                                 WREG32(CP_MEC_ME2_UCODE_DATA, le32_to_cpup(fw_data++));
4581                         WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4582                 }
4583         } else {
4584                 const __be32 *fw_data;
4585
4586                 /* MEC1 */
4587                 fw_data = (const __be32 *)rdev->mec_fw->data;
4588                 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4589                 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4590                         WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
4591                 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4592
4593                 if (rdev->family == CHIP_KAVERI) {
4594                         /* MEC2 */
4595                         fw_data = (const __be32 *)rdev->mec_fw->data;
4596                         WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4597                         for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4598                                 WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
4599                         WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4600                 }
4601         }
4602
4603         return 0;
4604 }
4605
4606 /**
4607  * cik_cp_compute_start - start the compute queues
4608  *
4609  * @rdev: radeon_device pointer
4610  *
4611  * Enable the compute queues.
4612  * Returns 0 for success, error for failure.
4613  */
4614 static int cik_cp_compute_start(struct radeon_device *rdev)
4615 {
4616         cik_cp_compute_enable(rdev, true);
4617
4618         return 0;
4619 }
4620
4621 /**
4622  * cik_cp_compute_fini - stop the compute queues
4623  *
4624  * @rdev: radeon_device pointer
4625  *
4626  * Stop the compute queues and tear down the driver queue
4627  * info.
4628  */
4629 static void cik_cp_compute_fini(struct radeon_device *rdev)
4630 {
4631         int i, idx, r;
4632
4633         cik_cp_compute_enable(rdev, false);
4634
4635         for (i = 0; i < 2; i++) {
4636                 if (i == 0)
4637                         idx = CAYMAN_RING_TYPE_CP1_INDEX;
4638                 else
4639                         idx = CAYMAN_RING_TYPE_CP2_INDEX;
4640
4641                 if (rdev->ring[idx].mqd_obj) {
4642                         r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4643                         if (unlikely(r != 0))
4644                                 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
4645
4646                         radeon_bo_unpin(rdev->ring[idx].mqd_obj);
4647                         radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4648
4649                         radeon_bo_unref(&rdev->ring[idx].mqd_obj);
4650                         rdev->ring[idx].mqd_obj = NULL;
4651                 }
4652         }
4653 }
4654
4655 static void cik_mec_fini(struct radeon_device *rdev)
4656 {
4657         int r;
4658
4659         if (rdev->mec.hpd_eop_obj) {
4660                 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4661                 if (unlikely(r != 0))
4662                         dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
4663                 radeon_bo_unpin(rdev->mec.hpd_eop_obj);
4664                 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4665
4666                 radeon_bo_unref(&rdev->mec.hpd_eop_obj);
4667                 rdev->mec.hpd_eop_obj = NULL;
4668         }
4669 }
4670
4671 #define MEC_HPD_SIZE 2048
4672
4673 static int cik_mec_init(struct radeon_device *rdev)
4674 {
4675         int r;
4676         u32 *hpd;
4677
4678         /*
4679          * KV:    2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
4680          * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
4681          */
4682         if (rdev->family == CHIP_KAVERI)
4683                 rdev->mec.num_mec = 2;
4684         else
4685                 rdev->mec.num_mec = 1;
4686         rdev->mec.num_pipe = 4;
4687         rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
4688
4689         if (rdev->mec.hpd_eop_obj == NULL) {
4690                 r = radeon_bo_create(rdev,
4691                                      rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
4692                                      PAGE_SIZE, true,
4693                                      RADEON_GEM_DOMAIN_GTT, 0, NULL,
4694                                      &rdev->mec.hpd_eop_obj);
4695                 if (r) {
4696                         dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
4697                         return r;
4698                 }
4699         }
4700
4701         r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4702         if (unlikely(r != 0)) {
4703                 cik_mec_fini(rdev);
4704                 return r;
4705         }
4706         r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
4707                           &rdev->mec.hpd_eop_gpu_addr);
4708         if (r) {
4709                 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
4710                 cik_mec_fini(rdev);
4711                 return r;
4712         }
4713         r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
4714         if (r) {
4715                 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
4716                 cik_mec_fini(rdev);
4717                 return r;
4718         }
4719
4720         /* clear memory.  Not sure if this is required or not */
4721         memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
4722
4723         radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
4724         radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4725
4726         return 0;
4727 }
4728
4729 struct hqd_registers
4730 {
4731         u32 cp_mqd_base_addr;
4732         u32 cp_mqd_base_addr_hi;
4733         u32 cp_hqd_active;
4734         u32 cp_hqd_vmid;
4735         u32 cp_hqd_persistent_state;
4736         u32 cp_hqd_pipe_priority;
4737         u32 cp_hqd_queue_priority;
4738         u32 cp_hqd_quantum;
4739         u32 cp_hqd_pq_base;
4740         u32 cp_hqd_pq_base_hi;
4741         u32 cp_hqd_pq_rptr;
4742         u32 cp_hqd_pq_rptr_report_addr;
4743         u32 cp_hqd_pq_rptr_report_addr_hi;
4744         u32 cp_hqd_pq_wptr_poll_addr;
4745         u32 cp_hqd_pq_wptr_poll_addr_hi;
4746         u32 cp_hqd_pq_doorbell_control;
4747         u32 cp_hqd_pq_wptr;
4748         u32 cp_hqd_pq_control;
4749         u32 cp_hqd_ib_base_addr;
4750         u32 cp_hqd_ib_base_addr_hi;
4751         u32 cp_hqd_ib_rptr;
4752         u32 cp_hqd_ib_control;
4753         u32 cp_hqd_iq_timer;
4754         u32 cp_hqd_iq_rptr;
4755         u32 cp_hqd_dequeue_request;
4756         u32 cp_hqd_dma_offload;
4757         u32 cp_hqd_sema_cmd;
4758         u32 cp_hqd_msg_type;
4759         u32 cp_hqd_atomic0_preop_lo;
4760         u32 cp_hqd_atomic0_preop_hi;
4761         u32 cp_hqd_atomic1_preop_lo;
4762         u32 cp_hqd_atomic1_preop_hi;
4763         u32 cp_hqd_hq_scheduler0;
4764         u32 cp_hqd_hq_scheduler1;
4765         u32 cp_mqd_control;
4766 };
4767
4768 struct bonaire_mqd
4769 {
4770         u32 header;
4771         u32 dispatch_initiator;
4772         u32 dimensions[3];
4773         u32 start_idx[3];
4774         u32 num_threads[3];
4775         u32 pipeline_stat_enable;
4776         u32 perf_counter_enable;
4777         u32 pgm[2];
4778         u32 tba[2];
4779         u32 tma[2];
4780         u32 pgm_rsrc[2];
4781         u32 vmid;
4782         u32 resource_limits;
4783         u32 static_thread_mgmt01[2];
4784         u32 tmp_ring_size;
4785         u32 static_thread_mgmt23[2];
4786         u32 restart[3];
4787         u32 thread_trace_enable;
4788         u32 reserved1;
4789         u32 user_data[16];
4790         u32 vgtcs_invoke_count[2];
4791         struct hqd_registers queue_state;
4792         u32 dequeue_cntr;
4793         u32 interrupt_queue[64];
4794 };
4795
4796 /**
4797  * cik_cp_compute_resume - setup the compute queue registers
4798  *
4799  * @rdev: radeon_device pointer
4800  *
4801  * Program the compute queues and test them to make sure they
4802  * are working.
4803  * Returns 0 for success, error for failure.
4804  */
4805 static int cik_cp_compute_resume(struct radeon_device *rdev)
4806 {
4807         int r, i, idx;
4808         u32 tmp;
4809         bool use_doorbell = true;
4810         u64 hqd_gpu_addr;
4811         u64 mqd_gpu_addr;
4812         u64 eop_gpu_addr;
4813         u64 wb_gpu_addr;
4814         u32 *buf;
4815         struct bonaire_mqd *mqd;
4816
4817         r = cik_cp_compute_start(rdev);
4818         if (r)
4819                 return r;
4820
4821         /* fix up chicken bits */
4822         tmp = RREG32(CP_CPF_DEBUG);
4823         tmp |= (1 << 23);
4824         WREG32(CP_CPF_DEBUG, tmp);
4825
4826         /* init the pipes */
4827         mutex_lock(&rdev->srbm_mutex);
4828         for (i = 0; i < (rdev->mec.num_pipe * rdev->mec.num_mec); i++) {
4829                 int me = (i < 4) ? 1 : 2;
4830                 int pipe = (i < 4) ? i : (i - 4);
4831
4832                 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2);
4833
4834                 cik_srbm_select(rdev, me, pipe, 0, 0);
4835
4836                 /* write the EOP addr */
4837                 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
4838                 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
4839
4840                 /* set the VMID assigned */
4841                 WREG32(CP_HPD_EOP_VMID, 0);
4842
4843                 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4844                 tmp = RREG32(CP_HPD_EOP_CONTROL);
4845                 tmp &= ~EOP_SIZE_MASK;
4846                 tmp |= order_base_2(MEC_HPD_SIZE / 8);
4847                 WREG32(CP_HPD_EOP_CONTROL, tmp);
4848         }
4849         cik_srbm_select(rdev, 0, 0, 0, 0);
4850         mutex_unlock(&rdev->srbm_mutex);
4851
4852         /* init the queues.  Just two for now. */
4853         for (i = 0; i < 2; i++) {
4854                 if (i == 0)
4855                         idx = CAYMAN_RING_TYPE_CP1_INDEX;
4856                 else
4857                         idx = CAYMAN_RING_TYPE_CP2_INDEX;
4858
4859                 if (rdev->ring[idx].mqd_obj == NULL) {
4860                         r = radeon_bo_create(rdev,
4861                                              sizeof(struct bonaire_mqd),
4862                                              PAGE_SIZE, true,
4863                                              RADEON_GEM_DOMAIN_GTT, 0, NULL,
4864                                              &rdev->ring[idx].mqd_obj);
4865                         if (r) {
4866                                 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
4867                                 return r;
4868                         }
4869                 }
4870
4871                 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4872                 if (unlikely(r != 0)) {
4873                         cik_cp_compute_fini(rdev);
4874                         return r;
4875                 }
4876                 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
4877                                   &mqd_gpu_addr);
4878                 if (r) {
4879                         dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
4880                         cik_cp_compute_fini(rdev);
4881                         return r;
4882                 }
4883                 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
4884                 if (r) {
4885                         dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
4886                         cik_cp_compute_fini(rdev);
4887                         return r;
4888                 }
4889
4890                 /* init the mqd struct */
4891                 memset(buf, 0, sizeof(struct bonaire_mqd));
4892
4893                 mqd = (struct bonaire_mqd *)buf;
4894                 mqd->header = 0xC0310800;
4895                 mqd->static_thread_mgmt01[0] = 0xffffffff;
4896                 mqd->static_thread_mgmt01[1] = 0xffffffff;
4897                 mqd->static_thread_mgmt23[0] = 0xffffffff;
4898                 mqd->static_thread_mgmt23[1] = 0xffffffff;
4899
4900                 mutex_lock(&rdev->srbm_mutex);
4901                 cik_srbm_select(rdev, rdev->ring[idx].me,
4902                                 rdev->ring[idx].pipe,
4903                                 rdev->ring[idx].queue, 0);
4904
4905                 /* disable wptr polling */
4906                 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4907                 tmp &= ~WPTR_POLL_EN;
4908                 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4909
4910                 /* enable doorbell? */
4911                 mqd->queue_state.cp_hqd_pq_doorbell_control =
4912                         RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4913                 if (use_doorbell)
4914                         mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4915                 else
4916                         mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
4917                 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4918                        mqd->queue_state.cp_hqd_pq_doorbell_control);
4919
4920                 /* disable the queue if it's active */
4921                 mqd->queue_state.cp_hqd_dequeue_request = 0;
4922                 mqd->queue_state.cp_hqd_pq_rptr = 0;
4923                 mqd->queue_state.cp_hqd_pq_wptr= 0;
4924                 if (RREG32(CP_HQD_ACTIVE) & 1) {
4925                         WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
4926                         for (i = 0; i < rdev->usec_timeout; i++) {
4927                                 if (!(RREG32(CP_HQD_ACTIVE) & 1))
4928                                         break;
4929                                 udelay(1);
4930                         }
4931                         WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
4932                         WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
4933                         WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4934                 }
4935
4936                 /* set the pointer to the MQD */
4937                 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
4938                 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
4939                 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
4940                 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
4941                 /* set MQD vmid to 0 */
4942                 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
4943                 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
4944                 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
4945
4946                 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4947                 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
4948                 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
4949                 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4950                 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
4951                 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
4952
4953                 /* set up the HQD, this is similar to CP_RB0_CNTL */
4954                 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
4955                 mqd->queue_state.cp_hqd_pq_control &=
4956                         ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
4957
4958                 mqd->queue_state.cp_hqd_pq_control |=
4959                         order_base_2(rdev->ring[idx].ring_size / 8);
4960                 mqd->queue_state.cp_hqd_pq_control |=
4961                         (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8);
4962 #ifdef __BIG_ENDIAN
4963                 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
4964 #endif
4965                 mqd->queue_state.cp_hqd_pq_control &=
4966                         ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
4967                 mqd->queue_state.cp_hqd_pq_control |=
4968                         PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
4969                 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
4970
4971                 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
4972                 if (i == 0)
4973                         wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
4974                 else
4975                         wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
4976                 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
4977                 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4978                 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
4979                 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
4980                        mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
4981
4982                 /* set the wb address wether it's enabled or not */
4983                 if (i == 0)
4984                         wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
4985                 else
4986                         wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
4987                 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
4988                 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
4989                         upper_32_bits(wb_gpu_addr) & 0xffff;
4990                 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
4991                        mqd->queue_state.cp_hqd_pq_rptr_report_addr);
4992                 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
4993                        mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
4994
4995                 /* enable the doorbell if requested */
4996                 if (use_doorbell) {
4997                         mqd->queue_state.cp_hqd_pq_doorbell_control =
4998                                 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4999                         mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
5000                         mqd->queue_state.cp_hqd_pq_doorbell_control |=
5001                                 DOORBELL_OFFSET(rdev->ring[idx].doorbell_index);
5002                         mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
5003                         mqd->queue_state.cp_hqd_pq_doorbell_control &=
5004                                 ~(DOORBELL_SOURCE | DOORBELL_HIT);
5005
5006                 } else {
5007                         mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
5008                 }
5009                 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
5010                        mqd->queue_state.cp_hqd_pq_doorbell_control);
5011
5012                 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
5013                 rdev->ring[idx].wptr = 0;
5014                 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
5015                 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
5016                 mqd->queue_state.cp_hqd_pq_rptr = RREG32(CP_HQD_PQ_RPTR);
5017
5018                 /* set the vmid for the queue */
5019                 mqd->queue_state.cp_hqd_vmid = 0;
5020                 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
5021
5022                 /* activate the queue */
5023                 mqd->queue_state.cp_hqd_active = 1;
5024                 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
5025
5026                 cik_srbm_select(rdev, 0, 0, 0, 0);
5027                 mutex_unlock(&rdev->srbm_mutex);
5028
5029                 radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
5030                 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
5031
5032                 rdev->ring[idx].ready = true;
5033                 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
5034                 if (r)
5035                         rdev->ring[idx].ready = false;
5036         }
5037
5038         return 0;
5039 }
5040
5041 static void cik_cp_enable(struct radeon_device *rdev, bool enable)
5042 {
5043         cik_cp_gfx_enable(rdev, enable);
5044         cik_cp_compute_enable(rdev, enable);
5045 }
5046
5047 static int cik_cp_load_microcode(struct radeon_device *rdev)
5048 {
5049         int r;
5050
5051         r = cik_cp_gfx_load_microcode(rdev);
5052         if (r)
5053                 return r;
5054         r = cik_cp_compute_load_microcode(rdev);
5055         if (r)
5056                 return r;
5057
5058         return 0;
5059 }
5060
5061 static void cik_cp_fini(struct radeon_device *rdev)
5062 {
5063         cik_cp_gfx_fini(rdev);
5064         cik_cp_compute_fini(rdev);
5065 }
5066
5067 static int cik_cp_resume(struct radeon_device *rdev)
5068 {
5069         int r;
5070
5071         cik_enable_gui_idle_interrupt(rdev, false);
5072
5073         r = cik_cp_load_microcode(rdev);
5074         if (r)
5075                 return r;
5076
5077         r = cik_cp_gfx_resume(rdev);
5078         if (r)
5079                 return r;
5080         r = cik_cp_compute_resume(rdev);
5081         if (r)
5082                 return r;
5083
5084         cik_enable_gui_idle_interrupt(rdev, true);
5085
5086         return 0;
5087 }
5088
5089 static void cik_print_gpu_status_regs(struct radeon_device *rdev)
5090 {
5091         dev_info(rdev->dev, "  GRBM_STATUS=0x%08X\n",
5092                 RREG32(GRBM_STATUS));
5093         dev_info(rdev->dev, "  GRBM_STATUS2=0x%08X\n",
5094                 RREG32(GRBM_STATUS2));
5095         dev_info(rdev->dev, "  GRBM_STATUS_SE0=0x%08X\n",
5096                 RREG32(GRBM_STATUS_SE0));
5097         dev_info(rdev->dev, "  GRBM_STATUS_SE1=0x%08X\n",
5098                 RREG32(GRBM_STATUS_SE1));
5099         dev_info(rdev->dev, "  GRBM_STATUS_SE2=0x%08X\n",
5100                 RREG32(GRBM_STATUS_SE2));
5101         dev_info(rdev->dev, "  GRBM_STATUS_SE3=0x%08X\n",
5102                 RREG32(GRBM_STATUS_SE3));
5103         dev_info(rdev->dev, "  SRBM_STATUS=0x%08X\n",
5104                 RREG32(SRBM_STATUS));
5105         dev_info(rdev->dev, "  SRBM_STATUS2=0x%08X\n",
5106                 RREG32(SRBM_STATUS2));
5107         dev_info(rdev->dev, "  SDMA0_STATUS_REG   = 0x%08X\n",
5108                 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
5109         dev_info(rdev->dev, "  SDMA1_STATUS_REG   = 0x%08X\n",
5110                  RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
5111         dev_info(rdev->dev, "  CP_STAT = 0x%08x\n", RREG32(CP_STAT));
5112         dev_info(rdev->dev, "  CP_STALLED_STAT1 = 0x%08x\n",
5113                  RREG32(CP_STALLED_STAT1));
5114         dev_info(rdev->dev, "  CP_STALLED_STAT2 = 0x%08x\n",
5115                  RREG32(CP_STALLED_STAT2));
5116         dev_info(rdev->dev, "  CP_STALLED_STAT3 = 0x%08x\n",
5117                  RREG32(CP_STALLED_STAT3));
5118         dev_info(rdev->dev, "  CP_CPF_BUSY_STAT = 0x%08x\n",
5119                  RREG32(CP_CPF_BUSY_STAT));
5120         dev_info(rdev->dev, "  CP_CPF_STALLED_STAT1 = 0x%08x\n",
5121                  RREG32(CP_CPF_STALLED_STAT1));
5122         dev_info(rdev->dev, "  CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
5123         dev_info(rdev->dev, "  CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
5124         dev_info(rdev->dev, "  CP_CPC_STALLED_STAT1 = 0x%08x\n",
5125                  RREG32(CP_CPC_STALLED_STAT1));
5126         dev_info(rdev->dev, "  CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
5127 }
5128
5129 /**
5130  * cik_gpu_check_soft_reset - check which blocks are busy
5131  *
5132  * @rdev: radeon_device pointer
5133  *
5134  * Check which blocks are busy and return the relevant reset
5135  * mask to be used by cik_gpu_soft_reset().
5136  * Returns a mask of the blocks to be reset.
5137  */
5138 u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
5139 {
5140         u32 reset_mask = 0;
5141         u32 tmp;
5142
5143         /* GRBM_STATUS */
5144         tmp = RREG32(GRBM_STATUS);
5145         if (tmp & (PA_BUSY | SC_BUSY |
5146                    BCI_BUSY | SX_BUSY |
5147                    TA_BUSY | VGT_BUSY |
5148                    DB_BUSY | CB_BUSY |
5149                    GDS_BUSY | SPI_BUSY |
5150                    IA_BUSY | IA_BUSY_NO_DMA))
5151                 reset_mask |= RADEON_RESET_GFX;
5152
5153         if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
5154                 reset_mask |= RADEON_RESET_CP;
5155
5156         /* GRBM_STATUS2 */
5157         tmp = RREG32(GRBM_STATUS2);
5158         if (tmp & RLC_BUSY)
5159                 reset_mask |= RADEON_RESET_RLC;
5160
5161         /* SDMA0_STATUS_REG */
5162         tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
5163         if (!(tmp & SDMA_IDLE))
5164                 reset_mask |= RADEON_RESET_DMA;
5165
5166         /* SDMA1_STATUS_REG */
5167         tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
5168         if (!(tmp & SDMA_IDLE))
5169                 reset_mask |= RADEON_RESET_DMA1;
5170
5171         /* SRBM_STATUS2 */
5172         tmp = RREG32(SRBM_STATUS2);
5173         if (tmp & SDMA_BUSY)
5174                 reset_mask |= RADEON_RESET_DMA;
5175
5176         if (tmp & SDMA1_BUSY)
5177                 reset_mask |= RADEON_RESET_DMA1;
5178
5179         /* SRBM_STATUS */
5180         tmp = RREG32(SRBM_STATUS);
5181
5182         if (tmp & IH_BUSY)
5183                 reset_mask |= RADEON_RESET_IH;
5184
5185         if (tmp & SEM_BUSY)
5186                 reset_mask |= RADEON_RESET_SEM;
5187
5188         if (tmp & GRBM_RQ_PENDING)
5189                 reset_mask |= RADEON_RESET_GRBM;
5190
5191         if (tmp & VMC_BUSY)
5192                 reset_mask |= RADEON_RESET_VMC;
5193
5194         if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
5195                    MCC_BUSY | MCD_BUSY))
5196                 reset_mask |= RADEON_RESET_MC;
5197
5198         if (evergreen_is_display_hung(rdev))
5199                 reset_mask |= RADEON_RESET_DISPLAY;
5200
5201         /* Skip MC reset as it's mostly likely not hung, just busy */
5202         if (reset_mask & RADEON_RESET_MC) {
5203                 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
5204                 reset_mask &= ~RADEON_RESET_MC;
5205         }
5206
5207         return reset_mask;
5208 }
5209
5210 /**
5211  * cik_gpu_soft_reset - soft reset GPU
5212  *
5213  * @rdev: radeon_device pointer
5214  * @reset_mask: mask of which blocks to reset
5215  *
5216  * Soft reset the blocks specified in @reset_mask.
5217  */
5218 static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
5219 {
5220         struct evergreen_mc_save save;
5221         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
5222         u32 tmp;
5223
5224         if (reset_mask == 0)
5225                 return;
5226
5227         dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
5228
5229         cik_print_gpu_status_regs(rdev);
5230         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
5231                  RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
5232         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
5233                  RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
5234
5235         /* disable CG/PG */
5236         cik_fini_pg(rdev);
5237         cik_fini_cg(rdev);
5238
5239         /* stop the rlc */
5240         cik_rlc_stop(rdev);
5241
5242         /* Disable GFX parsing/prefetching */
5243         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
5244
5245         /* Disable MEC parsing/prefetching */
5246         WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
5247
5248         if (reset_mask & RADEON_RESET_DMA) {
5249                 /* sdma0 */
5250                 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5251                 tmp |= SDMA_HALT;
5252                 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5253         }
5254         if (reset_mask & RADEON_RESET_DMA1) {
5255                 /* sdma1 */
5256                 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5257                 tmp |= SDMA_HALT;
5258                 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5259         }
5260
5261         evergreen_mc_stop(rdev, &save);
5262         if (evergreen_mc_wait_for_idle(rdev)) {
5263                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5264         }
5265
5266         if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
5267                 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
5268
5269         if (reset_mask & RADEON_RESET_CP) {
5270                 grbm_soft_reset |= SOFT_RESET_CP;
5271
5272                 srbm_soft_reset |= SOFT_RESET_GRBM;
5273         }
5274
5275         if (reset_mask & RADEON_RESET_DMA)
5276                 srbm_soft_reset |= SOFT_RESET_SDMA;
5277
5278         if (reset_mask & RADEON_RESET_DMA1)
5279                 srbm_soft_reset |= SOFT_RESET_SDMA1;
5280
5281         if (reset_mask & RADEON_RESET_DISPLAY)
5282                 srbm_soft_reset |= SOFT_RESET_DC;
5283
5284         if (reset_mask & RADEON_RESET_RLC)
5285                 grbm_soft_reset |= SOFT_RESET_RLC;
5286
5287         if (reset_mask & RADEON_RESET_SEM)
5288                 srbm_soft_reset |= SOFT_RESET_SEM;
5289
5290         if (reset_mask & RADEON_RESET_IH)
5291                 srbm_soft_reset |= SOFT_RESET_IH;
5292
5293         if (reset_mask & RADEON_RESET_GRBM)
5294                 srbm_soft_reset |= SOFT_RESET_GRBM;
5295
5296         if (reset_mask & RADEON_RESET_VMC)
5297                 srbm_soft_reset |= SOFT_RESET_VMC;
5298
5299         if (!(rdev->flags & RADEON_IS_IGP)) {
5300                 if (reset_mask & RADEON_RESET_MC)
5301                         srbm_soft_reset |= SOFT_RESET_MC;
5302         }
5303
5304         if (grbm_soft_reset) {
5305                 tmp = RREG32(GRBM_SOFT_RESET);
5306                 tmp |= grbm_soft_reset;
5307                 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
5308                 WREG32(GRBM_SOFT_RESET, tmp);
5309                 tmp = RREG32(GRBM_SOFT_RESET);
5310
5311                 udelay(50);
5312
5313                 tmp &= ~grbm_soft_reset;
5314                 WREG32(GRBM_SOFT_RESET, tmp);
5315                 tmp = RREG32(GRBM_SOFT_RESET);
5316         }
5317
5318         if (srbm_soft_reset) {
5319                 tmp = RREG32(SRBM_SOFT_RESET);
5320                 tmp |= srbm_soft_reset;
5321                 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
5322                 WREG32(SRBM_SOFT_RESET, tmp);
5323                 tmp = RREG32(SRBM_SOFT_RESET);
5324
5325                 udelay(50);
5326
5327                 tmp &= ~srbm_soft_reset;
5328                 WREG32(SRBM_SOFT_RESET, tmp);
5329                 tmp = RREG32(SRBM_SOFT_RESET);
5330         }
5331
5332         /* Wait a little for things to settle down */
5333         udelay(50);
5334
5335         evergreen_mc_resume(rdev, &save);
5336         udelay(50);
5337
5338         cik_print_gpu_status_regs(rdev);
5339 }
5340
5341 struct kv_reset_save_regs {
5342         u32 gmcon_reng_execute;
5343         u32 gmcon_misc;
5344         u32 gmcon_misc3;
5345 };
5346
5347 static void kv_save_regs_for_reset(struct radeon_device *rdev,
5348                                    struct kv_reset_save_regs *save)
5349 {
5350         save->gmcon_reng_execute = RREG32(GMCON_RENG_EXECUTE);
5351         save->gmcon_misc = RREG32(GMCON_MISC);
5352         save->gmcon_misc3 = RREG32(GMCON_MISC3);
5353
5354         WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute & ~RENG_EXECUTE_ON_PWR_UP);
5355         WREG32(GMCON_MISC, save->gmcon_misc & ~(RENG_EXECUTE_ON_REG_UPDATE |
5356                                                 STCTRL_STUTTER_EN));
5357 }
5358
5359 static void kv_restore_regs_for_reset(struct radeon_device *rdev,
5360                                       struct kv_reset_save_regs *save)
5361 {
5362         int i;
5363
5364         WREG32(GMCON_PGFSM_WRITE, 0);
5365         WREG32(GMCON_PGFSM_CONFIG, 0x200010ff);
5366
5367         for (i = 0; i < 5; i++)
5368                 WREG32(GMCON_PGFSM_WRITE, 0);
5369
5370         WREG32(GMCON_PGFSM_WRITE, 0);
5371         WREG32(GMCON_PGFSM_CONFIG, 0x300010ff);
5372
5373         for (i = 0; i < 5; i++)
5374                 WREG32(GMCON_PGFSM_WRITE, 0);
5375
5376         WREG32(GMCON_PGFSM_WRITE, 0x210000);
5377         WREG32(GMCON_PGFSM_CONFIG, 0xa00010ff);
5378
5379         for (i = 0; i < 5; i++)
5380                 WREG32(GMCON_PGFSM_WRITE, 0);
5381
5382         WREG32(GMCON_PGFSM_WRITE, 0x21003);
5383         WREG32(GMCON_PGFSM_CONFIG, 0xb00010ff);
5384
5385         for (i = 0; i < 5; i++)
5386                 WREG32(GMCON_PGFSM_WRITE, 0);
5387
5388         WREG32(GMCON_PGFSM_WRITE, 0x2b00);
5389         WREG32(GMCON_PGFSM_CONFIG, 0xc00010ff);
5390
5391         for (i = 0; i < 5; i++)
5392                 WREG32(GMCON_PGFSM_WRITE, 0);
5393
5394         WREG32(GMCON_PGFSM_WRITE, 0);
5395         WREG32(GMCON_PGFSM_CONFIG, 0xd00010ff);
5396
5397         for (i = 0; i < 5; i++)
5398                 WREG32(GMCON_PGFSM_WRITE, 0);
5399
5400         WREG32(GMCON_PGFSM_WRITE, 0x420000);
5401         WREG32(GMCON_PGFSM_CONFIG, 0x100010ff);
5402
5403         for (i = 0; i < 5; i++)
5404                 WREG32(GMCON_PGFSM_WRITE, 0);
5405
5406         WREG32(GMCON_PGFSM_WRITE, 0x120202);
5407         WREG32(GMCON_PGFSM_CONFIG, 0x500010ff);
5408
5409         for (i = 0; i < 5; i++)
5410                 WREG32(GMCON_PGFSM_WRITE, 0);
5411
5412         WREG32(GMCON_PGFSM_WRITE, 0x3e3e36);
5413         WREG32(GMCON_PGFSM_CONFIG, 0x600010ff);
5414
5415         for (i = 0; i < 5; i++)
5416                 WREG32(GMCON_PGFSM_WRITE, 0);
5417
5418         WREG32(GMCON_PGFSM_WRITE, 0x373f3e);
5419         WREG32(GMCON_PGFSM_CONFIG, 0x700010ff);
5420
5421         for (i = 0; i < 5; i++)
5422                 WREG32(GMCON_PGFSM_WRITE, 0);
5423
5424         WREG32(GMCON_PGFSM_WRITE, 0x3e1332);
5425         WREG32(GMCON_PGFSM_CONFIG, 0xe00010ff);
5426
5427         WREG32(GMCON_MISC3, save->gmcon_misc3);
5428         WREG32(GMCON_MISC, save->gmcon_misc);
5429         WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute);
5430 }
5431
5432 static void cik_gpu_pci_config_reset(struct radeon_device *rdev)
5433 {
5434         struct evergreen_mc_save save;
5435         struct kv_reset_save_regs kv_save = { 0 };
5436         u32 tmp, i;
5437
5438         dev_info(rdev->dev, "GPU pci config reset\n");
5439
5440         /* disable dpm? */
5441
5442         /* disable cg/pg */
5443         cik_fini_pg(rdev);
5444         cik_fini_cg(rdev);
5445
5446         /* Disable GFX parsing/prefetching */
5447         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
5448
5449         /* Disable MEC parsing/prefetching */
5450         WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
5451
5452         /* sdma0 */
5453         tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5454         tmp |= SDMA_HALT;
5455         WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5456         /* sdma1 */
5457         tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5458         tmp |= SDMA_HALT;
5459         WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5460         /* XXX other engines? */
5461
5462         /* halt the rlc, disable cp internal ints */
5463         cik_rlc_stop(rdev);
5464
5465         udelay(50);
5466
5467         /* disable mem access */
5468         evergreen_mc_stop(rdev, &save);
5469         if (evergreen_mc_wait_for_idle(rdev)) {
5470                 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
5471         }
5472
5473         if (rdev->flags & RADEON_IS_IGP)
5474                 kv_save_regs_for_reset(rdev, &kv_save);
5475
5476         /* disable BM */
5477         pci_clear_master(rdev->pdev);
5478         /* reset */
5479         radeon_pci_config_reset(rdev);
5480
5481         udelay(100);
5482
5483         /* wait for asic to come out of reset */
5484         for (i = 0; i < rdev->usec_timeout; i++) {
5485                 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
5486                         break;
5487                 udelay(1);
5488         }
5489
5490         /* does asic init need to be run first??? */
5491         if (rdev->flags & RADEON_IS_IGP)
5492                 kv_restore_regs_for_reset(rdev, &kv_save);
5493 }
5494
5495 /**
5496  * cik_asic_reset - soft reset GPU
5497  *
5498  * @rdev: radeon_device pointer
5499  *
5500  * Look up which blocks are hung and attempt
5501  * to reset them.
5502  * Returns 0 for success.
5503  */
5504 int cik_asic_reset(struct radeon_device *rdev)
5505 {
5506         u32 reset_mask;
5507
5508         reset_mask = cik_gpu_check_soft_reset(rdev);
5509
5510         if (reset_mask)
5511                 r600_set_bios_scratch_engine_hung(rdev, true);
5512
5513         /* try soft reset */
5514         cik_gpu_soft_reset(rdev, reset_mask);
5515
5516         reset_mask = cik_gpu_check_soft_reset(rdev);
5517
5518         /* try pci config reset */
5519         if (reset_mask && radeon_hard_reset)
5520                 cik_gpu_pci_config_reset(rdev);
5521
5522         reset_mask = cik_gpu_check_soft_reset(rdev);
5523
5524         if (!reset_mask)
5525                 r600_set_bios_scratch_engine_hung(rdev, false);
5526
5527         return 0;
5528 }
5529
5530 /**
5531  * cik_gfx_is_lockup - check if the 3D engine is locked up
5532  *
5533  * @rdev: radeon_device pointer
5534  * @ring: radeon_ring structure holding ring information
5535  *
5536  * Check if the 3D engine is locked up (CIK).
5537  * Returns true if the engine is locked, false if not.
5538  */
5539 bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
5540 {
5541         u32 reset_mask = cik_gpu_check_soft_reset(rdev);
5542
5543         if (!(reset_mask & (RADEON_RESET_GFX |
5544                             RADEON_RESET_COMPUTE |
5545                             RADEON_RESET_CP))) {
5546                 radeon_ring_lockup_update(rdev, ring);
5547                 return false;
5548         }
5549         return radeon_ring_test_lockup(rdev, ring);
5550 }
5551
5552 /* MC */
5553 /**
5554  * cik_mc_program - program the GPU memory controller
5555  *
5556  * @rdev: radeon_device pointer
5557  *
5558  * Set the location of vram, gart, and AGP in the GPU's
5559  * physical address space (CIK).
5560  */
5561 static void cik_mc_program(struct radeon_device *rdev)
5562 {
5563         struct evergreen_mc_save save;
5564         u32 tmp;
5565         int i, j;
5566
5567         /* Initialize HDP */
5568         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
5569                 WREG32((0x2c14 + j), 0x00000000);
5570                 WREG32((0x2c18 + j), 0x00000000);
5571                 WREG32((0x2c1c + j), 0x00000000);
5572                 WREG32((0x2c20 + j), 0x00000000);
5573                 WREG32((0x2c24 + j), 0x00000000);
5574         }
5575         WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
5576
5577         evergreen_mc_stop(rdev, &save);
5578         if (radeon_mc_wait_for_idle(rdev)) {
5579                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5580         }
5581         /* Lockout access through VGA aperture*/
5582         WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
5583         /* Update configuration */
5584         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
5585                rdev->mc.vram_start >> 12);
5586         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
5587                rdev->mc.vram_end >> 12);
5588         WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
5589                rdev->vram_scratch.gpu_addr >> 12);
5590         tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
5591         tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
5592         WREG32(MC_VM_FB_LOCATION, tmp);
5593         /* XXX double check these! */
5594         WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
5595         WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
5596         WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
5597         WREG32(MC_VM_AGP_BASE, 0);
5598         WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
5599         WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
5600         if (radeon_mc_wait_for_idle(rdev)) {
5601                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5602         }
5603         evergreen_mc_resume(rdev, &save);
5604         /* we need to own VRAM, so turn off the VGA renderer here
5605          * to stop it overwriting our objects */
5606         rv515_vga_render_disable(rdev);
5607 }
5608
5609 /**
5610  * cik_mc_init - initialize the memory controller driver params
5611  *
5612  * @rdev: radeon_device pointer
5613  *
5614  * Look up the amount of vram, vram width, and decide how to place
5615  * vram and gart within the GPU's physical address space (CIK).
5616  * Returns 0 for success.
5617  */
5618 static int cik_mc_init(struct radeon_device *rdev)
5619 {
5620         u32 tmp;
5621         int chansize, numchan;
5622
5623         /* Get VRAM informations */
5624         rdev->mc.vram_is_ddr = true;
5625         tmp = RREG32(MC_ARB_RAMCFG);
5626         if (tmp & CHANSIZE_MASK) {
5627                 chansize = 64;
5628         } else {
5629                 chansize = 32;
5630         }
5631         tmp = RREG32(MC_SHARED_CHMAP);
5632         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
5633         case 0:
5634         default:
5635                 numchan = 1;
5636                 break;
5637         case 1:
5638                 numchan = 2;
5639                 break;
5640         case 2:
5641                 numchan = 4;
5642                 break;
5643         case 3:
5644                 numchan = 8;
5645                 break;
5646         case 4:
5647                 numchan = 3;
5648                 break;
5649         case 5:
5650                 numchan = 6;
5651                 break;
5652         case 6:
5653                 numchan = 10;
5654                 break;
5655         case 7:
5656                 numchan = 12;
5657                 break;
5658         case 8:
5659                 numchan = 16;
5660                 break;
5661         }
5662         rdev->mc.vram_width = numchan * chansize;
5663         /* Could aper size report 0 ? */
5664         rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
5665         rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
5666         /* size in MB on si */
5667         rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5668         rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5669         rdev->mc.visible_vram_size = rdev->mc.aper_size;
5670         si_vram_gtt_location(rdev, &rdev->mc);
5671         radeon_update_bandwidth_info(rdev);
5672
5673         return 0;
5674 }
5675
5676 /*
5677  * GART
5678  * VMID 0 is the physical GPU addresses as used by the kernel.
5679  * VMIDs 1-15 are used for userspace clients and are handled
5680  * by the radeon vm/hsa code.
5681  */
5682 /**
5683  * cik_pcie_gart_tlb_flush - gart tlb flush callback
5684  *
5685  * @rdev: radeon_device pointer
5686  *
5687  * Flush the TLB for the VMID 0 page table (CIK).
5688  */
5689 void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
5690 {
5691         /* flush hdp cache */
5692         WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
5693
5694         /* bits 0-15 are the VM contexts0-15 */
5695         WREG32(VM_INVALIDATE_REQUEST, 0x1);
5696 }
5697
5698 /**
5699  * cik_pcie_gart_enable - gart enable
5700  *
5701  * @rdev: radeon_device pointer
5702  *
5703  * This sets up the TLBs, programs the page tables for VMID0,
5704  * sets up the hw for VMIDs 1-15 which are allocated on
5705  * demand, and sets up the global locations for the LDS, GDS,
5706  * and GPUVM for FSA64 clients (CIK).
5707  * Returns 0 for success, errors for failure.
5708  */
5709 static int cik_pcie_gart_enable(struct radeon_device *rdev)
5710 {
5711         int r, i;
5712
5713         if (rdev->gart.robj == NULL) {
5714                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
5715                 return -EINVAL;
5716         }
5717         r = radeon_gart_table_vram_pin(rdev);
5718         if (r)
5719                 return r;
5720         /* Setup TLB control */
5721         WREG32(MC_VM_MX_L1_TLB_CNTL,
5722                (0xA << 7) |
5723                ENABLE_L1_TLB |
5724                ENABLE_L1_FRAGMENT_PROCESSING |
5725                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5726                ENABLE_ADVANCED_DRIVER_MODEL |
5727                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5728         /* Setup L2 cache */
5729         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
5730                ENABLE_L2_FRAGMENT_PROCESSING |
5731                ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5732                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5733                EFFECTIVE_L2_QUEUE_SIZE(7) |
5734                CONTEXT1_IDENTITY_ACCESS_MODE(1));
5735         WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
5736         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5737                BANK_SELECT(4) |
5738                L2_CACHE_BIGK_FRAGMENT_SIZE(4));
5739         /* setup context0 */
5740         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
5741         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
5742         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
5743         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
5744                         (u32)(rdev->dummy_page.addr >> 12));
5745         WREG32(VM_CONTEXT0_CNTL2, 0);
5746         WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
5747                                   RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
5748
5749         WREG32(0x15D4, 0);
5750         WREG32(0x15D8, 0);
5751         WREG32(0x15DC, 0);
5752
5753         /* empty context1-15 */
5754         /* FIXME start with 4G, once using 2 level pt switch to full
5755          * vm size space
5756          */
5757         /* set vm size, must be a multiple of 4 */
5758         WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
5759         WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
5760         for (i = 1; i < 16; i++) {
5761                 if (i < 8)
5762                         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
5763                                rdev->gart.table_addr >> 12);
5764                 else
5765                         WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
5766                                rdev->gart.table_addr >> 12);
5767         }
5768
5769         /* enable context1-15 */
5770         WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
5771                (u32)(rdev->dummy_page.addr >> 12));
5772         WREG32(VM_CONTEXT1_CNTL2, 4);
5773         WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
5774                                 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
5775                                 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5776                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5777                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5778                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5779                                 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
5780                                 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
5781                                 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
5782                                 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
5783                                 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
5784                                 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
5785                                 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5786                                 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
5787
5788         if (rdev->family == CHIP_KAVERI) {
5789                 u32 tmp = RREG32(CHUB_CONTROL);
5790                 tmp &= ~BYPASS_VM;
5791                 WREG32(CHUB_CONTROL, tmp);
5792         }
5793
5794         /* XXX SH_MEM regs */
5795         /* where to put LDS, scratch, GPUVM in FSA64 space */
5796         mutex_lock(&rdev->srbm_mutex);
5797         for (i = 0; i < 16; i++) {
5798                 cik_srbm_select(rdev, 0, 0, 0, i);
5799                 /* CP and shaders */
5800                 WREG32(SH_MEM_CONFIG, 0);
5801                 WREG32(SH_MEM_APE1_BASE, 1);
5802                 WREG32(SH_MEM_APE1_LIMIT, 0);
5803                 WREG32(SH_MEM_BASES, 0);
5804                 /* SDMA GFX */
5805                 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
5806                 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
5807                 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
5808                 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
5809                 /* XXX SDMA RLC - todo */
5810         }
5811         cik_srbm_select(rdev, 0, 0, 0, 0);
5812         mutex_unlock(&rdev->srbm_mutex);
5813
5814         cik_pcie_gart_tlb_flush(rdev);
5815         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
5816                  (unsigned)(rdev->mc.gtt_size >> 20),
5817                  (unsigned long long)rdev->gart.table_addr);
5818         rdev->gart.ready = true;
5819         return 0;
5820 }
5821
5822 /**
5823  * cik_pcie_gart_disable - gart disable
5824  *
5825  * @rdev: radeon_device pointer
5826  *
5827  * This disables all VM page table (CIK).
5828  */
5829 static void cik_pcie_gart_disable(struct radeon_device *rdev)
5830 {
5831         /* Disable all tables */
5832         WREG32(VM_CONTEXT0_CNTL, 0);
5833         WREG32(VM_CONTEXT1_CNTL, 0);
5834         /* Setup TLB control */
5835         WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5836                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5837         /* Setup L2 cache */
5838         WREG32(VM_L2_CNTL,
5839                ENABLE_L2_FRAGMENT_PROCESSING |
5840                ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5841                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5842                EFFECTIVE_L2_QUEUE_SIZE(7) |
5843                CONTEXT1_IDENTITY_ACCESS_MODE(1));
5844         WREG32(VM_L2_CNTL2, 0);
5845         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5846                L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5847         radeon_gart_table_vram_unpin(rdev);
5848 }
5849
5850 /**
5851  * cik_pcie_gart_fini - vm fini callback
5852  *
5853  * @rdev: radeon_device pointer
5854  *
5855  * Tears down the driver GART/VM setup (CIK).
5856  */
5857 static void cik_pcie_gart_fini(struct radeon_device *rdev)
5858 {
5859         cik_pcie_gart_disable(rdev);
5860         radeon_gart_table_vram_free(rdev);
5861         radeon_gart_fini(rdev);
5862 }
5863
5864 /* vm parser */
5865 /**
5866  * cik_ib_parse - vm ib_parse callback
5867  *
5868  * @rdev: radeon_device pointer
5869  * @ib: indirect buffer pointer
5870  *
5871  * CIK uses hw IB checking so this is a nop (CIK).
5872  */
5873 int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
5874 {
5875         return 0;
5876 }
5877
5878 /*
5879  * vm
5880  * VMID 0 is the physical GPU addresses as used by the kernel.
5881  * VMIDs 1-15 are used for userspace clients and are handled
5882  * by the radeon vm/hsa code.
5883  */
5884 /**
5885  * cik_vm_init - cik vm init callback
5886  *
5887  * @rdev: radeon_device pointer
5888  *
5889  * Inits cik specific vm parameters (number of VMs, base of vram for
5890  * VMIDs 1-15) (CIK).
5891  * Returns 0 for success.
5892  */
5893 int cik_vm_init(struct radeon_device *rdev)
5894 {
5895         /* number of VMs */
5896         rdev->vm_manager.nvm = 16;
5897         /* base offset of vram pages */
5898         if (rdev->flags & RADEON_IS_IGP) {
5899                 u64 tmp = RREG32(MC_VM_FB_OFFSET);
5900                 tmp <<= 22;
5901                 rdev->vm_manager.vram_base_offset = tmp;
5902         } else
5903                 rdev->vm_manager.vram_base_offset = 0;
5904
5905         return 0;
5906 }
5907
5908 /**
5909  * cik_vm_fini - cik vm fini callback
5910  *
5911  * @rdev: radeon_device pointer
5912  *
5913  * Tear down any asic specific VM setup (CIK).
5914  */
5915 void cik_vm_fini(struct radeon_device *rdev)
5916 {
5917 }
5918
5919 /**
5920  * cik_vm_decode_fault - print human readable fault info
5921  *
5922  * @rdev: radeon_device pointer
5923  * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
5924  * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
5925  *
5926  * Print human readable fault information (CIK).
5927  */
5928 static void cik_vm_decode_fault(struct radeon_device *rdev,
5929                                 u32 status, u32 addr, u32 mc_client)
5930 {
5931         u32 mc_id;
5932         u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
5933         u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
5934         char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
5935                 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
5936
5937         if (rdev->family == CHIP_HAWAII)
5938                 mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5939         else
5940                 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5941
5942         printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
5943                protections, vmid, addr,
5944                (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
5945                block, mc_client, mc_id);
5946 }
5947
5948 /**
5949  * cik_vm_flush - cik vm flush using the CP
5950  *
5951  * @rdev: radeon_device pointer
5952  *
5953  * Update the page table base and flush the VM TLB
5954  * using the CP (CIK).
5955  */
5956 void cik_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
5957 {
5958         struct radeon_ring *ring = &rdev->ring[ridx];
5959         int usepfp = (ridx == RADEON_RING_TYPE_GFX_INDEX);
5960
5961         if (vm == NULL)
5962                 return;
5963
5964         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5965         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
5966                                  WRITE_DATA_DST_SEL(0)));
5967         if (vm->id < 8) {
5968                 radeon_ring_write(ring,
5969                                   (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
5970         } else {
5971                 radeon_ring_write(ring,
5972                                   (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
5973         }
5974         radeon_ring_write(ring, 0);
5975         radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
5976
5977         /* update SH_MEM_* regs */
5978         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5979         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
5980                                  WRITE_DATA_DST_SEL(0)));
5981         radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5982         radeon_ring_write(ring, 0);
5983         radeon_ring_write(ring, VMID(vm->id));
5984
5985         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
5986         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
5987                                  WRITE_DATA_DST_SEL(0)));
5988         radeon_ring_write(ring, SH_MEM_BASES >> 2);
5989         radeon_ring_write(ring, 0);
5990
5991         radeon_ring_write(ring, 0); /* SH_MEM_BASES */
5992         radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */
5993         radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
5994         radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
5995
5996         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5997         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
5998                                  WRITE_DATA_DST_SEL(0)));
5999         radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
6000         radeon_ring_write(ring, 0);
6001         radeon_ring_write(ring, VMID(0));
6002
6003         /* HDP flush */
6004         cik_hdp_flush_cp_ring_emit(rdev, ridx);
6005
6006         /* bits 0-15 are the VM contexts0-15 */
6007         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
6008         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
6009                                  WRITE_DATA_DST_SEL(0)));
6010         radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
6011         radeon_ring_write(ring, 0);
6012         radeon_ring_write(ring, 1 << vm->id);
6013
6014         /* compute doesn't have PFP */
6015         if (usepfp) {
6016                 /* sync PFP to ME, otherwise we might get invalid PFP reads */
6017                 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
6018                 radeon_ring_write(ring, 0x0);
6019         }
6020 }
6021
6022 /*
6023  * RLC
6024  * The RLC is a multi-purpose microengine that handles a
6025  * variety of functions, the most important of which is
6026  * the interrupt controller.
6027  */
6028 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
6029                                           bool enable)
6030 {
6031         u32 tmp = RREG32(CP_INT_CNTL_RING0);
6032
6033         if (enable)
6034                 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6035         else
6036                 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6037         WREG32(CP_INT_CNTL_RING0, tmp);
6038 }
6039
6040 static void cik_enable_lbpw(struct radeon_device *rdev, bool enable)
6041 {
6042         u32 tmp;
6043
6044         tmp = RREG32(RLC_LB_CNTL);
6045         if (enable)
6046                 tmp |= LOAD_BALANCE_ENABLE;
6047         else
6048                 tmp &= ~LOAD_BALANCE_ENABLE;
6049         WREG32(RLC_LB_CNTL, tmp);
6050 }
6051
6052 static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)
6053 {
6054         u32 i, j, k;
6055         u32 mask;
6056
6057         for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6058                 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6059                         cik_select_se_sh(rdev, i, j);
6060                         for (k = 0; k < rdev->usec_timeout; k++) {
6061                                 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
6062                                         break;
6063                                 udelay(1);
6064                         }
6065                 }
6066         }
6067         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6068
6069         mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
6070         for (k = 0; k < rdev->usec_timeout; k++) {
6071                 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
6072                         break;
6073                 udelay(1);
6074         }
6075 }
6076
6077 static void cik_update_rlc(struct radeon_device *rdev, u32 rlc)
6078 {
6079         u32 tmp;
6080
6081         tmp = RREG32(RLC_CNTL);
6082         if (tmp != rlc)
6083                 WREG32(RLC_CNTL, rlc);
6084 }
6085
6086 static u32 cik_halt_rlc(struct radeon_device *rdev)
6087 {
6088         u32 data, orig;
6089
6090         orig = data = RREG32(RLC_CNTL);
6091
6092         if (data & RLC_ENABLE) {
6093                 u32 i;
6094
6095                 data &= ~RLC_ENABLE;
6096                 WREG32(RLC_CNTL, data);
6097
6098                 for (i = 0; i < rdev->usec_timeout; i++) {
6099                         if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0)
6100                                 break;
6101                         udelay(1);
6102                 }
6103
6104                 cik_wait_for_rlc_serdes(rdev);
6105         }
6106
6107         return orig;
6108 }
6109
6110 void cik_enter_rlc_safe_mode(struct radeon_device *rdev)
6111 {
6112         u32 tmp, i, mask;
6113
6114         tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE);
6115         WREG32(RLC_GPR_REG2, tmp);
6116
6117         mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS;
6118         for (i = 0; i < rdev->usec_timeout; i++) {
6119                 if ((RREG32(RLC_GPM_STAT) & mask) == mask)
6120                         break;
6121                 udelay(1);
6122         }
6123
6124         for (i = 0; i < rdev->usec_timeout; i++) {
6125                 if ((RREG32(RLC_GPR_REG2) & REQ) == 0)
6126                         break;
6127                 udelay(1);
6128         }
6129 }
6130
6131 void cik_exit_rlc_safe_mode(struct radeon_device *rdev)
6132 {
6133         u32 tmp;
6134
6135         tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE);
6136         WREG32(RLC_GPR_REG2, tmp);
6137 }
6138
6139 /**
6140  * cik_rlc_stop - stop the RLC ME
6141  *
6142  * @rdev: radeon_device pointer
6143  *
6144  * Halt the RLC ME (MicroEngine) (CIK).
6145  */
6146 static void cik_rlc_stop(struct radeon_device *rdev)
6147 {
6148         WREG32(RLC_CNTL, 0);
6149
6150         cik_enable_gui_idle_interrupt(rdev, false);
6151
6152         cik_wait_for_rlc_serdes(rdev);
6153 }
6154
6155 /**
6156  * cik_rlc_start - start the RLC ME
6157  *
6158  * @rdev: radeon_device pointer
6159  *
6160  * Unhalt the RLC ME (MicroEngine) (CIK).
6161  */
6162 static void cik_rlc_start(struct radeon_device *rdev)
6163 {
6164         WREG32(RLC_CNTL, RLC_ENABLE);
6165
6166         cik_enable_gui_idle_interrupt(rdev, true);
6167
6168         udelay(50);
6169 }
6170
6171 /**
6172  * cik_rlc_resume - setup the RLC hw
6173  *
6174  * @rdev: radeon_device pointer
6175  *
6176  * Initialize the RLC registers, load the ucode,
6177  * and start the RLC (CIK).
6178  * Returns 0 for success, -EINVAL if the ucode is not available.
6179  */
6180 static int cik_rlc_resume(struct radeon_device *rdev)
6181 {
6182         u32 i, size, tmp;
6183
6184         if (!rdev->rlc_fw)
6185                 return -EINVAL;
6186
6187         cik_rlc_stop(rdev);
6188
6189         /* disable CG */
6190         tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
6191         WREG32(RLC_CGCG_CGLS_CTRL, tmp);
6192
6193         si_rlc_reset(rdev);
6194
6195         cik_init_pg(rdev);
6196
6197         cik_init_cg(rdev);
6198
6199         WREG32(RLC_LB_CNTR_INIT, 0);
6200         WREG32(RLC_LB_CNTR_MAX, 0x00008000);
6201
6202         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6203         WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
6204         WREG32(RLC_LB_PARAMS, 0x00600408);
6205         WREG32(RLC_LB_CNTL, 0x80000004);
6206
6207         WREG32(RLC_MC_CNTL, 0);
6208         WREG32(RLC_UCODE_CNTL, 0);
6209
6210         if (rdev->new_fw) {
6211                 const struct rlc_firmware_header_v1_0 *hdr =
6212                         (const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data;
6213                 const __le32 *fw_data = (const __le32 *)
6214                         (rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6215
6216                 radeon_ucode_print_rlc_hdr(&hdr->header);
6217
6218                 size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
6219                 WREG32(RLC_GPM_UCODE_ADDR, 0);
6220                 for (i = 0; i < size; i++)
6221                         WREG32(RLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++));
6222                 WREG32(RLC_GPM_UCODE_ADDR, 0);
6223         } else {
6224                 const __be32 *fw_data;
6225
6226                 switch (rdev->family) {
6227                 case CHIP_BONAIRE:
6228                 case CHIP_HAWAII:
6229                 default:
6230                         size = BONAIRE_RLC_UCODE_SIZE;
6231                         break;
6232                 case CHIP_KAVERI:
6233                         size = KV_RLC_UCODE_SIZE;
6234                         break;
6235                 case CHIP_KABINI:
6236                         size = KB_RLC_UCODE_SIZE;
6237                         break;
6238                 case CHIP_MULLINS:
6239                         size = ML_RLC_UCODE_SIZE;
6240                         break;
6241                 }
6242
6243                 fw_data = (const __be32 *)rdev->rlc_fw->data;
6244                 WREG32(RLC_GPM_UCODE_ADDR, 0);
6245                 for (i = 0; i < size; i++)
6246                         WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
6247                 WREG32(RLC_GPM_UCODE_ADDR, 0);
6248         }
6249
6250         /* XXX - find out what chips support lbpw */
6251         cik_enable_lbpw(rdev, false);
6252
6253         if (rdev->family == CHIP_BONAIRE)
6254                 WREG32(RLC_DRIVER_DMA_STATUS, 0);
6255
6256         cik_rlc_start(rdev);
6257
6258         return 0;
6259 }
6260
6261 static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)
6262 {
6263         u32 data, orig, tmp, tmp2;
6264
6265         orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
6266
6267         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
6268                 cik_enable_gui_idle_interrupt(rdev, true);
6269
6270                 tmp = cik_halt_rlc(rdev);
6271
6272                 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6273                 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6274                 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6275                 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
6276                 WREG32(RLC_SERDES_WR_CTRL, tmp2);
6277
6278                 cik_update_rlc(rdev, tmp);
6279
6280                 data |= CGCG_EN | CGLS_EN;
6281         } else {
6282                 cik_enable_gui_idle_interrupt(rdev, false);
6283
6284                 RREG32(CB_CGTT_SCLK_CTRL);
6285                 RREG32(CB_CGTT_SCLK_CTRL);
6286                 RREG32(CB_CGTT_SCLK_CTRL);
6287                 RREG32(CB_CGTT_SCLK_CTRL);
6288
6289                 data &= ~(CGCG_EN | CGLS_EN);
6290         }
6291
6292         if (orig != data)
6293                 WREG32(RLC_CGCG_CGLS_CTRL, data);
6294
6295 }
6296
6297 static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)
6298 {
6299         u32 data, orig, tmp = 0;
6300
6301         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
6302                 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {
6303                         if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
6304                                 orig = data = RREG32(CP_MEM_SLP_CNTL);
6305                                 data |= CP_MEM_LS_EN;
6306                                 if (orig != data)
6307                                         WREG32(CP_MEM_SLP_CNTL, data);
6308                         }
6309                 }
6310
6311                 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
6312                 data &= 0xfffffffd;
6313                 if (orig != data)
6314                         WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
6315
6316                 tmp = cik_halt_rlc(rdev);
6317
6318                 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6319                 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6320                 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6321                 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
6322                 WREG32(RLC_SERDES_WR_CTRL, data);
6323
6324                 cik_update_rlc(rdev, tmp);
6325
6326                 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {
6327                         orig = data = RREG32(CGTS_SM_CTRL_REG);
6328                         data &= ~SM_MODE_MASK;
6329                         data |= SM_MODE(0x2);
6330                         data |= SM_MODE_ENABLE;
6331                         data &= ~CGTS_OVERRIDE;
6332                         if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&
6333                             (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))
6334                                 data &= ~CGTS_LS_OVERRIDE;
6335                         data &= ~ON_MONITOR_ADD_MASK;
6336                         data |= ON_MONITOR_ADD_EN;
6337                         data |= ON_MONITOR_ADD(0x96);
6338                         if (orig != data)
6339                                 WREG32(CGTS_SM_CTRL_REG, data);
6340                 }
6341         } else {
6342                 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
6343                 data |= 0x00000002;
6344                 if (orig != data)
6345                         WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
6346
6347                 data = RREG32(RLC_MEM_SLP_CNTL);
6348                 if (data & RLC_MEM_LS_EN) {
6349                         data &= ~RLC_MEM_LS_EN;
6350                         WREG32(RLC_MEM_SLP_CNTL, data);
6351                 }
6352
6353                 data = RREG32(CP_MEM_SLP_CNTL);
6354                 if (data & CP_MEM_LS_EN) {
6355                         data &= ~CP_MEM_LS_EN;
6356                         WREG32(CP_MEM_SLP_CNTL, data);
6357                 }
6358
6359                 orig = data = RREG32(CGTS_SM_CTRL_REG);
6360                 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE;
6361                 if (orig != data)
6362                         WREG32(CGTS_SM_CTRL_REG, data);
6363
6364                 tmp = cik_halt_rlc(rdev);
6365
6366                 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6367                 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6368                 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6369                 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
6370                 WREG32(RLC_SERDES_WR_CTRL, data);
6371
6372                 cik_update_rlc(rdev, tmp);
6373         }
6374 }
6375
6376 static const u32 mc_cg_registers[] =
6377 {
6378         MC_HUB_MISC_HUB_CG,
6379         MC_HUB_MISC_SIP_CG,
6380         MC_HUB_MISC_VM_CG,
6381         MC_XPB_CLK_GAT,
6382         ATC_MISC_CG,
6383         MC_CITF_MISC_WR_CG,
6384         MC_CITF_MISC_RD_CG,
6385         MC_CITF_MISC_VM_CG,
6386         VM_L2_CG,
6387 };
6388
6389 static void cik_enable_mc_ls(struct radeon_device *rdev,
6390                              bool enable)
6391 {
6392         int i;
6393         u32 orig, data;
6394
6395         for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6396                 orig = data = RREG32(mc_cg_registers[i]);
6397                 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
6398                         data |= MC_LS_ENABLE;
6399                 else
6400                         data &= ~MC_LS_ENABLE;
6401                 if (data != orig)
6402                         WREG32(mc_cg_registers[i], data);
6403         }
6404 }
6405
6406 static void cik_enable_mc_mgcg(struct radeon_device *rdev,
6407                                bool enable)
6408 {
6409         int i;
6410         u32 orig, data;
6411
6412         for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6413                 orig = data = RREG32(mc_cg_registers[i]);
6414                 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
6415                         data |= MC_CG_ENABLE;
6416                 else
6417                         data &= ~MC_CG_ENABLE;
6418                 if (data != orig)
6419                         WREG32(mc_cg_registers[i], data);
6420         }
6421 }
6422
6423 static void cik_enable_sdma_mgcg(struct radeon_device *rdev,
6424                                  bool enable)
6425 {
6426         u32 orig, data;
6427
6428         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
6429                 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);
6430                 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);
6431         } else {
6432                 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET);
6433                 data |= 0xff000000;
6434                 if (data != orig)
6435                         WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data);
6436
6437                 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET);
6438                 data |= 0xff000000;
6439                 if (data != orig)
6440                         WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data);
6441         }
6442 }
6443
6444 static void cik_enable_sdma_mgls(struct radeon_device *rdev,
6445                                  bool enable)
6446 {
6447         u32 orig, data;
6448
6449         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {
6450                 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6451                 data |= 0x100;
6452                 if (orig != data)
6453                         WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6454
6455                 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6456                 data |= 0x100;
6457                 if (orig != data)
6458                         WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6459         } else {
6460                 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6461                 data &= ~0x100;
6462                 if (orig != data)
6463                         WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6464
6465                 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6466                 data &= ~0x100;
6467                 if (orig != data)
6468                         WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6469         }
6470 }
6471
6472 static void cik_enable_uvd_mgcg(struct radeon_device *rdev,
6473                                 bool enable)
6474 {
6475         u32 orig, data;
6476
6477         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
6478                 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6479                 data = 0xfff;
6480                 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6481
6482                 orig = data = RREG32(UVD_CGC_CTRL);
6483                 data |= DCM;
6484                 if (orig != data)
6485                         WREG32(UVD_CGC_CTRL, data);
6486         } else {
6487                 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6488                 data &= ~0xfff;
6489                 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6490
6491                 orig = data = RREG32(UVD_CGC_CTRL);
6492                 data &= ~DCM;
6493                 if (orig != data)
6494                         WREG32(UVD_CGC_CTRL, data);
6495         }
6496 }
6497
6498 static void cik_enable_bif_mgls(struct radeon_device *rdev,
6499                                bool enable)
6500 {
6501         u32 orig, data;
6502
6503         orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
6504
6505         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
6506                 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
6507                         REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
6508         else
6509                 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
6510                           REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
6511
6512         if (orig != data)
6513                 WREG32_PCIE_PORT(PCIE_CNTL2, data);
6514 }
6515
6516 static void cik_enable_hdp_mgcg(struct radeon_device *rdev,
6517                                 bool enable)
6518 {
6519         u32 orig, data;
6520
6521         orig = data = RREG32(HDP_HOST_PATH_CNTL);
6522
6523         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
6524                 data &= ~CLOCK_GATING_DIS;
6525         else
6526                 data |= CLOCK_GATING_DIS;
6527
6528         if (orig != data)
6529                 WREG32(HDP_HOST_PATH_CNTL, data);
6530 }
6531
6532 static void cik_enable_hdp_ls(struct radeon_device *rdev,
6533                               bool enable)
6534 {
6535         u32 orig, data;
6536
6537         orig = data = RREG32(HDP_MEM_POWER_LS);
6538
6539         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
6540                 data |= HDP_LS_ENABLE;
6541         else
6542                 data &= ~HDP_LS_ENABLE;
6543
6544         if (orig != data)
6545                 WREG32(HDP_MEM_POWER_LS, data);
6546 }
6547
6548 void cik_update_cg(struct radeon_device *rdev,
6549                    u32 block, bool enable)
6550 {
6551
6552         if (block & RADEON_CG_BLOCK_GFX) {
6553                 cik_enable_gui_idle_interrupt(rdev, false);
6554                 /* order matters! */
6555                 if (enable) {
6556                         cik_enable_mgcg(rdev, true);
6557                         cik_enable_cgcg(rdev, true);
6558                 } else {
6559                         cik_enable_cgcg(rdev, false);
6560                         cik_enable_mgcg(rdev, false);
6561                 }
6562                 cik_enable_gui_idle_interrupt(rdev, true);
6563         }
6564
6565         if (block & RADEON_CG_BLOCK_MC) {
6566                 if (!(rdev->flags & RADEON_IS_IGP)) {
6567                         cik_enable_mc_mgcg(rdev, enable);
6568                         cik_enable_mc_ls(rdev, enable);
6569                 }
6570         }
6571
6572         if (block & RADEON_CG_BLOCK_SDMA) {
6573                 cik_enable_sdma_mgcg(rdev, enable);
6574                 cik_enable_sdma_mgls(rdev, enable);
6575         }
6576
6577         if (block & RADEON_CG_BLOCK_BIF) {
6578                 cik_enable_bif_mgls(rdev, enable);
6579         }
6580
6581         if (block & RADEON_CG_BLOCK_UVD) {
6582                 if (rdev->has_uvd)
6583                         cik_enable_uvd_mgcg(rdev, enable);
6584         }
6585
6586         if (block & RADEON_CG_BLOCK_HDP) {
6587                 cik_enable_hdp_mgcg(rdev, enable);
6588                 cik_enable_hdp_ls(rdev, enable);
6589         }
6590
6591         if (block & RADEON_CG_BLOCK_VCE) {
6592                 vce_v2_0_enable_mgcg(rdev, enable);
6593         }
6594 }
6595
6596 static void cik_init_cg(struct radeon_device *rdev)
6597 {
6598
6599         cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true);
6600
6601         if (rdev->has_uvd)
6602                 si_init_uvd_internal_cg(rdev);
6603
6604         cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6605                              RADEON_CG_BLOCK_SDMA |
6606                              RADEON_CG_BLOCK_BIF |
6607                              RADEON_CG_BLOCK_UVD |
6608                              RADEON_CG_BLOCK_HDP), true);
6609 }
6610
6611 static void cik_fini_cg(struct radeon_device *rdev)
6612 {
6613         cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6614                              RADEON_CG_BLOCK_SDMA |
6615                              RADEON_CG_BLOCK_BIF |
6616                              RADEON_CG_BLOCK_UVD |
6617                              RADEON_CG_BLOCK_HDP), false);
6618
6619         cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false);
6620 }
6621
6622 static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev,
6623                                           bool enable)
6624 {
6625         u32 data, orig;
6626
6627         orig = data = RREG32(RLC_PG_CNTL);
6628         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
6629                 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6630         else
6631                 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6632         if (orig != data)
6633                 WREG32(RLC_PG_CNTL, data);
6634 }
6635
6636 static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev,
6637                                           bool enable)
6638 {
6639         u32 data, orig;
6640
6641         orig = data = RREG32(RLC_PG_CNTL);
6642         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
6643                 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6644         else
6645                 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6646         if (orig != data)
6647                 WREG32(RLC_PG_CNTL, data);
6648 }
6649
6650 static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable)
6651 {
6652         u32 data, orig;
6653
6654         orig = data = RREG32(RLC_PG_CNTL);
6655         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))
6656                 data &= ~DISABLE_CP_PG;
6657         else
6658                 data |= DISABLE_CP_PG;
6659         if (orig != data)
6660                 WREG32(RLC_PG_CNTL, data);
6661 }
6662
6663 static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable)
6664 {
6665         u32 data, orig;
6666
6667         orig = data = RREG32(RLC_PG_CNTL);
6668         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))
6669                 data &= ~DISABLE_GDS_PG;
6670         else
6671                 data |= DISABLE_GDS_PG;
6672         if (orig != data)
6673                 WREG32(RLC_PG_CNTL, data);
6674 }
6675
6676 #define CP_ME_TABLE_SIZE    96
6677 #define CP_ME_TABLE_OFFSET  2048
6678 #define CP_MEC_TABLE_OFFSET 4096
6679
6680 void cik_init_cp_pg_table(struct radeon_device *rdev)
6681 {
6682         volatile u32 *dst_ptr;
6683         int me, i, max_me = 4;
6684         u32 bo_offset = 0;
6685         u32 table_offset, table_size;
6686
6687         if (rdev->family == CHIP_KAVERI)
6688                 max_me = 5;
6689
6690         if (rdev->rlc.cp_table_ptr == NULL)
6691                 return;
6692
6693         /* write the cp table buffer */
6694         dst_ptr = rdev->rlc.cp_table_ptr;
6695         for (me = 0; me < max_me; me++) {
6696                 if (rdev->new_fw) {
6697                         const __le32 *fw_data;
6698                         const struct gfx_firmware_header_v1_0 *hdr;
6699
6700                         if (me == 0) {
6701                                 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
6702                                 fw_data = (const __le32 *)
6703                                         (rdev->ce_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6704                                 table_offset = le32_to_cpu(hdr->jt_offset);
6705                                 table_size = le32_to_cpu(hdr->jt_size);
6706                         } else if (me == 1) {
6707                                 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
6708                                 fw_data = (const __le32 *)
6709                                         (rdev->pfp_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6710                                 table_offset = le32_to_cpu(hdr->jt_offset);
6711                                 table_size = le32_to_cpu(hdr->jt_size);
6712                         } else if (me == 2) {
6713                                 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
6714                                 fw_data = (const __le32 *)
6715                                         (rdev->me_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6716                                 table_offset = le32_to_cpu(hdr->jt_offset);
6717                                 table_size = le32_to_cpu(hdr->jt_size);
6718                         } else if (me == 3) {
6719                                 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;
6720                                 fw_data = (const __le32 *)
6721                                         (rdev->mec_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6722                                 table_offset = le32_to_cpu(hdr->jt_offset);
6723                                 table_size = le32_to_cpu(hdr->jt_size);
6724                         } else {
6725                                 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;
6726                                 fw_data = (const __le32 *)
6727                                         (rdev->mec2_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6728                                 table_offset = le32_to_cpu(hdr->jt_offset);
6729                                 table_size = le32_to_cpu(hdr->jt_size);
6730                         }
6731
6732                         for (i = 0; i < table_size; i ++) {
6733                                 dst_ptr[bo_offset + i] =
6734                                         cpu_to_le32(le32_to_cpu(fw_data[table_offset + i]));
6735                         }
6736                         bo_offset += table_size;
6737                 } else {
6738                         const __be32 *fw_data;
6739                         table_size = CP_ME_TABLE_SIZE;
6740
6741                         if (me == 0) {
6742                                 fw_data = (const __be32 *)rdev->ce_fw->data;
6743                                 table_offset = CP_ME_TABLE_OFFSET;
6744                         } else if (me == 1) {
6745                                 fw_data = (const __be32 *)rdev->pfp_fw->data;
6746                                 table_offset = CP_ME_TABLE_OFFSET;
6747                         } else if (me == 2) {
6748                                 fw_data = (const __be32 *)rdev->me_fw->data;
6749                                 table_offset = CP_ME_TABLE_OFFSET;
6750                         } else {
6751                                 fw_data = (const __be32 *)rdev->mec_fw->data;
6752                                 table_offset = CP_MEC_TABLE_OFFSET;
6753                         }
6754
6755                         for (i = 0; i < table_size; i ++) {
6756                                 dst_ptr[bo_offset + i] =
6757                                         cpu_to_le32(be32_to_cpu(fw_data[table_offset + i]));
6758                         }
6759                         bo_offset += table_size;
6760                 }
6761         }
6762 }
6763
6764 static void cik_enable_gfx_cgpg(struct radeon_device *rdev,
6765                                 bool enable)
6766 {
6767         u32 data, orig;
6768
6769         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
6770                 orig = data = RREG32(RLC_PG_CNTL);
6771                 data |= GFX_PG_ENABLE;
6772                 if (orig != data)
6773                         WREG32(RLC_PG_CNTL, data);
6774
6775                 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6776                 data |= AUTO_PG_EN;
6777                 if (orig != data)
6778                         WREG32(RLC_AUTO_PG_CTRL, data);
6779         } else {
6780                 orig = data = RREG32(RLC_PG_CNTL);
6781                 data &= ~GFX_PG_ENABLE;
6782                 if (orig != data)
6783                         WREG32(RLC_PG_CNTL, data);
6784
6785                 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6786                 data &= ~AUTO_PG_EN;
6787                 if (orig != data)
6788                         WREG32(RLC_AUTO_PG_CTRL, data);
6789
6790                 data = RREG32(DB_RENDER_CONTROL);
6791         }
6792 }
6793
6794 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
6795 {
6796         u32 mask = 0, tmp, tmp1;
6797         int i;
6798
6799         cik_select_se_sh(rdev, se, sh);
6800         tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
6801         tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
6802         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6803
6804         tmp &= 0xffff0000;
6805
6806         tmp |= tmp1;
6807         tmp >>= 16;
6808
6809         for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {
6810                 mask <<= 1;
6811                 mask |= 1;
6812         }
6813
6814         return (~tmp) & mask;
6815 }
6816
6817 static void cik_init_ao_cu_mask(struct radeon_device *rdev)
6818 {
6819         u32 i, j, k, active_cu_number = 0;
6820         u32 mask, counter, cu_bitmap;
6821         u32 tmp = 0;
6822
6823         for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6824                 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6825                         mask = 1;
6826                         cu_bitmap = 0;
6827                         counter = 0;
6828                         for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {
6829                                 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) {
6830                                         if (counter < 2)
6831                                                 cu_bitmap |= mask;
6832                                         counter ++;
6833                                 }
6834                                 mask <<= 1;
6835                         }
6836
6837                         active_cu_number += counter;
6838                         tmp |= (cu_bitmap << (i * 16 + j * 8));
6839                 }
6840         }
6841
6842         WREG32(RLC_PG_AO_CU_MASK, tmp);
6843
6844         tmp = RREG32(RLC_MAX_PG_CU);
6845         tmp &= ~MAX_PU_CU_MASK;
6846         tmp |= MAX_PU_CU(active_cu_number);
6847         WREG32(RLC_MAX_PG_CU, tmp);
6848 }
6849
6850 static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev,
6851                                        bool enable)
6852 {
6853         u32 data, orig;
6854
6855         orig = data = RREG32(RLC_PG_CNTL);
6856         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))
6857                 data |= STATIC_PER_CU_PG_ENABLE;
6858         else
6859                 data &= ~STATIC_PER_CU_PG_ENABLE;
6860         if (orig != data)
6861                 WREG32(RLC_PG_CNTL, data);
6862 }
6863
6864 static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev,
6865                                         bool enable)
6866 {
6867         u32 data, orig;
6868
6869         orig = data = RREG32(RLC_PG_CNTL);
6870         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))
6871                 data |= DYN_PER_CU_PG_ENABLE;
6872         else
6873                 data &= ~DYN_PER_CU_PG_ENABLE;
6874         if (orig != data)
6875                 WREG32(RLC_PG_CNTL, data);
6876 }
6877
6878 #define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
6879 #define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET    0x3D
6880
6881 static void cik_init_gfx_cgpg(struct radeon_device *rdev)
6882 {
6883         u32 data, orig;
6884         u32 i;
6885
6886         if (rdev->rlc.cs_data) {
6887                 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6888                 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));
6889                 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));
6890                 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);
6891         } else {
6892                 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6893                 for (i = 0; i < 3; i++)
6894                         WREG32(RLC_GPM_SCRATCH_DATA, 0);
6895         }
6896         if (rdev->rlc.reg_list) {
6897                 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET);
6898                 for (i = 0; i < rdev->rlc.reg_list_size; i++)
6899                         WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);
6900         }
6901
6902         orig = data = RREG32(RLC_PG_CNTL);
6903         data |= GFX_PG_SRC;
6904         if (orig != data)
6905                 WREG32(RLC_PG_CNTL, data);
6906
6907         WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
6908         WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);
6909
6910         data = RREG32(CP_RB_WPTR_POLL_CNTL);
6911         data &= ~IDLE_POLL_COUNT_MASK;
6912         data |= IDLE_POLL_COUNT(0x60);
6913         WREG32(CP_RB_WPTR_POLL_CNTL, data);
6914
6915         data = 0x10101010;
6916         WREG32(RLC_PG_DELAY, data);
6917
6918         data = RREG32(RLC_PG_DELAY_2);
6919         data &= ~0xff;
6920         data |= 0x3;
6921         WREG32(RLC_PG_DELAY_2, data);
6922
6923         data = RREG32(RLC_AUTO_PG_CTRL);
6924         data &= ~GRBM_REG_SGIT_MASK;
6925         data |= GRBM_REG_SGIT(0x700);
6926         WREG32(RLC_AUTO_PG_CTRL, data);
6927
6928 }
6929
6930 static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable)
6931 {
6932         cik_enable_gfx_cgpg(rdev, enable);
6933         cik_enable_gfx_static_mgpg(rdev, enable);
6934         cik_enable_gfx_dynamic_mgpg(rdev, enable);
6935 }
6936
6937 u32 cik_get_csb_size(struct radeon_device *rdev)
6938 {
6939         u32 count = 0;
6940         const struct cs_section_def *sect = NULL;
6941         const struct cs_extent_def *ext = NULL;
6942
6943         if (rdev->rlc.cs_data == NULL)
6944                 return 0;
6945
6946         /* begin clear state */
6947         count += 2;
6948         /* context control state */
6949         count += 3;
6950
6951         for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6952                 for (ext = sect->section; ext->extent != NULL; ++ext) {
6953                         if (sect->id == SECT_CONTEXT)
6954                                 count += 2 + ext->reg_count;
6955                         else
6956                                 return 0;
6957                 }
6958         }
6959         /* pa_sc_raster_config/pa_sc_raster_config1 */
6960         count += 4;
6961         /* end clear state */
6962         count += 2;
6963         /* clear state */
6964         count += 2;
6965
6966         return count;
6967 }
6968
6969 void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
6970 {
6971         u32 count = 0, i;
6972         const struct cs_section_def *sect = NULL;
6973         const struct cs_extent_def *ext = NULL;
6974
6975         if (rdev->rlc.cs_data == NULL)
6976                 return;
6977         if (buffer == NULL)
6978                 return;
6979
6980         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6981         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
6982
6983         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
6984         buffer[count++] = cpu_to_le32(0x80000000);
6985         buffer[count++] = cpu_to_le32(0x80000000);
6986
6987         for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6988                 for (ext = sect->section; ext->extent != NULL; ++ext) {
6989                         if (sect->id == SECT_CONTEXT) {
6990                                 buffer[count++] =
6991                                         cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
6992                                 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
6993                                 for (i = 0; i < ext->reg_count; i++)
6994                                         buffer[count++] = cpu_to_le32(ext->extent[i]);
6995                         } else {
6996                                 return;
6997                         }
6998                 }
6999         }
7000
7001         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
7002         buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
7003         switch (rdev->family) {
7004         case CHIP_BONAIRE:
7005                 buffer[count++] = cpu_to_le32(0x16000012);
7006                 buffer[count++] = cpu_to_le32(0x00000000);
7007                 break;
7008         case CHIP_KAVERI:
7009                 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
7010                 buffer[count++] = cpu_to_le32(0x00000000);
7011                 break;
7012         case CHIP_KABINI:
7013         case CHIP_MULLINS:
7014                 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
7015                 buffer[count++] = cpu_to_le32(0x00000000);
7016                 break;
7017         case CHIP_HAWAII:
7018                 buffer[count++] = cpu_to_le32(0x3a00161a);
7019                 buffer[count++] = cpu_to_le32(0x0000002e);
7020                 break;
7021         default:
7022                 buffer[count++] = cpu_to_le32(0x00000000);
7023                 buffer[count++] = cpu_to_le32(0x00000000);
7024                 break;
7025         }
7026
7027         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
7028         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
7029
7030         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
7031         buffer[count++] = cpu_to_le32(0);
7032 }
7033
7034 static void cik_init_pg(struct radeon_device *rdev)
7035 {
7036         if (rdev->pg_flags) {
7037                 cik_enable_sck_slowdown_on_pu(rdev, true);
7038                 cik_enable_sck_slowdown_on_pd(rdev, true);
7039                 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
7040                         cik_init_gfx_cgpg(rdev);
7041                         cik_enable_cp_pg(rdev, true);
7042                         cik_enable_gds_pg(rdev, true);
7043                 }
7044                 cik_init_ao_cu_mask(rdev);
7045                 cik_update_gfx_pg(rdev, true);
7046         }
7047 }
7048
7049 static void cik_fini_pg(struct radeon_device *rdev)
7050 {
7051         if (rdev->pg_flags) {
7052                 cik_update_gfx_pg(rdev, false);
7053                 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
7054                         cik_enable_cp_pg(rdev, false);
7055                         cik_enable_gds_pg(rdev, false);
7056                 }
7057         }
7058 }
7059
7060 /*
7061  * Interrupts
7062  * Starting with r6xx, interrupts are handled via a ring buffer.
7063  * Ring buffers are areas of GPU accessible memory that the GPU
7064  * writes interrupt vectors into and the host reads vectors out of.
7065  * There is a rptr (read pointer) that determines where the
7066  * host is currently reading, and a wptr (write pointer)
7067  * which determines where the GPU has written.  When the
7068  * pointers are equal, the ring is idle.  When the GPU
7069  * writes vectors to the ring buffer, it increments the
7070  * wptr.  When there is an interrupt, the host then starts
7071  * fetching commands and processing them until the pointers are
7072  * equal again at which point it updates the rptr.
7073  */
7074
7075 /**
7076  * cik_enable_interrupts - Enable the interrupt ring buffer
7077  *
7078  * @rdev: radeon_device pointer
7079  *
7080  * Enable the interrupt ring buffer (CIK).
7081  */
7082 static void cik_enable_interrupts(struct radeon_device *rdev)
7083 {
7084         u32 ih_cntl = RREG32(IH_CNTL);
7085         u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
7086
7087         ih_cntl |= ENABLE_INTR;
7088         ih_rb_cntl |= IH_RB_ENABLE;
7089         WREG32(IH_CNTL, ih_cntl);
7090         WREG32(IH_RB_CNTL, ih_rb_cntl);
7091         rdev->ih.enabled = true;
7092 }
7093
7094 /**
7095  * cik_disable_interrupts - Disable the interrupt ring buffer
7096  *
7097  * @rdev: radeon_device pointer
7098  *
7099  * Disable the interrupt ring buffer (CIK).
7100  */
7101 static void cik_disable_interrupts(struct radeon_device *rdev)
7102 {
7103         u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
7104         u32 ih_cntl = RREG32(IH_CNTL);
7105
7106         ih_rb_cntl &= ~IH_RB_ENABLE;
7107         ih_cntl &= ~ENABLE_INTR;
7108         WREG32(IH_RB_CNTL, ih_rb_cntl);
7109         WREG32(IH_CNTL, ih_cntl);
7110         /* set rptr, wptr to 0 */
7111         WREG32(IH_RB_RPTR, 0);
7112         WREG32(IH_RB_WPTR, 0);
7113         rdev->ih.enabled = false;
7114         rdev->ih.rptr = 0;
7115 }
7116
7117 /**
7118  * cik_disable_interrupt_state - Disable all interrupt sources
7119  *
7120  * @rdev: radeon_device pointer
7121  *
7122  * Clear all interrupt enable bits used by the driver (CIK).
7123  */
7124 static void cik_disable_interrupt_state(struct radeon_device *rdev)
7125 {
7126         u32 tmp;
7127
7128         /* gfx ring */
7129         tmp = RREG32(CP_INT_CNTL_RING0) &
7130                 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
7131         WREG32(CP_INT_CNTL_RING0, tmp);
7132         /* sdma */
7133         tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
7134         WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
7135         tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
7136         WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
7137         /* compute queues */
7138         WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
7139         WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
7140         WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
7141         WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
7142         WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
7143         WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
7144         WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
7145         WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
7146         /* grbm */
7147         WREG32(GRBM_INT_CNTL, 0);
7148         /* vline/vblank, etc. */
7149         WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
7150         WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
7151         if (rdev->num_crtc >= 4) {
7152                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
7153                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
7154         }
7155         if (rdev->num_crtc >= 6) {
7156                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
7157                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
7158         }
7159         /* pflip */
7160         if (rdev->num_crtc >= 2) {
7161                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
7162                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
7163         }
7164         if (rdev->num_crtc >= 4) {
7165                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
7166                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
7167         }
7168         if (rdev->num_crtc >= 6) {
7169                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
7170                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
7171         }
7172
7173         /* dac hotplug */
7174         WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
7175
7176         /* digital hotplug */
7177         tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7178         WREG32(DC_HPD1_INT_CONTROL, tmp);
7179         tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7180         WREG32(DC_HPD2_INT_CONTROL, tmp);
7181         tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7182         WREG32(DC_HPD3_INT_CONTROL, tmp);
7183         tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7184         WREG32(DC_HPD4_INT_CONTROL, tmp);
7185         tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7186         WREG32(DC_HPD5_INT_CONTROL, tmp);
7187         tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7188         WREG32(DC_HPD6_INT_CONTROL, tmp);
7189
7190 }
7191
7192 /**
7193  * cik_irq_init - init and enable the interrupt ring
7194  *
7195  * @rdev: radeon_device pointer
7196  *
7197  * Allocate a ring buffer for the interrupt controller,
7198  * enable the RLC, disable interrupts, enable the IH
7199  * ring buffer and enable it (CIK).
7200  * Called at device load and reume.
7201  * Returns 0 for success, errors for failure.
7202  */
7203 static int cik_irq_init(struct radeon_device *rdev)
7204 {
7205         int ret = 0;
7206         int rb_bufsz;
7207         u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
7208
7209         /* allocate ring */
7210         ret = r600_ih_ring_alloc(rdev);
7211         if (ret)
7212                 return ret;
7213
7214         /* disable irqs */
7215         cik_disable_interrupts(rdev);
7216
7217         /* init rlc */
7218         ret = cik_rlc_resume(rdev);
7219         if (ret) {
7220                 r600_ih_ring_fini(rdev);
7221                 return ret;
7222         }
7223
7224         /* setup interrupt control */
7225         /* XXX this should actually be a bus address, not an MC address. same on older asics */
7226         WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
7227         interrupt_cntl = RREG32(INTERRUPT_CNTL);
7228         /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
7229          * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
7230          */
7231         interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
7232         /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
7233         interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
7234         WREG32(INTERRUPT_CNTL, interrupt_cntl);
7235
7236         WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
7237         rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
7238
7239         ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
7240                       IH_WPTR_OVERFLOW_CLEAR |
7241                       (rb_bufsz << 1));
7242
7243         if (rdev->wb.enabled)
7244                 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
7245
7246         /* set the writeback address whether it's enabled or not */
7247         WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
7248         WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
7249
7250         WREG32(IH_RB_CNTL, ih_rb_cntl);
7251
7252         /* set rptr, wptr to 0 */
7253         WREG32(IH_RB_RPTR, 0);
7254         WREG32(IH_RB_WPTR, 0);
7255
7256         /* Default settings for IH_CNTL (disabled at first) */
7257         ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
7258         /* RPTR_REARM only works if msi's are enabled */
7259         if (rdev->msi_enabled)
7260                 ih_cntl |= RPTR_REARM;
7261         WREG32(IH_CNTL, ih_cntl);
7262
7263         /* force the active interrupt state to all disabled */
7264         cik_disable_interrupt_state(rdev);
7265
7266         pci_set_master(rdev->pdev);
7267
7268         /* enable irqs */
7269         cik_enable_interrupts(rdev);
7270
7271         return ret;
7272 }
7273
7274 /**
7275  * cik_irq_set - enable/disable interrupt sources
7276  *
7277  * @rdev: radeon_device pointer
7278  *
7279  * Enable interrupt sources on the GPU (vblanks, hpd,
7280  * etc.) (CIK).
7281  * Returns 0 for success, errors for failure.
7282  */
7283 int cik_irq_set(struct radeon_device *rdev)
7284 {
7285         u32 cp_int_cntl;
7286         u32 cp_m1p0, cp_m1p1, cp_m1p2, cp_m1p3;
7287         u32 cp_m2p0, cp_m2p1, cp_m2p2, cp_m2p3;
7288         u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
7289         u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
7290         u32 grbm_int_cntl = 0;
7291         u32 dma_cntl, dma_cntl1;
7292         u32 thermal_int;
7293
7294         if (!rdev->irq.installed) {
7295                 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
7296                 return -EINVAL;
7297         }
7298         /* don't enable anything if the ih is disabled */
7299         if (!rdev->ih.enabled) {
7300                 cik_disable_interrupts(rdev);
7301                 /* force the active interrupt state to all disabled */
7302                 cik_disable_interrupt_state(rdev);
7303                 return 0;
7304         }
7305
7306         cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
7307                 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
7308         cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
7309
7310         hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
7311         hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
7312         hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
7313         hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
7314         hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
7315         hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
7316
7317         dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
7318         dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
7319
7320         cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7321         cp_m1p1 = RREG32(CP_ME1_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7322         cp_m1p2 = RREG32(CP_ME1_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7323         cp_m1p3 = RREG32(CP_ME1_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7324         cp_m2p0 = RREG32(CP_ME2_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7325         cp_m2p1 = RREG32(CP_ME2_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7326         cp_m2p2 = RREG32(CP_ME2_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7327         cp_m2p3 = RREG32(CP_ME2_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7328
7329         if (rdev->flags & RADEON_IS_IGP)
7330                 thermal_int = RREG32_SMC(CG_THERMAL_INT_CTRL) &
7331                         ~(THERM_INTH_MASK | THERM_INTL_MASK);
7332         else
7333                 thermal_int = RREG32_SMC(CG_THERMAL_INT) &
7334                         ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
7335
7336         /* enable CP interrupts on all rings */
7337         if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
7338                 DRM_DEBUG("cik_irq_set: sw int gfx\n");
7339                 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
7340         }
7341         if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
7342                 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7343                 DRM_DEBUG("si_irq_set: sw int cp1\n");
7344                 if (ring->me == 1) {
7345                         switch (ring->pipe) {
7346                         case 0:
7347                                 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7348                                 break;
7349                         case 1:
7350                                 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
7351                                 break;
7352                         case 2:
7353                                 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7354                                 break;
7355                         case 3:
7356                                 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7357                                 break;
7358                         default:
7359                                 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
7360                                 break;
7361                         }
7362                 } else if (ring->me == 2) {
7363                         switch (ring->pipe) {
7364                         case 0:
7365                                 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
7366                                 break;
7367                         case 1:
7368                                 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
7369                                 break;
7370                         case 2:
7371                                 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7372                                 break;
7373                         case 3:
7374                                 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7375                                 break;
7376                         default:
7377                                 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
7378                                 break;
7379                         }
7380                 } else {
7381                         DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
7382                 }
7383         }
7384         if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
7385                 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7386                 DRM_DEBUG("si_irq_set: sw int cp2\n");
7387                 if (ring->me == 1) {
7388                         switch (ring->pipe) {
7389                         case 0:
7390                                 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7391                                 break;
7392                         case 1:
7393                                 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
7394                                 break;
7395                         case 2:
7396                                 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7397                                 break;
7398                         case 3:
7399                                 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7400                                 break;
7401                         default:
7402                                 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
7403                                 break;
7404                         }
7405                 } else if (ring->me == 2) {
7406                         switch (ring->pipe) {
7407                         case 0:
7408                                 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
7409                                 break;
7410                         case 1:
7411                                 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
7412                                 break;
7413                         case 2:
7414                                 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7415                                 break;
7416                         case 3:
7417                                 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7418                                 break;
7419                         default:
7420                                 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
7421                                 break;
7422                         }
7423                 } else {
7424                         DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
7425                 }
7426         }
7427
7428         if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
7429                 DRM_DEBUG("cik_irq_set: sw int dma\n");
7430                 dma_cntl |= TRAP_ENABLE;
7431         }
7432
7433         if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
7434                 DRM_DEBUG("cik_irq_set: sw int dma1\n");
7435                 dma_cntl1 |= TRAP_ENABLE;
7436         }
7437
7438         if (rdev->irq.crtc_vblank_int[0] ||
7439             atomic_read(&rdev->irq.pflip[0])) {
7440                 DRM_DEBUG("cik_irq_set: vblank 0\n");
7441                 crtc1 |= VBLANK_INTERRUPT_MASK;
7442         }
7443         if (rdev->irq.crtc_vblank_int[1] ||
7444             atomic_read(&rdev->irq.pflip[1])) {
7445                 DRM_DEBUG("cik_irq_set: vblank 1\n");
7446                 crtc2 |= VBLANK_INTERRUPT_MASK;
7447         }
7448         if (rdev->irq.crtc_vblank_int[2] ||
7449             atomic_read(&rdev->irq.pflip[2])) {
7450                 DRM_DEBUG("cik_irq_set: vblank 2\n");
7451                 crtc3 |= VBLANK_INTERRUPT_MASK;
7452         }
7453         if (rdev->irq.crtc_vblank_int[3] ||
7454             atomic_read(&rdev->irq.pflip[3])) {
7455                 DRM_DEBUG("cik_irq_set: vblank 3\n");
7456                 crtc4 |= VBLANK_INTERRUPT_MASK;
7457         }
7458         if (rdev->irq.crtc_vblank_int[4] ||
7459             atomic_read(&rdev->irq.pflip[4])) {
7460                 DRM_DEBUG("cik_irq_set: vblank 4\n");
7461                 crtc5 |= VBLANK_INTERRUPT_MASK;
7462         }
7463         if (rdev->irq.crtc_vblank_int[5] ||
7464             atomic_read(&rdev->irq.pflip[5])) {
7465                 DRM_DEBUG("cik_irq_set: vblank 5\n");
7466                 crtc6 |= VBLANK_INTERRUPT_MASK;
7467         }
7468         if (rdev->irq.hpd[0]) {
7469                 DRM_DEBUG("cik_irq_set: hpd 1\n");
7470                 hpd1 |= DC_HPDx_INT_EN;
7471         }
7472         if (rdev->irq.hpd[1]) {
7473                 DRM_DEBUG("cik_irq_set: hpd 2\n");
7474                 hpd2 |= DC_HPDx_INT_EN;
7475         }
7476         if (rdev->irq.hpd[2]) {
7477                 DRM_DEBUG("cik_irq_set: hpd 3\n");
7478                 hpd3 |= DC_HPDx_INT_EN;
7479         }
7480         if (rdev->irq.hpd[3]) {
7481                 DRM_DEBUG("cik_irq_set: hpd 4\n");
7482                 hpd4 |= DC_HPDx_INT_EN;
7483         }
7484         if (rdev->irq.hpd[4]) {
7485                 DRM_DEBUG("cik_irq_set: hpd 5\n");
7486                 hpd5 |= DC_HPDx_INT_EN;
7487         }
7488         if (rdev->irq.hpd[5]) {
7489                 DRM_DEBUG("cik_irq_set: hpd 6\n");
7490                 hpd6 |= DC_HPDx_INT_EN;
7491         }
7492
7493         if (rdev->irq.dpm_thermal) {
7494                 DRM_DEBUG("dpm thermal\n");
7495                 if (rdev->flags & RADEON_IS_IGP)
7496                         thermal_int |= THERM_INTH_MASK | THERM_INTL_MASK;
7497                 else
7498                         thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
7499         }
7500
7501         WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
7502
7503         WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
7504         WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
7505
7506         WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
7507         WREG32(CP_ME1_PIPE1_INT_CNTL, cp_m1p1);
7508         WREG32(CP_ME1_PIPE2_INT_CNTL, cp_m1p2);
7509         WREG32(CP_ME1_PIPE3_INT_CNTL, cp_m1p3);
7510         WREG32(CP_ME2_PIPE0_INT_CNTL, cp_m2p0);
7511         WREG32(CP_ME2_PIPE1_INT_CNTL, cp_m2p1);
7512         WREG32(CP_ME2_PIPE2_INT_CNTL, cp_m2p2);
7513         WREG32(CP_ME2_PIPE3_INT_CNTL, cp_m2p3);
7514
7515         WREG32(GRBM_INT_CNTL, grbm_int_cntl);
7516
7517         WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
7518         WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
7519         if (rdev->num_crtc >= 4) {
7520                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
7521                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
7522         }
7523         if (rdev->num_crtc >= 6) {
7524                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
7525                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
7526         }
7527
7528         if (rdev->num_crtc >= 2) {
7529                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
7530                        GRPH_PFLIP_INT_MASK);
7531                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
7532                        GRPH_PFLIP_INT_MASK);
7533         }
7534         if (rdev->num_crtc >= 4) {
7535                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
7536                        GRPH_PFLIP_INT_MASK);
7537                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
7538                        GRPH_PFLIP_INT_MASK);
7539         }
7540         if (rdev->num_crtc >= 6) {
7541                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
7542                        GRPH_PFLIP_INT_MASK);
7543                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
7544                        GRPH_PFLIP_INT_MASK);
7545         }
7546
7547         WREG32(DC_HPD1_INT_CONTROL, hpd1);
7548         WREG32(DC_HPD2_INT_CONTROL, hpd2);
7549         WREG32(DC_HPD3_INT_CONTROL, hpd3);
7550         WREG32(DC_HPD4_INT_CONTROL, hpd4);
7551         WREG32(DC_HPD5_INT_CONTROL, hpd5);
7552         WREG32(DC_HPD6_INT_CONTROL, hpd6);
7553
7554         if (rdev->flags & RADEON_IS_IGP)
7555                 WREG32_SMC(CG_THERMAL_INT_CTRL, thermal_int);
7556         else
7557                 WREG32_SMC(CG_THERMAL_INT, thermal_int);
7558
7559         return 0;
7560 }
7561
7562 /**
7563  * cik_irq_ack - ack interrupt sources
7564  *
7565  * @rdev: radeon_device pointer
7566  *
7567  * Ack interrupt sources on the GPU (vblanks, hpd,
7568  * etc.) (CIK).  Certain interrupts sources are sw
7569  * generated and do not require an explicit ack.
7570  */
7571 static inline void cik_irq_ack(struct radeon_device *rdev)
7572 {
7573         u32 tmp;
7574
7575         rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
7576         rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
7577         rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
7578         rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
7579         rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
7580         rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
7581         rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
7582
7583         rdev->irq.stat_regs.cik.d1grph_int = RREG32(GRPH_INT_STATUS +
7584                 EVERGREEN_CRTC0_REGISTER_OFFSET);
7585         rdev->irq.stat_regs.cik.d2grph_int = RREG32(GRPH_INT_STATUS +
7586                 EVERGREEN_CRTC1_REGISTER_OFFSET);
7587         if (rdev->num_crtc >= 4) {
7588                 rdev->irq.stat_regs.cik.d3grph_int = RREG32(GRPH_INT_STATUS +
7589                         EVERGREEN_CRTC2_REGISTER_OFFSET);
7590                 rdev->irq.stat_regs.cik.d4grph_int = RREG32(GRPH_INT_STATUS +
7591                         EVERGREEN_CRTC3_REGISTER_OFFSET);
7592         }
7593         if (rdev->num_crtc >= 6) {
7594                 rdev->irq.stat_regs.cik.d5grph_int = RREG32(GRPH_INT_STATUS +
7595                         EVERGREEN_CRTC4_REGISTER_OFFSET);
7596                 rdev->irq.stat_regs.cik.d6grph_int = RREG32(GRPH_INT_STATUS +
7597                         EVERGREEN_CRTC5_REGISTER_OFFSET);
7598         }
7599
7600         if (rdev->irq.stat_regs.cik.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
7601                 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET,
7602                        GRPH_PFLIP_INT_CLEAR);
7603         if (rdev->irq.stat_regs.cik.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
7604                 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET,
7605                        GRPH_PFLIP_INT_CLEAR);
7606         if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
7607                 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
7608         if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
7609                 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
7610         if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
7611                 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
7612         if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
7613                 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
7614
7615         if (rdev->num_crtc >= 4) {
7616                 if (rdev->irq.stat_regs.cik.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
7617                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET,
7618                                GRPH_PFLIP_INT_CLEAR);
7619                 if (rdev->irq.stat_regs.cik.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
7620                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET,
7621                                GRPH_PFLIP_INT_CLEAR);
7622                 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
7623                         WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
7624                 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
7625                         WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
7626                 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
7627                         WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
7628                 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
7629                         WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
7630         }
7631
7632         if (rdev->num_crtc >= 6) {
7633                 if (rdev->irq.stat_regs.cik.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
7634                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET,
7635                                GRPH_PFLIP_INT_CLEAR);
7636                 if (rdev->irq.stat_regs.cik.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
7637                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET,
7638                                GRPH_PFLIP_INT_CLEAR);
7639                 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
7640                         WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
7641                 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
7642                         WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
7643                 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
7644                         WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
7645                 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
7646                         WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
7647         }
7648
7649         if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7650                 tmp = RREG32(DC_HPD1_INT_CONTROL);
7651                 tmp |= DC_HPDx_INT_ACK;
7652                 WREG32(DC_HPD1_INT_CONTROL, tmp);
7653         }
7654         if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7655                 tmp = RREG32(DC_HPD2_INT_CONTROL);
7656                 tmp |= DC_HPDx_INT_ACK;
7657                 WREG32(DC_HPD2_INT_CONTROL, tmp);
7658         }
7659         if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7660                 tmp = RREG32(DC_HPD3_INT_CONTROL);
7661                 tmp |= DC_HPDx_INT_ACK;
7662                 WREG32(DC_HPD3_INT_CONTROL, tmp);
7663         }
7664         if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7665                 tmp = RREG32(DC_HPD4_INT_CONTROL);
7666                 tmp |= DC_HPDx_INT_ACK;
7667                 WREG32(DC_HPD4_INT_CONTROL, tmp);
7668         }
7669         if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7670                 tmp = RREG32(DC_HPD5_INT_CONTROL);
7671                 tmp |= DC_HPDx_INT_ACK;
7672                 WREG32(DC_HPD5_INT_CONTROL, tmp);
7673         }
7674         if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7675                 tmp = RREG32(DC_HPD5_INT_CONTROL);
7676                 tmp |= DC_HPDx_INT_ACK;
7677                 WREG32(DC_HPD6_INT_CONTROL, tmp);
7678         }
7679 }
7680
7681 /**
7682  * cik_irq_disable - disable interrupts
7683  *
7684  * @rdev: radeon_device pointer
7685  *
7686  * Disable interrupts on the hw (CIK).
7687  */
7688 static void cik_irq_disable(struct radeon_device *rdev)
7689 {
7690         cik_disable_interrupts(rdev);
7691         /* Wait and acknowledge irq */
7692         mdelay(1);
7693         cik_irq_ack(rdev);
7694         cik_disable_interrupt_state(rdev);
7695 }
7696
7697 /**
7698  * cik_irq_disable - disable interrupts for suspend
7699  *
7700  * @rdev: radeon_device pointer
7701  *
7702  * Disable interrupts and stop the RLC (CIK).
7703  * Used for suspend.
7704  */
7705 static void cik_irq_suspend(struct radeon_device *rdev)
7706 {
7707         cik_irq_disable(rdev);
7708         cik_rlc_stop(rdev);
7709 }
7710
7711 /**
7712  * cik_irq_fini - tear down interrupt support
7713  *
7714  * @rdev: radeon_device pointer
7715  *
7716  * Disable interrupts on the hw and free the IH ring
7717  * buffer (CIK).
7718  * Used for driver unload.
7719  */
7720 static void cik_irq_fini(struct radeon_device *rdev)
7721 {
7722         cik_irq_suspend(rdev);
7723         r600_ih_ring_fini(rdev);
7724 }
7725
7726 /**
7727  * cik_get_ih_wptr - get the IH ring buffer wptr
7728  *
7729  * @rdev: radeon_device pointer
7730  *
7731  * Get the IH ring buffer wptr from either the register
7732  * or the writeback memory buffer (CIK).  Also check for
7733  * ring buffer overflow and deal with it.
7734  * Used by cik_irq_process().
7735  * Returns the value of the wptr.
7736  */
7737 static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
7738 {
7739         u32 wptr, tmp;
7740
7741         if (rdev->wb.enabled)
7742                 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
7743         else
7744                 wptr = RREG32(IH_RB_WPTR);
7745
7746         if (wptr & RB_OVERFLOW) {
7747                 /* When a ring buffer overflow happen start parsing interrupt
7748                  * from the last not overwritten vector (wptr + 16). Hopefully
7749                  * this should allow us to catchup.
7750                  */
7751                 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
7752                         wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
7753                 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
7754                 tmp = RREG32(IH_RB_CNTL);
7755                 tmp |= IH_WPTR_OVERFLOW_CLEAR;
7756                 WREG32(IH_RB_CNTL, tmp);
7757                 wptr &= ~RB_OVERFLOW;
7758         }
7759         return (wptr & rdev->ih.ptr_mask);
7760 }
7761
7762 /*        CIK IV Ring
7763  * Each IV ring entry is 128 bits:
7764  * [7:0]    - interrupt source id
7765  * [31:8]   - reserved
7766  * [59:32]  - interrupt source data
7767  * [63:60]  - reserved
7768  * [71:64]  - RINGID
7769  *            CP:
7770  *            ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
7771  *            QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
7772  *                     - for gfx, hw shader state (0=PS...5=LS, 6=CS)
7773  *            ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
7774  *            PIPE_ID - ME0 0=3D
7775  *                    - ME1&2 compute dispatcher (4 pipes each)
7776  *            SDMA:
7777  *            INSTANCE_ID [1:0], QUEUE_ID[1:0]
7778  *            INSTANCE_ID - 0 = sdma0, 1 = sdma1
7779  *            QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
7780  * [79:72]  - VMID
7781  * [95:80]  - PASID
7782  * [127:96] - reserved
7783  */
7784 /**
7785  * cik_irq_process - interrupt handler
7786  *
7787  * @rdev: radeon_device pointer
7788  *
7789  * Interrupt hander (CIK).  Walk the IH ring,
7790  * ack interrupts and schedule work to handle
7791  * interrupt events.
7792  * Returns irq process return code.
7793  */
7794 int cik_irq_process(struct radeon_device *rdev)
7795 {
7796         struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7797         struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7798         u32 wptr;
7799         u32 rptr;
7800         u32 src_id, src_data, ring_id;
7801         u8 me_id, pipe_id, queue_id;
7802         u32 ring_index;
7803         bool queue_hotplug = false;
7804         bool queue_reset = false;
7805         u32 addr, status, mc_client;
7806         bool queue_thermal = false;
7807
7808         if (!rdev->ih.enabled || rdev->shutdown)
7809                 return IRQ_NONE;
7810
7811         wptr = cik_get_ih_wptr(rdev);
7812
7813 restart_ih:
7814         /* is somebody else already processing irqs? */
7815         if (atomic_xchg(&rdev->ih.lock, 1))
7816                 return IRQ_NONE;
7817
7818         rptr = rdev->ih.rptr;
7819         DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
7820
7821         /* Order reading of wptr vs. reading of IH ring data */
7822         rmb();
7823
7824         /* display interrupts */
7825         cik_irq_ack(rdev);
7826
7827         while (rptr != wptr) {
7828                 /* wptr/rptr are in bytes! */
7829                 ring_index = rptr / 4;
7830                 src_id =  le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
7831                 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
7832                 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
7833
7834                 switch (src_id) {
7835                 case 1: /* D1 vblank/vline */
7836                         switch (src_data) {
7837                         case 0: /* D1 vblank */
7838                                 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) {
7839                                         if (rdev->irq.crtc_vblank_int[0]) {
7840                                                 drm_handle_vblank(rdev->ddev, 0);
7841                                                 rdev->pm.vblank_sync = true;
7842                                                 wake_up(&rdev->irq.vblank_queue);
7843                                         }
7844                                         if (atomic_read(&rdev->irq.pflip[0]))
7845                                                 radeon_crtc_handle_vblank(rdev, 0);
7846                                         rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
7847                                         DRM_DEBUG("IH: D1 vblank\n");
7848                                 }
7849                                 break;
7850                         case 1: /* D1 vline */
7851                                 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) {
7852                                         rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
7853                                         DRM_DEBUG("IH: D1 vline\n");
7854                                 }
7855                                 break;
7856                         default:
7857                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7858                                 break;
7859                         }
7860                         break;
7861                 case 2: /* D2 vblank/vline */
7862                         switch (src_data) {
7863                         case 0: /* D2 vblank */
7864                                 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
7865                                         if (rdev->irq.crtc_vblank_int[1]) {
7866                                                 drm_handle_vblank(rdev->ddev, 1);
7867                                                 rdev->pm.vblank_sync = true;
7868                                                 wake_up(&rdev->irq.vblank_queue);
7869                                         }
7870                                         if (atomic_read(&rdev->irq.pflip[1]))
7871                                                 radeon_crtc_handle_vblank(rdev, 1);
7872                                         rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
7873                                         DRM_DEBUG("IH: D2 vblank\n");
7874                                 }
7875                                 break;
7876                         case 1: /* D2 vline */
7877                                 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
7878                                         rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
7879                                         DRM_DEBUG("IH: D2 vline\n");
7880                                 }
7881                                 break;
7882                         default:
7883                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7884                                 break;
7885                         }
7886                         break;
7887                 case 3: /* D3 vblank/vline */
7888                         switch (src_data) {
7889                         case 0: /* D3 vblank */
7890                                 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
7891                                         if (rdev->irq.crtc_vblank_int[2]) {
7892                                                 drm_handle_vblank(rdev->ddev, 2);
7893                                                 rdev->pm.vblank_sync = true;
7894                                                 wake_up(&rdev->irq.vblank_queue);
7895                                         }
7896                                         if (atomic_read(&rdev->irq.pflip[2]))
7897                                                 radeon_crtc_handle_vblank(rdev, 2);
7898                                         rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
7899                                         DRM_DEBUG("IH: D3 vblank\n");
7900                                 }
7901                                 break;
7902                         case 1: /* D3 vline */
7903                                 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
7904                                         rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
7905                                         DRM_DEBUG("IH: D3 vline\n");
7906                                 }
7907                                 break;
7908                         default:
7909                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7910                                 break;
7911                         }
7912                         break;
7913                 case 4: /* D4 vblank/vline */
7914                         switch (src_data) {
7915                         case 0: /* D4 vblank */
7916                                 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
7917                                         if (rdev->irq.crtc_vblank_int[3]) {
7918                                                 drm_handle_vblank(rdev->ddev, 3);
7919                                                 rdev->pm.vblank_sync = true;
7920                                                 wake_up(&rdev->irq.vblank_queue);
7921                                         }
7922                                         if (atomic_read(&rdev->irq.pflip[3]))
7923                                                 radeon_crtc_handle_vblank(rdev, 3);
7924                                         rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
7925                                         DRM_DEBUG("IH: D4 vblank\n");
7926                                 }
7927                                 break;
7928                         case 1: /* D4 vline */
7929                                 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
7930                                         rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
7931                                         DRM_DEBUG("IH: D4 vline\n");
7932                                 }
7933                                 break;
7934                         default:
7935                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7936                                 break;
7937                         }
7938                         break;
7939                 case 5: /* D5 vblank/vline */
7940                         switch (src_data) {
7941                         case 0: /* D5 vblank */
7942                                 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
7943                                         if (rdev->irq.crtc_vblank_int[4]) {
7944                                                 drm_handle_vblank(rdev->ddev, 4);
7945                                                 rdev->pm.vblank_sync = true;
7946                                                 wake_up(&rdev->irq.vblank_queue);
7947                                         }
7948                                         if (atomic_read(&rdev->irq.pflip[4]))
7949                                                 radeon_crtc_handle_vblank(rdev, 4);
7950                                         rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
7951                                         DRM_DEBUG("IH: D5 vblank\n");
7952                                 }
7953                                 break;
7954                         case 1: /* D5 vline */
7955                                 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
7956                                         rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
7957                                         DRM_DEBUG("IH: D5 vline\n");
7958                                 }
7959                                 break;
7960                         default:
7961                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7962                                 break;
7963                         }
7964                         break;
7965                 case 6: /* D6 vblank/vline */
7966                         switch (src_data) {
7967                         case 0: /* D6 vblank */
7968                                 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
7969                                         if (rdev->irq.crtc_vblank_int[5]) {
7970                                                 drm_handle_vblank(rdev->ddev, 5);
7971                                                 rdev->pm.vblank_sync = true;
7972                                                 wake_up(&rdev->irq.vblank_queue);
7973                                         }
7974                                         if (atomic_read(&rdev->irq.pflip[5]))
7975                                                 radeon_crtc_handle_vblank(rdev, 5);
7976                                         rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
7977                                         DRM_DEBUG("IH: D6 vblank\n");
7978                                 }
7979                                 break;
7980                         case 1: /* D6 vline */
7981                                 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
7982                                         rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
7983                                         DRM_DEBUG("IH: D6 vline\n");
7984                                 }
7985                                 break;
7986                         default:
7987                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7988                                 break;
7989                         }
7990                         break;
7991                 case 8: /* D1 page flip */
7992                 case 10: /* D2 page flip */
7993                 case 12: /* D3 page flip */
7994                 case 14: /* D4 page flip */
7995                 case 16: /* D5 page flip */
7996                 case 18: /* D6 page flip */
7997                         DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
7998                         if (radeon_use_pflipirq > 0)
7999                                 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
8000                         break;
8001                 case 42: /* HPD hotplug */
8002                         switch (src_data) {
8003                         case 0:
8004                                 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
8005                                         rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
8006                                         queue_hotplug = true;
8007                                         DRM_DEBUG("IH: HPD1\n");
8008                                 }
8009                                 break;
8010                         case 1:
8011                                 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
8012                                         rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
8013                                         queue_hotplug = true;
8014                                         DRM_DEBUG("IH: HPD2\n");
8015                                 }
8016                                 break;
8017                         case 2:
8018                                 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
8019                                         rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
8020                                         queue_hotplug = true;
8021                                         DRM_DEBUG("IH: HPD3\n");
8022                                 }
8023                                 break;
8024                         case 3:
8025                                 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
8026                                         rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
8027                                         queue_hotplug = true;
8028                                         DRM_DEBUG("IH: HPD4\n");
8029                                 }
8030                                 break;
8031                         case 4:
8032                                 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
8033                                         rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
8034                                         queue_hotplug = true;
8035                                         DRM_DEBUG("IH: HPD5\n");
8036                                 }
8037                                 break;
8038                         case 5:
8039                                 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
8040                                         rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
8041                                         queue_hotplug = true;
8042                                         DRM_DEBUG("IH: HPD6\n");
8043                                 }
8044                                 break;
8045                         default:
8046                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
8047                                 break;
8048                         }
8049                         break;
8050                 case 124: /* UVD */
8051                         DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
8052                         radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
8053                         break;
8054                 case 146:
8055                 case 147:
8056                         addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
8057                         status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
8058                         mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
8059                         /* reset addr and status */
8060                         WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
8061                         if (addr == 0x0 && status == 0x0)
8062                                 break;
8063                         dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
8064                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
8065                                 addr);
8066                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
8067                                 status);
8068                         cik_vm_decode_fault(rdev, status, addr, mc_client);
8069                         break;
8070                 case 167: /* VCE */
8071                         DRM_DEBUG("IH: VCE int: 0x%08x\n", src_data);
8072                         switch (src_data) {
8073                         case 0:
8074                                 radeon_fence_process(rdev, TN_RING_TYPE_VCE1_INDEX);
8075                                 break;
8076                         case 1:
8077                                 radeon_fence_process(rdev, TN_RING_TYPE_VCE2_INDEX);
8078                                 break;
8079                         default:
8080                                 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
8081                                 break;
8082                         }
8083                         break;
8084                 case 176: /* GFX RB CP_INT */
8085                 case 177: /* GFX IB CP_INT */
8086                         radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
8087                         break;
8088                 case 181: /* CP EOP event */
8089                         DRM_DEBUG("IH: CP EOP\n");
8090                         /* XXX check the bitfield order! */
8091                         me_id = (ring_id & 0x60) >> 5;
8092                         pipe_id = (ring_id & 0x18) >> 3;
8093                         queue_id = (ring_id & 0x7) >> 0;
8094                         switch (me_id) {
8095                         case 0:
8096                                 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
8097                                 break;
8098                         case 1:
8099                         case 2:
8100                                 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
8101                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
8102                                 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
8103                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
8104                                 break;
8105                         }
8106                         break;
8107                 case 184: /* CP Privileged reg access */
8108                         DRM_ERROR("Illegal register access in command stream\n");
8109                         /* XXX check the bitfield order! */
8110                         me_id = (ring_id & 0x60) >> 5;
8111                         pipe_id = (ring_id & 0x18) >> 3;
8112                         queue_id = (ring_id & 0x7) >> 0;
8113                         switch (me_id) {
8114                         case 0:
8115                                 /* This results in a full GPU reset, but all we need to do is soft
8116                                  * reset the CP for gfx
8117                                  */
8118                                 queue_reset = true;
8119                                 break;
8120                         case 1:
8121                                 /* XXX compute */
8122                                 queue_reset = true;
8123                                 break;
8124                         case 2:
8125                                 /* XXX compute */
8126                                 queue_reset = true;
8127                                 break;
8128                         }
8129                         break;
8130                 case 185: /* CP Privileged inst */
8131                         DRM_ERROR("Illegal instruction in command stream\n");
8132                         /* XXX check the bitfield order! */
8133                         me_id = (ring_id & 0x60) >> 5;
8134                         pipe_id = (ring_id & 0x18) >> 3;
8135                         queue_id = (ring_id & 0x7) >> 0;
8136                         switch (me_id) {
8137                         case 0:
8138                                 /* This results in a full GPU reset, but all we need to do is soft
8139                                  * reset the CP for gfx
8140                                  */
8141                                 queue_reset = true;
8142                                 break;
8143                         case 1:
8144                                 /* XXX compute */
8145                                 queue_reset = true;
8146                                 break;
8147                         case 2:
8148                                 /* XXX compute */
8149                                 queue_reset = true;
8150                                 break;
8151                         }
8152                         break;
8153                 case 224: /* SDMA trap event */
8154                         /* XXX check the bitfield order! */
8155                         me_id = (ring_id & 0x3) >> 0;
8156                         queue_id = (ring_id & 0xc) >> 2;
8157                         DRM_DEBUG("IH: SDMA trap\n");
8158                         switch (me_id) {
8159                         case 0:
8160                                 switch (queue_id) {
8161                                 case 0:
8162                                         radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
8163                                         break;
8164                                 case 1:
8165                                         /* XXX compute */
8166                                         break;
8167                                 case 2:
8168                                         /* XXX compute */
8169                                         break;
8170                                 }
8171                                 break;
8172                         case 1:
8173                                 switch (queue_id) {
8174                                 case 0:
8175                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
8176                                         break;
8177                                 case 1:
8178                                         /* XXX compute */
8179                                         break;
8180                                 case 2:
8181                                         /* XXX compute */
8182                                         break;
8183                                 }
8184                                 break;
8185                         }
8186                         break;
8187                 case 230: /* thermal low to high */
8188                         DRM_DEBUG("IH: thermal low to high\n");
8189                         rdev->pm.dpm.thermal.high_to_low = false;
8190                         queue_thermal = true;
8191                         break;
8192                 case 231: /* thermal high to low */
8193                         DRM_DEBUG("IH: thermal high to low\n");
8194                         rdev->pm.dpm.thermal.high_to_low = true;
8195                         queue_thermal = true;
8196                         break;
8197                 case 233: /* GUI IDLE */
8198                         DRM_DEBUG("IH: GUI idle\n");
8199                         break;
8200                 case 241: /* SDMA Privileged inst */
8201                 case 247: /* SDMA Privileged inst */
8202                         DRM_ERROR("Illegal instruction in SDMA command stream\n");
8203                         /* XXX check the bitfield order! */
8204                         me_id = (ring_id & 0x3) >> 0;
8205                         queue_id = (ring_id & 0xc) >> 2;
8206                         switch (me_id) {
8207                         case 0:
8208                                 switch (queue_id) {
8209                                 case 0:
8210                                         queue_reset = true;
8211                                         break;
8212                                 case 1:
8213                                         /* XXX compute */
8214                                         queue_reset = true;
8215                                         break;
8216                                 case 2:
8217                                         /* XXX compute */
8218                                         queue_reset = true;
8219                                         break;
8220                                 }
8221                                 break;
8222                         case 1:
8223                                 switch (queue_id) {
8224                                 case 0:
8225                                         queue_reset = true;
8226                                         break;
8227                                 case 1:
8228                                         /* XXX compute */
8229                                         queue_reset = true;
8230                                         break;
8231                                 case 2:
8232                                         /* XXX compute */
8233                                         queue_reset = true;
8234                                         break;
8235                                 }
8236                                 break;
8237                         }
8238                         break;
8239                 default:
8240                         DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
8241                         break;
8242                 }
8243
8244                 /* wptr/rptr are in bytes! */
8245                 rptr += 16;
8246                 rptr &= rdev->ih.ptr_mask;
8247         }
8248         if (queue_hotplug)
8249                 schedule_work(&rdev->hotplug_work);
8250         if (queue_reset) {
8251                 rdev->needs_reset = true;
8252                 wake_up_all(&rdev->fence_queue);
8253         }
8254         if (queue_thermal)
8255                 schedule_work(&rdev->pm.dpm.thermal.work);
8256         rdev->ih.rptr = rptr;
8257         WREG32(IH_RB_RPTR, rdev->ih.rptr);
8258         atomic_set(&rdev->ih.lock, 0);
8259
8260         /* make sure wptr hasn't changed while processing */
8261         wptr = cik_get_ih_wptr(rdev);
8262         if (wptr != rptr)
8263                 goto restart_ih;
8264
8265         return IRQ_HANDLED;
8266 }
8267
8268 /*
8269  * startup/shutdown callbacks
8270  */
8271 /**
8272  * cik_startup - program the asic to a functional state
8273  *
8274  * @rdev: radeon_device pointer
8275  *
8276  * Programs the asic to a functional state (CIK).
8277  * Called by cik_init() and cik_resume().
8278  * Returns 0 for success, error for failure.
8279  */
8280 static int cik_startup(struct radeon_device *rdev)
8281 {
8282         struct radeon_ring *ring;
8283         u32 nop;
8284         int r;
8285
8286         /* enable pcie gen2/3 link */
8287         cik_pcie_gen3_enable(rdev);
8288         /* enable aspm */
8289         cik_program_aspm(rdev);
8290
8291         /* scratch needs to be initialized before MC */
8292         r = r600_vram_scratch_init(rdev);
8293         if (r)
8294                 return r;
8295
8296         cik_mc_program(rdev);
8297
8298         if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
8299                 r = ci_mc_load_microcode(rdev);
8300                 if (r) {
8301                         DRM_ERROR("Failed to load MC firmware!\n");
8302                         return r;
8303                 }
8304         }
8305
8306         r = cik_pcie_gart_enable(rdev);
8307         if (r)
8308                 return r;
8309         cik_gpu_init(rdev);
8310
8311         /* allocate rlc buffers */
8312         if (rdev->flags & RADEON_IS_IGP) {
8313                 if (rdev->family == CHIP_KAVERI) {
8314                         rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;
8315                         rdev->rlc.reg_list_size =
8316                                 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list);
8317                 } else {
8318                         rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;
8319                         rdev->rlc.reg_list_size =
8320                                 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list);
8321                 }
8322         }
8323         rdev->rlc.cs_data = ci_cs_data;
8324         rdev->rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4;
8325         r = sumo_rlc_init(rdev);
8326         if (r) {
8327                 DRM_ERROR("Failed to init rlc BOs!\n");
8328                 return r;
8329         }
8330
8331         /* allocate wb buffer */
8332         r = radeon_wb_init(rdev);
8333         if (r)
8334                 return r;
8335
8336         /* allocate mec buffers */
8337         r = cik_mec_init(rdev);
8338         if (r) {
8339                 DRM_ERROR("Failed to init MEC BOs!\n");
8340                 return r;
8341         }
8342
8343         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
8344         if (r) {
8345                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8346                 return r;
8347         }
8348
8349         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
8350         if (r) {
8351                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8352                 return r;
8353         }
8354
8355         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
8356         if (r) {
8357                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8358                 return r;
8359         }
8360
8361         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
8362         if (r) {
8363                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
8364                 return r;
8365         }
8366
8367         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
8368         if (r) {
8369                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
8370                 return r;
8371         }
8372
8373         r = radeon_uvd_resume(rdev);
8374         if (!r) {
8375                 r = uvd_v4_2_resume(rdev);
8376                 if (!r) {
8377                         r = radeon_fence_driver_start_ring(rdev,
8378                                                            R600_RING_TYPE_UVD_INDEX);
8379                         if (r)
8380                                 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
8381                 }
8382         }
8383         if (r)
8384                 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
8385
8386         r = radeon_vce_resume(rdev);
8387         if (!r) {
8388                 r = vce_v2_0_resume(rdev);
8389                 if (!r)
8390                         r = radeon_fence_driver_start_ring(rdev,
8391                                                            TN_RING_TYPE_VCE1_INDEX);
8392                 if (!r)
8393                         r = radeon_fence_driver_start_ring(rdev,
8394                                                            TN_RING_TYPE_VCE2_INDEX);
8395         }
8396         if (r) {
8397                 dev_err(rdev->dev, "VCE init error (%d).\n", r);
8398                 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
8399                 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
8400         }
8401
8402         /* Enable IRQ */
8403         if (!rdev->irq.installed) {
8404                 r = radeon_irq_kms_init(rdev);
8405                 if (r)
8406                         return r;
8407         }
8408
8409         r = cik_irq_init(rdev);
8410         if (r) {
8411                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
8412                 radeon_irq_kms_fini(rdev);
8413                 return r;
8414         }
8415         cik_irq_set(rdev);
8416
8417         if (rdev->family == CHIP_HAWAII) {
8418                 if (rdev->new_fw)
8419                         nop = PACKET3(PACKET3_NOP, 0x3FFF);
8420                 else
8421                         nop = RADEON_CP_PACKET2;
8422         } else {
8423                 nop = PACKET3(PACKET3_NOP, 0x3FFF);
8424         }
8425
8426         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8427         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
8428                              nop);
8429         if (r)
8430                 return r;
8431
8432         /* set up the compute queues */
8433         /* type-2 packets are deprecated on MEC, use type-3 instead */
8434         ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8435         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
8436                              nop);
8437         if (r)
8438                 return r;
8439         ring->me = 1; /* first MEC */
8440         ring->pipe = 0; /* first pipe */
8441         ring->queue = 0; /* first queue */
8442         ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
8443
8444         /* type-2 packets are deprecated on MEC, use type-3 instead */
8445         ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8446         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
8447                              nop);
8448         if (r)
8449                 return r;
8450         /* dGPU only have 1 MEC */
8451         ring->me = 1; /* first MEC */
8452         ring->pipe = 0; /* first pipe */
8453         ring->queue = 1; /* second queue */
8454         ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
8455
8456         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8457         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
8458                              SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
8459         if (r)
8460                 return r;
8461
8462         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8463         r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
8464                              SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
8465         if (r)
8466                 return r;
8467
8468         r = cik_cp_resume(rdev);
8469         if (r)
8470                 return r;
8471
8472         r = cik_sdma_resume(rdev);
8473         if (r)
8474                 return r;
8475
8476         ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8477         if (ring->ring_size) {
8478                 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8479                                      RADEON_CP_PACKET2);
8480                 if (!r)
8481                         r = uvd_v1_0_init(rdev);
8482                 if (r)
8483                         DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
8484         }
8485
8486         r = -ENOENT;
8487
8488         ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
8489         if (ring->ring_size)
8490                 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8491                                      VCE_CMD_NO_OP);
8492
8493         ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
8494         if (ring->ring_size)
8495                 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8496                                      VCE_CMD_NO_OP);
8497
8498         if (!r)
8499                 r = vce_v1_0_init(rdev);
8500         else if (r != -ENOENT)
8501                 DRM_ERROR("radeon: failed initializing VCE (%d).\n", r);
8502
8503         r = radeon_ib_pool_init(rdev);
8504         if (r) {
8505                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
8506                 return r;
8507         }
8508
8509         r = radeon_vm_manager_init(rdev);
8510         if (r) {
8511                 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
8512                 return r;
8513         }
8514
8515         r = dce6_audio_init(rdev);
8516         if (r)
8517                 return r;
8518
8519         return 0;
8520 }
8521
8522 /**
8523  * cik_resume - resume the asic to a functional state
8524  *
8525  * @rdev: radeon_device pointer
8526  *
8527  * Programs the asic to a functional state (CIK).
8528  * Called at resume.
8529  * Returns 0 for success, error for failure.
8530  */
8531 int cik_resume(struct radeon_device *rdev)
8532 {
8533         int r;
8534
8535         /* post card */
8536         atom_asic_init(rdev->mode_info.atom_context);
8537
8538         /* init golden registers */
8539         cik_init_golden_registers(rdev);
8540
8541         if (rdev->pm.pm_method == PM_METHOD_DPM)
8542                 radeon_pm_resume(rdev);
8543
8544         rdev->accel_working = true;
8545         r = cik_startup(rdev);
8546         if (r) {
8547                 DRM_ERROR("cik startup failed on resume\n");
8548                 rdev->accel_working = false;
8549                 return r;
8550         }
8551
8552         return r;
8553
8554 }
8555
8556 /**
8557  * cik_suspend - suspend the asic
8558  *
8559  * @rdev: radeon_device pointer
8560  *
8561  * Bring the chip into a state suitable for suspend (CIK).
8562  * Called at suspend.
8563  * Returns 0 for success.
8564  */
8565 int cik_suspend(struct radeon_device *rdev)
8566 {
8567         radeon_pm_suspend(rdev);
8568         dce6_audio_fini(rdev);
8569         radeon_vm_manager_fini(rdev);
8570         cik_cp_enable(rdev, false);
8571         cik_sdma_enable(rdev, false);
8572         uvd_v1_0_fini(rdev);
8573         radeon_uvd_suspend(rdev);
8574         radeon_vce_suspend(rdev);
8575         cik_fini_pg(rdev);
8576         cik_fini_cg(rdev);
8577         cik_irq_suspend(rdev);
8578         radeon_wb_disable(rdev);
8579         cik_pcie_gart_disable(rdev);
8580         return 0;
8581 }
8582
8583 /* Plan is to move initialization in that function and use
8584  * helper function so that radeon_device_init pretty much
8585  * do nothing more than calling asic specific function. This
8586  * should also allow to remove a bunch of callback function
8587  * like vram_info.
8588  */
8589 /**
8590  * cik_init - asic specific driver and hw init
8591  *
8592  * @rdev: radeon_device pointer
8593  *
8594  * Setup asic specific driver variables and program the hw
8595  * to a functional state (CIK).
8596  * Called at driver startup.
8597  * Returns 0 for success, errors for failure.
8598  */
8599 int cik_init(struct radeon_device *rdev)
8600 {
8601         struct radeon_ring *ring;
8602         int r;
8603
8604         /* Read BIOS */
8605         if (!radeon_get_bios(rdev)) {
8606                 if (ASIC_IS_AVIVO(rdev))
8607                         return -EINVAL;
8608         }
8609         /* Must be an ATOMBIOS */
8610         if (!rdev->is_atom_bios) {
8611                 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
8612                 return -EINVAL;
8613         }
8614         r = radeon_atombios_init(rdev);
8615         if (r)
8616                 return r;
8617
8618         /* Post card if necessary */
8619         if (!radeon_card_posted(rdev)) {
8620                 if (!rdev->bios) {
8621                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
8622                         return -EINVAL;
8623                 }
8624                 DRM_INFO("GPU not posted. posting now...\n");
8625                 atom_asic_init(rdev->mode_info.atom_context);
8626         }
8627         /* init golden registers */
8628         cik_init_golden_registers(rdev);
8629         /* Initialize scratch registers */
8630         cik_scratch_init(rdev);
8631         /* Initialize surface registers */
8632         radeon_surface_init(rdev);
8633         /* Initialize clocks */
8634         radeon_get_clock_info(rdev->ddev);
8635
8636         /* Fence driver */
8637         r = radeon_fence_driver_init(rdev);
8638         if (r)
8639                 return r;
8640
8641         /* initialize memory controller */
8642         r = cik_mc_init(rdev);
8643         if (r)
8644                 return r;
8645         /* Memory manager */
8646         r = radeon_bo_init(rdev);
8647         if (r)
8648                 return r;
8649
8650         if (rdev->flags & RADEON_IS_IGP) {
8651                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8652                     !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
8653                         r = cik_init_microcode(rdev);
8654                         if (r) {
8655                                 DRM_ERROR("Failed to load firmware!\n");
8656                                 return r;
8657                         }
8658                 }
8659         } else {
8660                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8661                     !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
8662                     !rdev->mc_fw) {
8663                         r = cik_init_microcode(rdev);
8664                         if (r) {
8665                                 DRM_ERROR("Failed to load firmware!\n");
8666                                 return r;
8667                         }
8668                 }
8669         }
8670
8671         /* Initialize power management */
8672         radeon_pm_init(rdev);
8673
8674         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8675         ring->ring_obj = NULL;
8676         r600_ring_init(rdev, ring, 1024 * 1024);
8677
8678         ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8679         ring->ring_obj = NULL;
8680         r600_ring_init(rdev, ring, 1024 * 1024);
8681         r = radeon_doorbell_get(rdev, &ring->doorbell_index);
8682         if (r)
8683                 return r;
8684
8685         ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8686         ring->ring_obj = NULL;
8687         r600_ring_init(rdev, ring, 1024 * 1024);
8688         r = radeon_doorbell_get(rdev, &ring->doorbell_index);
8689         if (r)
8690                 return r;
8691
8692         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8693         ring->ring_obj = NULL;
8694         r600_ring_init(rdev, ring, 256 * 1024);
8695
8696         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8697         ring->ring_obj = NULL;
8698         r600_ring_init(rdev, ring, 256 * 1024);
8699
8700         r = radeon_uvd_init(rdev);
8701         if (!r) {
8702                 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8703                 ring->ring_obj = NULL;
8704                 r600_ring_init(rdev, ring, 4096);
8705         }
8706
8707         r = radeon_vce_init(rdev);
8708         if (!r) {
8709                 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
8710                 ring->ring_obj = NULL;
8711                 r600_ring_init(rdev, ring, 4096);
8712
8713                 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
8714                 ring->ring_obj = NULL;
8715                 r600_ring_init(rdev, ring, 4096);
8716         }
8717
8718         rdev->ih.ring_obj = NULL;
8719         r600_ih_ring_init(rdev, 64 * 1024);
8720
8721         r = r600_pcie_gart_init(rdev);
8722         if (r)
8723                 return r;
8724
8725         rdev->accel_working = true;
8726         r = cik_startup(rdev);
8727         if (r) {
8728                 dev_err(rdev->dev, "disabling GPU acceleration\n");
8729                 cik_cp_fini(rdev);
8730                 cik_sdma_fini(rdev);
8731                 cik_irq_fini(rdev);
8732                 sumo_rlc_fini(rdev);
8733                 cik_mec_fini(rdev);
8734                 radeon_wb_fini(rdev);
8735                 radeon_ib_pool_fini(rdev);
8736                 radeon_vm_manager_fini(rdev);
8737                 radeon_irq_kms_fini(rdev);
8738                 cik_pcie_gart_fini(rdev);
8739                 rdev->accel_working = false;
8740         }
8741
8742         /* Don't start up if the MC ucode is missing.
8743          * The default clocks and voltages before the MC ucode
8744          * is loaded are not suffient for advanced operations.
8745          */
8746         if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
8747                 DRM_ERROR("radeon: MC ucode required for NI+.\n");
8748                 return -EINVAL;
8749         }
8750
8751         return 0;
8752 }
8753
8754 /**
8755  * cik_fini - asic specific driver and hw fini
8756  *
8757  * @rdev: radeon_device pointer
8758  *
8759  * Tear down the asic specific driver variables and program the hw
8760  * to an idle state (CIK).
8761  * Called at driver unload.
8762  */
8763 void cik_fini(struct radeon_device *rdev)
8764 {
8765         radeon_pm_fini(rdev);
8766         cik_cp_fini(rdev);
8767         cik_sdma_fini(rdev);
8768         cik_fini_pg(rdev);
8769         cik_fini_cg(rdev);
8770         cik_irq_fini(rdev);
8771         sumo_rlc_fini(rdev);
8772         cik_mec_fini(rdev);
8773         radeon_wb_fini(rdev);
8774         radeon_vm_manager_fini(rdev);
8775         radeon_ib_pool_fini(rdev);
8776         radeon_irq_kms_fini(rdev);
8777         uvd_v1_0_fini(rdev);
8778         radeon_uvd_fini(rdev);
8779         radeon_vce_fini(rdev);
8780         cik_pcie_gart_fini(rdev);
8781         r600_vram_scratch_fini(rdev);
8782         radeon_gem_fini(rdev);
8783         radeon_fence_driver_fini(rdev);
8784         radeon_bo_fini(rdev);
8785         radeon_atombios_fini(rdev);
8786         kfree(rdev->bios);
8787         rdev->bios = NULL;
8788 }
8789
8790 void dce8_program_fmt(struct drm_encoder *encoder)
8791 {
8792         struct drm_device *dev = encoder->dev;
8793         struct radeon_device *rdev = dev->dev_private;
8794         struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
8795         struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
8796         struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
8797         int bpc = 0;
8798         u32 tmp = 0;
8799         enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
8800
8801         if (connector) {
8802                 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
8803                 bpc = radeon_get_monitor_bpc(connector);
8804                 dither = radeon_connector->dither;
8805         }
8806
8807         /* LVDS/eDP FMT is set up by atom */
8808         if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
8809                 return;
8810
8811         /* not needed for analog */
8812         if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
8813             (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
8814                 return;
8815
8816         if (bpc == 0)
8817                 return;
8818
8819         switch (bpc) {
8820         case 6:
8821                 if (dither == RADEON_FMT_DITHER_ENABLE)
8822                         /* XXX sort out optimal dither settings */
8823                         tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8824                                 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0));
8825                 else
8826                         tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0));
8827                 break;
8828         case 8:
8829                 if (dither == RADEON_FMT_DITHER_ENABLE)
8830                         /* XXX sort out optimal dither settings */
8831                         tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8832                                 FMT_RGB_RANDOM_ENABLE |
8833                                 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1));
8834                 else
8835                         tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1));
8836                 break;
8837         case 10:
8838                 if (dither == RADEON_FMT_DITHER_ENABLE)
8839                         /* XXX sort out optimal dither settings */
8840                         tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8841                                 FMT_RGB_RANDOM_ENABLE |
8842                                 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2));
8843                 else
8844                         tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2));
8845                 break;
8846         default:
8847                 /* not needed */
8848                 break;
8849         }
8850
8851         WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
8852 }
8853
8854 /* display watermark setup */
8855 /**
8856  * dce8_line_buffer_adjust - Set up the line buffer
8857  *
8858  * @rdev: radeon_device pointer
8859  * @radeon_crtc: the selected display controller
8860  * @mode: the current display mode on the selected display
8861  * controller
8862  *
8863  * Setup up the line buffer allocation for
8864  * the selected display controller (CIK).
8865  * Returns the line buffer size in pixels.
8866  */
8867 static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
8868                                    struct radeon_crtc *radeon_crtc,
8869                                    struct drm_display_mode *mode)
8870 {
8871         u32 tmp, buffer_alloc, i;
8872         u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
8873         /*
8874          * Line Buffer Setup
8875          * There are 6 line buffers, one for each display controllers.
8876          * There are 3 partitions per LB. Select the number of partitions
8877          * to enable based on the display width.  For display widths larger
8878          * than 4096, you need use to use 2 display controllers and combine
8879          * them using the stereo blender.
8880          */
8881         if (radeon_crtc->base.enabled && mode) {
8882                 if (mode->crtc_hdisplay < 1920) {
8883                         tmp = 1;
8884                         buffer_alloc = 2;
8885                 } else if (mode->crtc_hdisplay < 2560) {
8886                         tmp = 2;
8887                         buffer_alloc = 2;
8888                 } else if (mode->crtc_hdisplay < 4096) {
8889                         tmp = 0;
8890                         buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8891                 } else {
8892                         DRM_DEBUG_KMS("Mode too big for LB!\n");
8893                         tmp = 0;
8894                         buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8895                 }
8896         } else {
8897                 tmp = 1;
8898                 buffer_alloc = 0;
8899         }
8900
8901         WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
8902                LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
8903
8904         WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
8905                DMIF_BUFFERS_ALLOCATED(buffer_alloc));
8906         for (i = 0; i < rdev->usec_timeout; i++) {
8907                 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
8908                     DMIF_BUFFERS_ALLOCATED_COMPLETED)
8909                         break;
8910                 udelay(1);
8911         }
8912
8913         if (radeon_crtc->base.enabled && mode) {
8914                 switch (tmp) {
8915                 case 0:
8916                 default:
8917                         return 4096 * 2;
8918                 case 1:
8919                         return 1920 * 2;
8920                 case 2:
8921                         return 2560 * 2;
8922                 }
8923         }
8924
8925         /* controller not enabled, so no lb used */
8926         return 0;
8927 }
8928
8929 /**
8930  * cik_get_number_of_dram_channels - get the number of dram channels
8931  *
8932  * @rdev: radeon_device pointer
8933  *
8934  * Look up the number of video ram channels (CIK).
8935  * Used for display watermark bandwidth calculations
8936  * Returns the number of dram channels
8937  */
8938 static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
8939 {
8940         u32 tmp = RREG32(MC_SHARED_CHMAP);
8941
8942         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
8943         case 0:
8944         default:
8945                 return 1;
8946         case 1:
8947                 return 2;
8948         case 2:
8949                 return 4;
8950         case 3:
8951                 return 8;
8952         case 4:
8953                 return 3;
8954         case 5:
8955                 return 6;
8956         case 6:
8957                 return 10;
8958         case 7:
8959                 return 12;
8960         case 8:
8961                 return 16;
8962         }
8963 }
8964
8965 struct dce8_wm_params {
8966         u32 dram_channels; /* number of dram channels */
8967         u32 yclk;          /* bandwidth per dram data pin in kHz */
8968         u32 sclk;          /* engine clock in kHz */
8969         u32 disp_clk;      /* display clock in kHz */
8970         u32 src_width;     /* viewport width */
8971         u32 active_time;   /* active display time in ns */
8972         u32 blank_time;    /* blank time in ns */
8973         bool interlaced;    /* mode is interlaced */
8974         fixed20_12 vsc;    /* vertical scale ratio */
8975         u32 num_heads;     /* number of active crtcs */
8976         u32 bytes_per_pixel; /* bytes per pixel display + overlay */
8977         u32 lb_size;       /* line buffer allocated to pipe */
8978         u32 vtaps;         /* vertical scaler taps */
8979 };
8980
8981 /**
8982  * dce8_dram_bandwidth - get the dram bandwidth
8983  *
8984  * @wm: watermark calculation data
8985  *
8986  * Calculate the raw dram bandwidth (CIK).
8987  * Used for display watermark bandwidth calculations
8988  * Returns the dram bandwidth in MBytes/s
8989  */
8990 static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
8991 {
8992         /* Calculate raw DRAM Bandwidth */
8993         fixed20_12 dram_efficiency; /* 0.7 */
8994         fixed20_12 yclk, dram_channels, bandwidth;
8995         fixed20_12 a;
8996
8997         a.full = dfixed_const(1000);
8998         yclk.full = dfixed_const(wm->yclk);
8999         yclk.full = dfixed_div(yclk, a);
9000         dram_channels.full = dfixed_const(wm->dram_channels * 4);
9001         a.full = dfixed_const(10);
9002         dram_efficiency.full = dfixed_const(7);
9003         dram_efficiency.full = dfixed_div(dram_efficiency, a);
9004         bandwidth.full = dfixed_mul(dram_channels, yclk);
9005         bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
9006
9007         return dfixed_trunc(bandwidth);
9008 }
9009
9010 /**
9011  * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
9012  *
9013  * @wm: watermark calculation data
9014  *
9015  * Calculate the dram bandwidth used for display (CIK).
9016  * Used for display watermark bandwidth calculations
9017  * Returns the dram bandwidth for display in MBytes/s
9018  */
9019 static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
9020 {
9021         /* Calculate DRAM Bandwidth and the part allocated to display. */
9022         fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
9023         fixed20_12 yclk, dram_channels, bandwidth;
9024         fixed20_12 a;
9025
9026         a.full = dfixed_const(1000);
9027         yclk.full = dfixed_const(wm->yclk);
9028         yclk.full = dfixed_div(yclk, a);
9029         dram_channels.full = dfixed_const(wm->dram_channels * 4);
9030         a.full = dfixed_const(10);
9031         disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
9032         disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
9033         bandwidth.full = dfixed_mul(dram_channels, yclk);
9034         bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
9035
9036         return dfixed_trunc(bandwidth);
9037 }
9038
9039 /**
9040  * dce8_data_return_bandwidth - get the data return bandwidth
9041  *
9042  * @wm: watermark calculation data
9043  *
9044  * Calculate the data return bandwidth used for display (CIK).
9045  * Used for display watermark bandwidth calculations
9046  * Returns the data return bandwidth in MBytes/s
9047  */
9048 static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
9049 {
9050         /* Calculate the display Data return Bandwidth */
9051         fixed20_12 return_efficiency; /* 0.8 */
9052         fixed20_12 sclk, bandwidth;
9053         fixed20_12 a;
9054
9055         a.full = dfixed_const(1000);
9056         sclk.full = dfixed_const(wm->sclk);
9057         sclk.full = dfixed_div(sclk, a);
9058         a.full = dfixed_const(10);
9059         return_efficiency.full = dfixed_const(8);
9060         return_efficiency.full = dfixed_div(return_efficiency, a);
9061         a.full = dfixed_const(32);
9062         bandwidth.full = dfixed_mul(a, sclk);
9063         bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
9064
9065         return dfixed_trunc(bandwidth);
9066 }
9067
9068 /**
9069  * dce8_dmif_request_bandwidth - get the dmif bandwidth
9070  *
9071  * @wm: watermark calculation data
9072  *
9073  * Calculate the dmif bandwidth used for display (CIK).
9074  * Used for display watermark bandwidth calculations
9075  * Returns the dmif bandwidth in MBytes/s
9076  */
9077 static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
9078 {
9079         /* Calculate the DMIF Request Bandwidth */
9080         fixed20_12 disp_clk_request_efficiency; /* 0.8 */
9081         fixed20_12 disp_clk, bandwidth;
9082         fixed20_12 a, b;
9083
9084         a.full = dfixed_const(1000);
9085         disp_clk.full = dfixed_const(wm->disp_clk);
9086         disp_clk.full = dfixed_div(disp_clk, a);
9087         a.full = dfixed_const(32);
9088         b.full = dfixed_mul(a, disp_clk);
9089
9090         a.full = dfixed_const(10);
9091         disp_clk_request_efficiency.full = dfixed_const(8);
9092         disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
9093
9094         bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
9095
9096         return dfixed_trunc(bandwidth);
9097 }
9098
9099 /**
9100  * dce8_available_bandwidth - get the min available bandwidth
9101  *
9102  * @wm: watermark calculation data
9103  *
9104  * Calculate the min available bandwidth used for display (CIK).
9105  * Used for display watermark bandwidth calculations
9106  * Returns the min available bandwidth in MBytes/s
9107  */
9108 static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
9109 {
9110         /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
9111         u32 dram_bandwidth = dce8_dram_bandwidth(wm);
9112         u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
9113         u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
9114
9115         return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
9116 }
9117
9118 /**
9119  * dce8_average_bandwidth - get the average available bandwidth
9120  *
9121  * @wm: watermark calculation data
9122  *
9123  * Calculate the average available bandwidth used for display (CIK).
9124  * Used for display watermark bandwidth calculations
9125  * Returns the average available bandwidth in MBytes/s
9126  */
9127 static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
9128 {
9129         /* Calculate the display mode Average Bandwidth
9130          * DisplayMode should contain the source and destination dimensions,
9131          * timing, etc.
9132          */
9133         fixed20_12 bpp;
9134         fixed20_12 line_time;
9135         fixed20_12 src_width;
9136         fixed20_12 bandwidth;
9137         fixed20_12 a;
9138
9139         a.full = dfixed_const(1000);
9140         line_time.full = dfixed_const(wm->active_time + wm->blank_time);
9141         line_time.full = dfixed_div(line_time, a);
9142         bpp.full = dfixed_const(wm->bytes_per_pixel);
9143         src_width.full = dfixed_const(wm->src_width);
9144         bandwidth.full = dfixed_mul(src_width, bpp);
9145         bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
9146         bandwidth.full = dfixed_div(bandwidth, line_time);
9147
9148         return dfixed_trunc(bandwidth);
9149 }
9150
9151 /**
9152  * dce8_latency_watermark - get the latency watermark
9153  *
9154  * @wm: watermark calculation data
9155  *
9156  * Calculate the latency watermark (CIK).
9157  * Used for display watermark bandwidth calculations
9158  * Returns the latency watermark in ns
9159  */
9160 static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
9161 {
9162         /* First calculate the latency in ns */
9163         u32 mc_latency = 2000; /* 2000 ns. */
9164         u32 available_bandwidth = dce8_available_bandwidth(wm);
9165         u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
9166         u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
9167         u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
9168         u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
9169                 (wm->num_heads * cursor_line_pair_return_time);
9170         u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
9171         u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
9172         u32 tmp, dmif_size = 12288;
9173         fixed20_12 a, b, c;
9174
9175         if (wm->num_heads == 0)
9176                 return 0;
9177
9178         a.full = dfixed_const(2);
9179         b.full = dfixed_const(1);
9180         if ((wm->vsc.full > a.full) ||
9181             ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
9182             (wm->vtaps >= 5) ||
9183             ((wm->vsc.full >= a.full) && wm->interlaced))
9184                 max_src_lines_per_dst_line = 4;
9185         else
9186                 max_src_lines_per_dst_line = 2;
9187
9188         a.full = dfixed_const(available_bandwidth);
9189         b.full = dfixed_const(wm->num_heads);
9190         a.full = dfixed_div(a, b);
9191
9192         b.full = dfixed_const(mc_latency + 512);
9193         c.full = dfixed_const(wm->disp_clk);
9194         b.full = dfixed_div(b, c);
9195
9196         c.full = dfixed_const(dmif_size);
9197         b.full = dfixed_div(c, b);
9198
9199         tmp = min(dfixed_trunc(a), dfixed_trunc(b));
9200
9201         b.full = dfixed_const(1000);
9202         c.full = dfixed_const(wm->disp_clk);
9203         b.full = dfixed_div(c, b);
9204         c.full = dfixed_const(wm->bytes_per_pixel);
9205         b.full = dfixed_mul(b, c);
9206
9207         lb_fill_bw = min(tmp, dfixed_trunc(b));
9208
9209         a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
9210         b.full = dfixed_const(1000);
9211         c.full = dfixed_const(lb_fill_bw);
9212         b.full = dfixed_div(c, b);
9213         a.full = dfixed_div(a, b);
9214         line_fill_time = dfixed_trunc(a);
9215
9216         if (line_fill_time < wm->active_time)
9217                 return latency;
9218         else
9219                 return latency + (line_fill_time - wm->active_time);
9220
9221 }
9222
9223 /**
9224  * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
9225  * average and available dram bandwidth
9226  *
9227  * @wm: watermark calculation data
9228  *
9229  * Check if the display average bandwidth fits in the display
9230  * dram bandwidth (CIK).
9231  * Used for display watermark bandwidth calculations
9232  * Returns true if the display fits, false if not.
9233  */
9234 static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
9235 {
9236         if (dce8_average_bandwidth(wm) <=
9237             (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
9238                 return true;
9239         else
9240                 return false;
9241 }
9242
9243 /**
9244  * dce8_average_bandwidth_vs_available_bandwidth - check
9245  * average and available bandwidth
9246  *
9247  * @wm: watermark calculation data
9248  *
9249  * Check if the display average bandwidth fits in the display
9250  * available bandwidth (CIK).
9251  * Used for display watermark bandwidth calculations
9252  * Returns true if the display fits, false if not.
9253  */
9254 static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
9255 {
9256         if (dce8_average_bandwidth(wm) <=
9257             (dce8_available_bandwidth(wm) / wm->num_heads))
9258                 return true;
9259         else
9260                 return false;
9261 }
9262
9263 /**
9264  * dce8_check_latency_hiding - check latency hiding
9265  *
9266  * @wm: watermark calculation data
9267  *
9268  * Check latency hiding (CIK).
9269  * Used for display watermark bandwidth calculations
9270  * Returns true if the display fits, false if not.
9271  */
9272 static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
9273 {
9274         u32 lb_partitions = wm->lb_size / wm->src_width;
9275         u32 line_time = wm->active_time + wm->blank_time;
9276         u32 latency_tolerant_lines;
9277         u32 latency_hiding;
9278         fixed20_12 a;
9279
9280         a.full = dfixed_const(1);
9281         if (wm->vsc.full > a.full)
9282                 latency_tolerant_lines = 1;
9283         else {
9284                 if (lb_partitions <= (wm->vtaps + 1))
9285                         latency_tolerant_lines = 1;
9286                 else
9287                         latency_tolerant_lines = 2;
9288         }
9289
9290         latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
9291
9292         if (dce8_latency_watermark(wm) <= latency_hiding)
9293                 return true;
9294         else
9295                 return false;
9296 }
9297
9298 /**
9299  * dce8_program_watermarks - program display watermarks
9300  *
9301  * @rdev: radeon_device pointer
9302  * @radeon_crtc: the selected display controller
9303  * @lb_size: line buffer size
9304  * @num_heads: number of display controllers in use
9305  *
9306  * Calculate and program the display watermarks for the
9307  * selected display controller (CIK).
9308  */
9309 static void dce8_program_watermarks(struct radeon_device *rdev,
9310                                     struct radeon_crtc *radeon_crtc,
9311                                     u32 lb_size, u32 num_heads)
9312 {
9313         struct drm_display_mode *mode = &radeon_crtc->base.mode;
9314         struct dce8_wm_params wm_low, wm_high;
9315         u32 pixel_period;
9316         u32 line_time = 0;
9317         u32 latency_watermark_a = 0, latency_watermark_b = 0;
9318         u32 tmp, wm_mask;
9319
9320         if (radeon_crtc->base.enabled && num_heads && mode) {
9321                 pixel_period = 1000000 / (u32)mode->clock;
9322                 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
9323
9324                 /* watermark for high clocks */
9325                 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
9326                     rdev->pm.dpm_enabled) {
9327                         wm_high.yclk =
9328                                 radeon_dpm_get_mclk(rdev, false) * 10;
9329                         wm_high.sclk =
9330                                 radeon_dpm_get_sclk(rdev, false) * 10;
9331                 } else {
9332                         wm_high.yclk = rdev->pm.current_mclk * 10;
9333                         wm_high.sclk = rdev->pm.current_sclk * 10;
9334                 }
9335
9336                 wm_high.disp_clk = mode->clock;
9337                 wm_high.src_width = mode->crtc_hdisplay;
9338                 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
9339                 wm_high.blank_time = line_time - wm_high.active_time;
9340                 wm_high.interlaced = false;
9341                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
9342                         wm_high.interlaced = true;
9343                 wm_high.vsc = radeon_crtc->vsc;
9344                 wm_high.vtaps = 1;
9345                 if (radeon_crtc->rmx_type != RMX_OFF)
9346                         wm_high.vtaps = 2;
9347                 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
9348                 wm_high.lb_size = lb_size;
9349                 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev);
9350                 wm_high.num_heads = num_heads;
9351
9352                 /* set for high clocks */
9353                 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535);
9354
9355                 /* possibly force display priority to high */
9356                 /* should really do this at mode validation time... */
9357                 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
9358                     !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) ||
9359                     !dce8_check_latency_hiding(&wm_high) ||
9360                     (rdev->disp_priority == 2)) {
9361                         DRM_DEBUG_KMS("force priority to high\n");
9362                 }
9363
9364                 /* watermark for low clocks */
9365                 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
9366                     rdev->pm.dpm_enabled) {
9367                         wm_low.yclk =
9368                                 radeon_dpm_get_mclk(rdev, true) * 10;
9369                         wm_low.sclk =
9370                                 radeon_dpm_get_sclk(rdev, true) * 10;
9371                 } else {
9372                         wm_low.yclk = rdev->pm.current_mclk * 10;
9373                         wm_low.sclk = rdev->pm.current_sclk * 10;
9374                 }
9375
9376                 wm_low.disp_clk = mode->clock;
9377                 wm_low.src_width = mode->crtc_hdisplay;
9378                 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
9379                 wm_low.blank_time = line_time - wm_low.active_time;
9380                 wm_low.interlaced = false;
9381                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
9382                         wm_low.interlaced = true;
9383                 wm_low.vsc = radeon_crtc->vsc;
9384                 wm_low.vtaps = 1;
9385                 if (radeon_crtc->rmx_type != RMX_OFF)
9386                         wm_low.vtaps = 2;
9387                 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
9388                 wm_low.lb_size = lb_size;
9389                 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev);
9390                 wm_low.num_heads = num_heads;
9391
9392                 /* set for low clocks */
9393                 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535);
9394
9395                 /* possibly force display priority to high */
9396                 /* should really do this at mode validation time... */
9397                 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
9398                     !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) ||
9399                     !dce8_check_latency_hiding(&wm_low) ||
9400                     (rdev->disp_priority == 2)) {
9401                         DRM_DEBUG_KMS("force priority to high\n");
9402                 }
9403         }
9404
9405         /* select wm A */
9406         wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9407         tmp = wm_mask;
9408         tmp &= ~LATENCY_WATERMARK_MASK(3);
9409         tmp |= LATENCY_WATERMARK_MASK(1);
9410         WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9411         WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9412                (LATENCY_LOW_WATERMARK(latency_watermark_a) |
9413                 LATENCY_HIGH_WATERMARK(line_time)));
9414         /* select wm B */
9415         tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9416         tmp &= ~LATENCY_WATERMARK_MASK(3);
9417         tmp |= LATENCY_WATERMARK_MASK(2);
9418         WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9419         WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9420                (LATENCY_LOW_WATERMARK(latency_watermark_b) |
9421                 LATENCY_HIGH_WATERMARK(line_time)));
9422         /* restore original selection */
9423         WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
9424
9425         /* save values for DPM */
9426         radeon_crtc->line_time = line_time;
9427         radeon_crtc->wm_high = latency_watermark_a;
9428         radeon_crtc->wm_low = latency_watermark_b;
9429 }
9430
9431 /**
9432  * dce8_bandwidth_update - program display watermarks
9433  *
9434  * @rdev: radeon_device pointer
9435  *
9436  * Calculate and program the display watermarks and line
9437  * buffer allocation (CIK).
9438  */
9439 void dce8_bandwidth_update(struct radeon_device *rdev)
9440 {
9441         struct drm_display_mode *mode = NULL;
9442         u32 num_heads = 0, lb_size;
9443         int i;
9444
9445         radeon_update_display_priority(rdev);
9446
9447         for (i = 0; i < rdev->num_crtc; i++) {
9448                 if (rdev->mode_info.crtcs[i]->base.enabled)
9449                         num_heads++;
9450         }
9451         for (i = 0; i < rdev->num_crtc; i++) {
9452                 mode = &rdev->mode_info.crtcs[i]->base.mode;
9453                 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
9454                 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
9455         }
9456 }
9457
9458 /**
9459  * cik_get_gpu_clock_counter - return GPU clock counter snapshot
9460  *
9461  * @rdev: radeon_device pointer
9462  *
9463  * Fetches a GPU clock counter snapshot (SI).
9464  * Returns the 64 bit clock counter snapshot.
9465  */
9466 uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
9467 {
9468         uint64_t clock;
9469
9470         mutex_lock(&rdev->gpu_clock_mutex);
9471         WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
9472         clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
9473                 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
9474         mutex_unlock(&rdev->gpu_clock_mutex);
9475         return clock;
9476 }
9477
9478 static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
9479                               u32 cntl_reg, u32 status_reg)
9480 {
9481         int r, i;
9482         struct atom_clock_dividers dividers;
9483         uint32_t tmp;
9484
9485         r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9486                                            clock, false, &dividers);
9487         if (r)
9488                 return r;
9489
9490         tmp = RREG32_SMC(cntl_reg);
9491         tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
9492         tmp |= dividers.post_divider;
9493         WREG32_SMC(cntl_reg, tmp);
9494
9495         for (i = 0; i < 100; i++) {
9496                 if (RREG32_SMC(status_reg) & DCLK_STATUS)
9497                         break;
9498                 mdelay(10);
9499         }
9500         if (i == 100)
9501                 return -ETIMEDOUT;
9502
9503         return 0;
9504 }
9505
9506 int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
9507 {
9508         int r = 0;
9509
9510         r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
9511         if (r)
9512                 return r;
9513
9514         r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
9515         return r;
9516 }
9517
9518 int cik_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
9519 {
9520         int r, i;
9521         struct atom_clock_dividers dividers;
9522         u32 tmp;
9523
9524         r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9525                                            ecclk, false, &dividers);
9526         if (r)
9527                 return r;
9528
9529         for (i = 0; i < 100; i++) {
9530                 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9531                         break;
9532                 mdelay(10);
9533         }
9534         if (i == 100)
9535                 return -ETIMEDOUT;
9536
9537         tmp = RREG32_SMC(CG_ECLK_CNTL);
9538         tmp &= ~(ECLK_DIR_CNTL_EN|ECLK_DIVIDER_MASK);
9539         tmp |= dividers.post_divider;
9540         WREG32_SMC(CG_ECLK_CNTL, tmp);
9541
9542         for (i = 0; i < 100; i++) {
9543                 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9544                         break;
9545                 mdelay(10);
9546         }
9547         if (i == 100)
9548                 return -ETIMEDOUT;
9549
9550         return 0;
9551 }
9552
9553 static void cik_pcie_gen3_enable(struct radeon_device *rdev)
9554 {
9555         struct pci_dev *root = rdev->pdev->bus->self;
9556         int bridge_pos, gpu_pos;
9557         u32 speed_cntl, mask, current_data_rate;
9558         int ret, i;
9559         u16 tmp16;
9560
9561         if (radeon_pcie_gen2 == 0)
9562                 return;
9563
9564         if (rdev->flags & RADEON_IS_IGP)
9565                 return;
9566
9567         if (!(rdev->flags & RADEON_IS_PCIE))
9568                 return;
9569
9570         ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
9571         if (ret != 0)
9572                 return;
9573
9574         if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
9575                 return;
9576
9577         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9578         current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
9579                 LC_CURRENT_DATA_RATE_SHIFT;
9580         if (mask & DRM_PCIE_SPEED_80) {
9581                 if (current_data_rate == 2) {
9582                         DRM_INFO("PCIE gen 3 link speeds already enabled\n");
9583                         return;
9584                 }
9585                 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
9586         } else if (mask & DRM_PCIE_SPEED_50) {
9587                 if (current_data_rate == 1) {
9588                         DRM_INFO("PCIE gen 2 link speeds already enabled\n");
9589                         return;
9590                 }
9591                 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
9592         }
9593
9594         bridge_pos = pci_pcie_cap(root);
9595         if (!bridge_pos)
9596                 return;
9597
9598         gpu_pos = pci_pcie_cap(rdev->pdev);
9599         if (!gpu_pos)
9600                 return;
9601
9602         if (mask & DRM_PCIE_SPEED_80) {
9603                 /* re-try equalization if gen3 is not already enabled */
9604                 if (current_data_rate != 2) {
9605                         u16 bridge_cfg, gpu_cfg;
9606                         u16 bridge_cfg2, gpu_cfg2;
9607                         u32 max_lw, current_lw, tmp;
9608
9609                         pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9610                         pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9611
9612                         tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
9613                         pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9614
9615                         tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
9616                         pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9617
9618                         tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9619                         max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
9620                         current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
9621
9622                         if (current_lw < max_lw) {
9623                                 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9624                                 if (tmp & LC_RENEGOTIATION_SUPPORT) {
9625                                         tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
9626                                         tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
9627                                         tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
9628                                         WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
9629                                 }
9630                         }
9631
9632                         for (i = 0; i < 10; i++) {
9633                                 /* check status */
9634                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
9635                                 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
9636                                         break;
9637
9638                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9639                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9640
9641                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
9642                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
9643
9644                                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9645                                 tmp |= LC_SET_QUIESCE;
9646                                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9647
9648                                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9649                                 tmp |= LC_REDO_EQ;
9650                                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9651
9652                                 mdelay(100);
9653
9654                                 /* linkctl */
9655                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
9656                                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9657                                 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
9658                                 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9659
9660                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
9661                                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9662                                 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
9663                                 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9664
9665                                 /* linkctl2 */
9666                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
9667                                 tmp16 &= ~((1 << 4) | (7 << 9));
9668                                 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
9669                                 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
9670
9671                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9672                                 tmp16 &= ~((1 << 4) | (7 << 9));
9673                                 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
9674                                 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9675
9676                                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9677                                 tmp &= ~LC_SET_QUIESCE;
9678                                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9679                         }
9680                 }
9681         }
9682
9683         /* set the link speed */
9684         speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
9685         speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
9686         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9687
9688         pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9689         tmp16 &= ~0xf;
9690         if (mask & DRM_PCIE_SPEED_80)
9691                 tmp16 |= 3; /* gen3 */
9692         else if (mask & DRM_PCIE_SPEED_50)
9693                 tmp16 |= 2; /* gen2 */
9694         else
9695                 tmp16 |= 1; /* gen1 */
9696         pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9697
9698         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9699         speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
9700         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9701
9702         for (i = 0; i < rdev->usec_timeout; i++) {
9703                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9704                 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
9705                         break;
9706                 udelay(1);
9707         }
9708 }
9709
9710 static void cik_program_aspm(struct radeon_device *rdev)
9711 {
9712         u32 data, orig;
9713         bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
9714         bool disable_clkreq = false;
9715
9716         if (radeon_aspm == 0)
9717                 return;
9718
9719         /* XXX double check IGPs */
9720         if (rdev->flags & RADEON_IS_IGP)
9721                 return;
9722
9723         if (!(rdev->flags & RADEON_IS_PCIE))
9724                 return;
9725
9726         orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9727         data &= ~LC_XMIT_N_FTS_MASK;
9728         data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
9729         if (orig != data)
9730                 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
9731
9732         orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
9733         data |= LC_GO_TO_RECOVERY;
9734         if (orig != data)
9735                 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
9736
9737         orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL);
9738         data |= P_IGNORE_EDB_ERR;
9739         if (orig != data)
9740                 WREG32_PCIE_PORT(PCIE_P_CNTL, data);
9741
9742         orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9743         data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
9744         data |= LC_PMI_TO_L1_DIS;
9745         if (!disable_l0s)
9746                 data |= LC_L0S_INACTIVITY(7);
9747
9748         if (!disable_l1) {
9749                 data |= LC_L1_INACTIVITY(7);
9750                 data &= ~LC_PMI_TO_L1_DIS;
9751                 if (orig != data)
9752                         WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9753
9754                 if (!disable_plloff_in_l1) {
9755                         bool clk_req_support;
9756
9757                         orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0);
9758                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9759                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9760                         if (orig != data)
9761                                 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data);
9762
9763                         orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1);
9764                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9765                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9766                         if (orig != data)
9767                                 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data);
9768
9769                         orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0);
9770                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9771                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9772                         if (orig != data)
9773                                 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data);
9774
9775                         orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1);
9776                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9777                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9778                         if (orig != data)
9779                                 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data);
9780
9781                         orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9782                         data &= ~LC_DYN_LANES_PWR_STATE_MASK;
9783                         data |= LC_DYN_LANES_PWR_STATE(3);
9784                         if (orig != data)
9785                                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
9786
9787                         if (!disable_clkreq) {
9788                                 struct pci_dev *root = rdev->pdev->bus->self;
9789                                 u32 lnkcap;
9790
9791                                 clk_req_support = false;
9792                                 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
9793                                 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
9794                                         clk_req_support = true;
9795                         } else {
9796                                 clk_req_support = false;
9797                         }
9798
9799                         if (clk_req_support) {
9800                                 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
9801                                 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
9802                                 if (orig != data)
9803                                         WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
9804
9805                                 orig = data = RREG32_SMC(THM_CLK_CNTL);
9806                                 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
9807                                 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
9808                                 if (orig != data)
9809                                         WREG32_SMC(THM_CLK_CNTL, data);
9810
9811                                 orig = data = RREG32_SMC(MISC_CLK_CTRL);
9812                                 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
9813                                 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
9814                                 if (orig != data)
9815                                         WREG32_SMC(MISC_CLK_CTRL, data);
9816
9817                                 orig = data = RREG32_SMC(CG_CLKPIN_CNTL);
9818                                 data &= ~BCLK_AS_XCLK;
9819                                 if (orig != data)
9820                                         WREG32_SMC(CG_CLKPIN_CNTL, data);
9821
9822                                 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2);
9823                                 data &= ~FORCE_BIF_REFCLK_EN;
9824                                 if (orig != data)
9825                                         WREG32_SMC(CG_CLKPIN_CNTL_2, data);
9826
9827                                 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL);
9828                                 data &= ~MPLL_CLKOUT_SEL_MASK;
9829                                 data |= MPLL_CLKOUT_SEL(4);
9830                                 if (orig != data)
9831                                         WREG32_SMC(MPLL_BYPASSCLK_SEL, data);
9832                         }
9833                 }
9834         } else {
9835                 if (orig != data)
9836                         WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9837         }
9838
9839         orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
9840         data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
9841         if (orig != data)
9842                 WREG32_PCIE_PORT(PCIE_CNTL2, data);
9843
9844         if (!disable_l0s) {
9845                 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9846                 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
9847                         data = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9848                         if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
9849                                 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9850                                 data &= ~LC_L0S_INACTIVITY_MASK;
9851                                 if (orig != data)
9852                                         WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9853                         }
9854                 }
9855         }
9856 }